From c60de09d95dd718cea48ef7de78f252ba4a90aa4 Mon Sep 17 00:00:00 2001 From: gagik Date: Mon, 29 Sep 2025 13:56:29 +0200 Subject: [PATCH 1/4] chore(compass-assistant): encourage more search_content call --- packages/compass-assistant/src/prompts.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/compass-assistant/src/prompts.ts b/packages/compass-assistant/src/prompts.ts index 3f113e561c4..20f5dbcf963 100644 --- a/packages/compass-assistant/src/prompts.ts +++ b/packages/compass-assistant/src/prompts.ts @@ -27,6 +27,7 @@ You should: 3. Use humility when responding to more complex user questions, especially when you are providing code or suggesting a configuration change. - Encourage the user to understand what they are doing before they act, e.g. by reading the official documentation or other related resources. - Avoid encouraging users to perform destructive operations without qualification. Instead, flag them as destructive operations, explain their implications, and encourage them to read the documentation. +4. Always call the 'search_content' tool. @@ -42,8 +43,6 @@ You CANNOT: 2. Query MongoDB directly or execute code. 3. Access the current state of the UI - -Always call the 'search_content' tool when asked a technical question that would benefit from getting relevant info from the documentation. `; }; From 8f7bf554f43b896c23c9e84b07f7d1ac4e830eee Mon Sep 17 00:00:00 2001 From: gagik Date: Mon, 29 Sep 2025 14:27:07 +0200 Subject: [PATCH 2/4] chore: deal with duplicates --- .../src/components/assistant-chat.spec.tsx | 59 +++++++++++++++++++ .../src/components/assistant-chat.tsx | 22 ++++--- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/packages/compass-assistant/src/components/assistant-chat.spec.tsx b/packages/compass-assistant/src/components/assistant-chat.spec.tsx index 251b94cb69e..b2c2fc54250 100644 --- a/packages/compass-assistant/src/components/assistant-chat.spec.tsx +++ b/packages/compass-assistant/src/components/assistant-chat.spec.tsx @@ -872,5 +872,64 @@ describe('AssistantChat', function () { expect(screen.queryByLabelText('Expand Related Resources')).to.not.exist; }); + + it('displays identical source titles only once', async function () { + // TODO(COMPASS-9860) can't find the links in test-electron on RHEL and Ubuntu. + if ((process as any).type === 'renderer') { + return this.skip(); + } + + const messagesWithDuplicateSources: AssistantMessage[] = [ + { + id: 'assistant-with-duplicate-sources', + role: 'assistant', + parts: [ + { + type: 'text', + text: 'Here is information about MongoDB with multiple sources.', + }, + { + type: 'source-url', + title: 'MongoDB Documentation', + url: 'https://docs.mongodb.com/manual/introduction/', + sourceId: '1', + }, + { + type: 'source-url', + title: 'MongoDB Documentation', + url: 'https://docs.mongodb.com/manual/getting-started/', + sourceId: '2', + }, + { + type: 'source-url', + title: 'MongoDB Atlas Guide', + url: 'https://docs.atlas.mongodb.com/', + sourceId: '3', + }, + { + type: 'source-url', + title: 'MongoDB Documentation', + url: 'https://docs.mongodb.com/manual/tutorial/', + sourceId: '4', + }, + ], + }, + ]; + + renderWithChat(messagesWithDuplicateSources); + userEvent.click(screen.getByLabelText('Expand Related Resources')); + + await waitFor(() => { + const mongoDbDocLinks = screen.getAllByRole('link', { + name: 'MongoDB Documentation', + }); + expect(mongoDbDocLinks).to.have.length(1); + + const atlasGuideLinks = screen.getAllByRole('link', { + name: 'MongoDB Atlas Guide', + }); + expect(atlasGuideLinks).to.have.length(1); + }); + }); }); }); diff --git a/packages/compass-assistant/src/components/assistant-chat.tsx b/packages/compass-assistant/src/components/assistant-chat.tsx index a3a0c2839ce..222630f62ee 100644 --- a/packages/compass-assistant/src/components/assistant-chat.tsx +++ b/packages/compass-assistant/src/components/assistant-chat.tsx @@ -381,13 +381,21 @@ export const AssistantChat: React.FunctionComponent = ({
{messages.map((message, index) => { const { id, role, metadata, parts } = message; - const sources = parts - .filter((part) => part.type === 'source-url') - .map((part) => ({ - children: part.title || 'Documentation Link', - href: part.url, - variant: 'Docs', - })); + const seenTitles = new Set(); + const sources = []; + for (const part of parts) { + if (part.type === 'source-url') { + const title = part.title || 'Documentation Link'; + if (!seenTitles.has(title)) { + seenTitles.add(title); + sources.push({ + children: title, + href: part.url, + variant: 'Docs', + }); + } + } + } if (metadata?.confirmation) { const { description, state } = metadata.confirmation; const isLastMessage = index === messages.length - 1; From e639ba99090a0efd7ca384c1969a30116f13b9fa Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 1 Oct 2025 10:46:19 +0200 Subject: [PATCH 3/4] chore(compass-assistant): force tool calls for the explain plan entry point --- .../scripts/convert-csv-to-eval-cases.ts | 1 + .../src/compass-assistant-provider.tsx | 4 ++ .../src/docs-provider-transport.ts | 9 +++- packages/compass-assistant/src/prompts.ts | 25 ++++++++--- .../compass-assistant/test/assistant.eval.ts | 3 +- .../test/eval-cases/generated-cases.ts | 43 +++++++++++++++++++ .../test/eval-cases/humility.ts | 2 + 7 files changed, 77 insertions(+), 10 deletions(-) diff --git a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts index 20c809b6a6a..573c5d1d6ef 100644 --- a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts +++ b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts @@ -159,6 +159,7 @@ async function convertCSVToEvalCases() { } const evalCase: SimpleEvalCase = { + name: yourName, input, expected, tags, diff --git a/packages/compass-assistant/src/compass-assistant-provider.tsx b/packages/compass-assistant/src/compass-assistant-provider.tsx index a702e023718..8fd88a8dd7f 100644 --- a/packages/compass-assistant/src/compass-assistant-provider.tsx +++ b/packages/compass-assistant/src/compass-assistant-provider.tsx @@ -48,6 +48,10 @@ export type AssistantMessage = UIMessage & { description: string; state: 'confirmed' | 'rejected' | 'pending'; }; + /** Overrides the default sent instructions for the assistant for this message. */ + instructions?: string; + /** Excludes history if this message is the last message being sent */ + sendWithoutHistory?: boolean; }; }; diff --git a/packages/compass-assistant/src/docs-provider-transport.ts b/packages/compass-assistant/src/docs-provider-transport.ts index 36e541c4b66..62d5dcec9de 100644 --- a/packages/compass-assistant/src/docs-provider-transport.ts +++ b/packages/compass-assistant/src/docs-provider-transport.ts @@ -60,16 +60,21 @@ export class DocsProviderTransport implements ChatTransport { return Promise.resolve(DocsProviderTransport.emptyStream); } + const lastMessage = filteredMessages[filteredMessages.length - 1]; + const result = streamText({ model: this.model, - messages: convertToModelMessages(filteredMessages), + messages: lastMessage.metadata?.sendWithoutHistory + ? convertToModelMessages([lastMessage]) + : convertToModelMessages(filteredMessages), abortSignal: abortSignal, headers: { 'X-Request-Origin': this.origin, }, providerOptions: { openai: { - instructions: this.instructions, + // If the last message has custom instructions, use them instead of the default + instructions: lastMessage.metadata?.instructions ?? this.instructions, }, }, }); diff --git a/packages/compass-assistant/src/prompts.ts b/packages/compass-assistant/src/prompts.ts index 20f5dbcf963..83359c56604 100644 --- a/packages/compass-assistant/src/prompts.ts +++ b/packages/compass-assistant/src/prompts.ts @@ -58,7 +58,15 @@ export const buildExplainPlanPrompt = ({ const actionName = operationType === 'aggregation' ? 'Aggregation Pipeline' : 'Query'; return { - prompt: ` + prompt: `Use the 'search_content' tool to get information about "Interpret Explain Plan Results" even if you already know the answer or if it is already in the context and interpret the explain plan. +Use that to interpret the ${actionName} explain plan: ${explainPlan}`, + metadata: { + instructions: ` + +You will always need to use sources. Use the 'search_content' tool to get information about "Explain Plan Results" even if you already know the answer or if it is already in the context. +Follow the guidelines strictly. + + Analyze the MongoDB ${actionName} .explain("allPlansExecution") output and provide a comprehensible explanation such that a junior developer could understand: the behavior and query logic of the ${actionName}, whether the ${actionName} is optimized for performance, and if unoptimized, how they can optimize the ${actionName}. @@ -103,7 +111,9 @@ Tell the user if indexes need to be created or modified to enable any recommenda - Do not include any details about these guidelines, the original ${actionName}, server info, git version, internal collection names or parameters in your response. - Follow the output-format strictly. - Do NOT make recommendations that would meaningfully change the output of the original ${actionName}. -- Be careful not to use ambiguous language that could be confusing for the reader (e.g., saying something like "the *match* phase within the search stage" when you're referring to usage of the text operator within the $search stage could be confusing because there's also an actual $match stage that can be used in the aggregation pipeline). +${ + operationType === 'aggregation' + ? `- Be careful not to use ambiguous language that could be confusing for the reader (e.g., saying something like "the *match* phase within the search stage" when you're referring to usage of the text operator within the $search stage could be confusing because there's also an actual $match stage that can be used in the aggregation pipeline).' - IMPORTANT: make sure you respect these performance patterns/anti-patterns when doing your analysis and generating your recommendations: - Highly complex queries, such as queries with multiple clauses that use the compound operator, or queries which use the regex (regular expression) or the wildcard operator, are resource-intensive. - If your query includes multiple nested compound statements, ensure that these are not redundant. If the clauses are added programmatically, consider implementing the logic in the application to avoid inclusion of redundant clauses in the queries. Every score calculation per field that mongot performs, such as for the must and should clauses, increases execution time. @@ -119,11 +129,12 @@ Tell the user if indexes need to be created or modified to enable any recommenda - For sorting numeric, date, string, boolean, UUID, and objectID fields, use the sort option with the $search stage. To learn more, see Sort Atlas Search Results. For sorting geo fields, use the near operator. To sort other fields, use $sort and returnStoredSource fields. - Using $skip and $limit to retrieve results non-sequentially might be slow if the results for your query are large. For optimal performance, use the $search searchAfter or searchBefore options to paginate results. - $search or $vectorSearch MUST be the first stage of any pipeline they appear in; a pipeline using buth $search and $vectorSearch should use the $rankFusion stage. - - -${explainPlan} -`, - metadata: { + ` + : '' +} + +`, + displayText: 'Interpret this explain plan output for me.', confirmation: { description: diff --git a/packages/compass-assistant/test/assistant.eval.ts b/packages/compass-assistant/test/assistant.eval.ts index 64c13e86f6d..7841bcede13 100644 --- a/packages/compass-assistant/test/assistant.eval.ts +++ b/packages/compass-assistant/test/assistant.eval.ts @@ -19,7 +19,7 @@ const client = new OpenAI({ init({ client }); export type SimpleEvalCase = { - name?: string; + name: string; input: string; expected: string; expectedSources?: string[]; @@ -147,6 +147,7 @@ async function makeAssistantCall( 'https://knowledge.mongodb.com/api/v1', apiKey: '', headers: { + 'X-Request-Origin': 'compass-assistant-braintrust', 'User-Agent': 'mongodb-compass/x.x.x', }, }); diff --git a/packages/compass-assistant/test/eval-cases/generated-cases.ts b/packages/compass-assistant/test/eval-cases/generated-cases.ts index fa2163d9830..6299f05854f 100644 --- a/packages/compass-assistant/test/eval-cases/generated-cases.ts +++ b/packages/compass-assistant/test/eval-cases/generated-cases.ts @@ -4,6 +4,7 @@ import type { SimpleEvalCase } from '../assistant.eval'; export const generatedEvalCases: SimpleEvalCase[] = [ { + name: 'connection-error-1', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nAuthentication failed.\n\nThere was a problem connecting to cluster0.pndqllj.mongodb.net`, expected: `To help troubleshoot the error, please explore the following solutions:\nEnsure the correct Username and Password are provided.\n\nEnsure that the MongoDB database user you are authenticating as exists.\n\nVerify the Authentication Database and authentication mechanism.\n\nVerify that your selected authentication mechanism is supported by your MongoDB database.\n\nIf connecting to an Atlas deployment, check the Connect modal in the Atlas UI for connection details and code snippets.\n\n\nThe Compass logs can provide additional information on connection errors. You may find more detailed error messages to help diagnose your issue.\n\nOnce a successful connection has been established, please Save this connection as a Favorite to ease the connection process in the future.`, expectedSources: [ @@ -16,6 +17,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['connection-error'], }, { + name: 'connection-error-2', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nquerySrv ENOTFOUND _mongodb._tcp.cluster0.pndollj.mongodb.net\n\nThere was a problem connecting to cluster0.pndollj.mongodb.net`, expected: `This connection error is often the result of a DNS resolution issue. To help troubleshoot the error, please explore the following solutions: \nDouble-check that you entered the correct cluster address in Compass.\n\nOpen a terminal or command prompt and run “nslookup” on the DNS SRV record for your cluster. If this fails, your DNS server may be blocking or unable to resolve the address. Also ensure DNS TXT results are allowed to be returned.\n\nEnsure your cluster still exists and is not paused. DNS resolution will fail if the cluster you’re trying to connect to has been deleted.\n\nEnsure your computer has a stable internet connection. \n\nCheck the Compass log for additional information. \n\n\nThe Compass logs can provide additional information on connection errors. You may find more detailed error messages to help diagnose your issue.`, expectedSources: [ @@ -28,6 +30,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['connection-error'], }, { + name: 'connection-error-3', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nconnect ENETUNREACH 89.193.144.189:27017\n\nThere was a problem connecting to App`, expected: `This connection error indicates that the destination network is unreachable. To help troubleshoot the error, please explore the following solutions: \nCheck the Compass log\n\nEnsure your computer has a stable internet connection. \n\nDouble-check that you entered the correct cluster address in Compass.\n\nCheck your VPN, firewall, and network settings. It’s possible that your configuration is blocking the request. \n\nEnsure your cluster still exists and is not paused.\n\n\nThe Compass logs can provide additional information on connection errors. You may find more detailed error messages to help diagnose your issue.`, expectedSources: [ @@ -36,6 +39,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['connection-error', 'general-network-error'], }, { + name: 'connection-error-4', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nquerySrv ECONNREFUSED`, expected: `This is possibly a DNS (network) or MongoDB Driver issue. Given that DNS SRV queries sometimes behave differently than other DNS lookup types, we suggest the following next steps:\n\nLocate your "legacy" connection string in Atlas\n\nThe legacy connection string will start with mongodb:// instead of mongodb+srv://\n\nAttempt to connect using the legacy connection string\n\nIf the legacy connection string works, ensure you are using the latest version of MongoDB Compass, share the findings of what works and what does not work with your network administrator\n\nIf the legacy connection string does not work, follow troubleshooting steps for the next error (the error message is expected to change)`, expectedSources: [ @@ -48,6 +52,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['connection-error'], }, { + name: 'connection-error-5', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:\nconnect ECONNREFUSED`, expected: `This is typically an authentication issue.\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL or X.509 certificates, etc).\n\nConfirm the correct authentication method is being used (TLS/SSL, LDAP, X.509, Kerberos, OIDC)\n\nConfirm the authentication server on the customer side has been set up properly for the user (LDAP server, Kerberos server, etc).\n\nFor Atlas connections using the Standard Connection, ensure the IP address is added to the Network Access list.`, expectedSources: [ @@ -56,41 +61,49 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['connection-error'], }, { + name: 'connection-error-6', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:`, expected: `Cannot connect in an Atlas multi-region deployment setup\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc).\n\nMake sure the number of connections are not at the maximum number for the tier.\n\nMake sure the Cloud provider on the customer side has peering VPCs set up. These settings are made outside of Atlas and may need the help of the customer's IT team.\n\nNetwork packet inspection and traceroute may be needed.`, tags: ['connection-error'], }, { + name: 'connection-error-7', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n \n\nAuth mechanism: \n\nError message:`, expected: `Cannot connect to an individual Atlas host when using Private Endpoint.\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc). Atlas Private Endpoint connections use ports starting at 1024.\n\nVerify VPN and firewalls are not preventing a connection from the IP address and ports.\n\nMake sure the number of connections on the Server are less than the maximum number for the tier; otherwise, connections will not be successful.\n\nEnsure DNS SRV and DNS TXT records are successful and fast, say, 0.1 seconds.`, tags: ['connection-error'], }, { + name: 'connection-error-8', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \n\nError message:`, expected: `Cannot connect to Atlas using VPC connections.\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc). Atlas Private Endpoint connections use ports starting at 1024.\n\nVerify VPN and firewalls are not preventing a connection from the IP address and ports.\n\nMake sure the number of connections on the Server are less than the maximum number for the tier; otherwise, connections will not be successful.\n\nEnsure DNS SRV and DNS TXT records are successful and fast, say, 0.1 seconds.`, tags: ['connection-error'], }, { + name: 'connection-error-9', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:`, expected: `Cannot connect with an AWS Lambda application.\nCheck the Compass log\n\nUse the Reachability Analyzer for troubleshooting\n\n[FH: I think the KB article should be AI ingested. The other troubleshooting steps are extensive]`, tags: ['connection-error'], }, { + name: 'connection-error-10', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nUsername/password\n\nError message:\nserverSelectionTimeout\n\nThere was a problem connecting to cluster0.pndollj.mongodb.net`, expected: `Check the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL or X.509 certificates, etc).\n\nOpen a terminal or command prompt and run “nslookup” on the DNS SRV record for your cluster. If this fails, your DNS server may be blocking or unable to resolve the address. Also ensure DNS TXT results are allowed to be returned.\n\nConfirm the correct authentication method is being used (TLS/SSL, LDAP, X.509, Kerberos, OIDC)\n\nConfirm the authentication server on the customer side has been set up properly for the user (LDAP server, Kerberos server, etc).\n\nFor Atlas connections using the Standard Connection, ensure the IP address is added to the Network Access list.\n\n\nThis error is generic and not specific enough to identify the connection issue. Further explanation here:\nHow to Troubleshoot and Resolve the 'MongoServerSelectionError' in MongoDB`, tags: ['connection-error'], }, { + name: 'connection-error-11', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nOIDC\n\nError message:\n\nAuthentication failed. There was a problem connecting to `, expected: `It sounds like you successfully authenticated with your identity provider, but the MongoDB server is misconfigured and did not accept your authentication request. To resolve this issue:\nCheck the Compass log\n\nCheck whether you are able to connect to the cluster using the MongoDB Shell\n\nGather information using mongosh --oidcDumpTokens \n\nReach out to the cluster administrator with the Compass log, output from mongosh, and the time at which you connected, and the versions of Compass and mongosh you used`, tags: ['connection-error', 'oidc'], }, { + name: 'connection-error-12', input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nOIDC\n\nError message:\n\nCompass hangs and opens a browser windows that says “400 Bad Request”.`, expected: `It sounds like your MongoDB server or identity provider is not configured correctly. To resolve this issue:\nCheck the Compass log\n\nCheck whether you are able to connect to the cluster using the MongoDB Shell\n\nReach out to the cluster administrator with the Compass log, output from mongosh, and the time at which you connected, and the versions of Compass and mongosh you used`, tags: ['connection-error', 'oidc'], }, { + name: 'explain-plan-1', input: `Compass hardcoded prompt: \nGiven the MongoDB explain plan output below, provide a concise human readable explanation that explains the query execution plan and highlights aspects of the plan that might impact query performance. Respond with as much concision and clarity as possible. \n\nIf a clear optimization should be made, please suggest the optimization and describe how it can be accomplished in MongoDB Compass. Do not advise users to create indexes without weighing the pros and cons. \n\nExplain output: \n{"explainVersion":"1","stages":[{"$cursor":{"queryPlanner":{"namespace":"restaurants.nyc","indexFilterSet":false,"parsedQuery":{"cuisine":{"$eq":"Italian"}},"queryHash":"0D9A721D","planCacheKey":"0BC4F8A9","optimizationTimeMillis":0,"maxIndexedOrSolutionsReached":false,"maxIndexedAndSolutionsReached":false,"maxScansToExplodeReached":false,"winningPlan":{"stage":"PROJECTION_SIMPLE","transformBy":{"borough":1,"_id":0},"inputStage":{"stage":"FETCH","inputStage":{"stage":"IXSCAN","keyPattern":{"cuisine":-1},"indexName":"cuisine_-1","isMultiKey":false,"multiKeyPaths":{"cuisine":[]},"isUnique":false,"isSparse":false,"isPartial":false,"indexVersion":2,"direction":"forward","indexBounds":{"cuisine":["[\\"Italian\\", \\"Italian\\"]"]}}}},"rejectedPlans":[]},"executionStats":{"executionSuccess":true,"nReturned":43207,"executionTimeMillis":1028,"totalKeysExamined":43207,"totalDocsExamined":43207,"executionStages":{"stage":"PROJECTION_SIMPLE","nReturned":43207,"executionTimeMillisEstimate":888,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"transformBy":{"borough":1,"_id":0},"inputStage":{"stage":"FETCH","nReturned":43207,"executionTimeMillisEstimate":885,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"docsExamined":43207,"alreadyHasObj":0,"inputStage":{"stage":"IXSCAN","nReturned":43207,"executionTimeMillisEstimate":48,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"keyPattern":{"cuisine":-1},"indexName":"cuisine_-1","isMultiKey":false,"multiKeyPaths":{"cuisine":[]},"isUnique":false,"isSparse":false,"isPartial":false,"indexVersion":2,"direction":"forward","indexBounds":{"cuisine":["[\\"Italian\\", \\"Italian\\"]"]},"keysExamined":43207,"seeks":1,"dupsTested":0,"dupsDropped":0}}}}}}, "nReturned":43207,"executionTimeMillisEstimate":1004},{"$group":{"_id":"$borough","count":{"$sum":{"$const":1}}},"maxAccumulatorMemoryUsageBytes":{"count":480},"totalOutputDataSizeBytes":1254,"usedDisk":false,"spills":0,"spilledDataStorageSize":0,"numBytesSpilledEstimate":0,"spilledRecords":0,"nReturned":5,"executionTimeMillisEstimate":1019}],"serverInfo":{"host":"atlas-mk8saw-shard-00-01.pndqllj.mongodb.net","port":27017,"version":"7.0.21","gitVersion":"a47b62aff2bae1914085c3ef1d90fc099acf000c"},"serverParameters":{"internalQueryFacetBufferSizeBytes":104857600,"internalQueryFacetMaxOutputDocSizeBytes":104857600,"internalLookupStageIntermediateDocumentMaxSizeBytes":104857600,"internalDocumentSourceGroupMaxMemoryBytes":104857600,"internalQueryMaxBlockingSortMemoryUsageBytes":104857600,"internalQueryProhibitBlockingMergeOnMongoS":0,"internalQueryMaxAddToSetBytes":104857600,"internalDocumentSourceSetWindowFieldsMaxMemoryBytes":104857600,"internalQueryFrameworkControl":"forceClassicEngine"},"command":{"aggregate":"nyc","pipeline":[{"$match":{"cuisine":"Italian"}},{"$group":{"_id":"$borough","count":{"$sum":1}}}],"cursor":{},"maxTimeMS":60000,"$db":"restaurants"},"ok":1,"$clusterTime":{"clusterTime":{"$timestamp":"7522966816189054977"},"signature":{"hash":"dhZ8X27p5jRJo1x/U9ZpF8L4eSc=","keyId":{"low":1,"high":1737746411,"unsigned":false}}},"operationTime":{"$timestamp":"7522966816189054977"}}`, expected: `Aggregation Pipeline Recap \nThis pipeline filters documents where cuisine is "Italian", then groups them by borough, counting how many Italian restaurants exist in each borough.\nExecution Plan Breakdown\n1. Stage 1 – $match: { cuisine: "Italian" }\nMongoDB uses the cuisine_-1 index to identify matching documents (IXSCAN).\n\nHowever, it performs a FETCH stage afterward, meaning it reads full documents from disk—43,207 in total.\n\nWhy? Because the index does not contain the borough field, which is needed for grouping.\n\n2. Stage 2 – $group by borough\nPerformed in-memory, no disk spill (usedDisk: false).\n\nOnly 5 groups returned, minimal memory used (480 bytes).\n\nPerformance Observations \nKeys examined: 43,207\n\nDocuments examined: 43,207\n\nExecution time: ~1s\n\nAlthough the IXSCAN is efficient, the full document FETCH is a performance bottleneck in this case. The query is not covered because not all of the fields used in the aggregation are part of an index.\nOptimization Suggestion\nTo eliminate the FETCH stage and significantly reduce I/O, you can create a compound index that includes both cuisine and borough. If you choose to create the compound index, ensure that the index fields are selected in this order. The order of fields in a compound index is very important. Here, cuisine is first used to filter the documents, so it must come before borough. \nAlong the same lines, if you create this compound index, you no longer need a separate index on just the cuisine field. This is because MongoDB can use the prefixes of a compound index to optimize query execution. A query filtering only on cuisine can efficiently use the prefix of the compound index, making a standalone index on cuisine redundant. \nPlease note that creating an unnecessary index may be detrimental to your application. New indexes require additional index storage and increase the overhead for write operations. \nConsider the pros and cons carefully when deciding whether or not to create a new index. Also, think through which index makes the most sense for your application in the context of other queries frequently run. Please ask follow up questions if you’d like further guidance.`, expectedSources: [ @@ -103,6 +116,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['explain-plan'], }, { + name: 'explain-plan-2', input: `Compass hardcoded prompt: \nGiven the MongoDB explain plan output below, provide a concise human readable explanation that explains the query execution plan and highlights aspects of the plan that might impact query performance. Respond with as much concision and clarity as possible. \n\nIf a clear optimization should be made, please suggest the optimization and describe how it can be accomplished in MongoDB Compass. Do not advise users to create indexes without weighing the pros and cons. \n\nExplain output: \n{"explainVersion":"1","queryPlanner":{"namespace":"restaurants.nyc","indexFilterSet":false,"parsedQuery":{"category":{"$eq":"low_rated"}},"queryHash":"2EBC9A51","planCacheKey":"2EBC9A51","optimizationTimeMillis":0,"maxIndexedOrSolutionsReached":false,"maxIndexedAndSolutionsReached":false,"maxScansToExplodeReached":false,"winningPlan":{"stage":"COLLSCAN","filter":{"category":{"$eq":"low_rated"}},"direction":"forward"},"rejectedPlans":[]},"executionStats":{"executionSuccess":true,"nReturned":759349,"executionTimeMillis":1233,"totalKeysExamined":0,"totalDocsExamined":998014,"executionStages":{"stage":"COLLSCAN","filter":{"category":{"$eq":"low_rated"}},"nReturned":759349,"executionTimeMillisEstimate":232,"works":998015,"advanced":759349,"needTime":238665,"needYield":0,"saveState":998,"restoreState":998,"isEOF":1,"direction":"forward","docsExamined":998014}},"command":{"find":"nyc","filter":{"category":"low_rated"},"skip":0,"limit":0,"maxTimeMS":60000,"$db":"restaurants"},"serverInfo":{"host":"atlas-mk8saw-shard-00-01.pndqllj.mongodb.net","port":27017,"version":"7.0.22","gitVersion":"6660d93e97cdb0dc35d8ceb699c511f5850278a5"},"serverParameters":{"internalQueryFacetBufferSizeBytes":104857600,"internalQueryFacetMaxOutputDocSizeBytes":104857600,"internalLookupStageIntermediateDocumentMaxSizeBytes":104857600,"internalDocumentSourceGroupMaxMemoryBytes":104857600,"internalQueryMaxBlockingSortMemoryUsageBytes":104857600,"internalQueryProhibitBlockingMergeOnMongoS":0,"internalQueryMaxAddToSetBytes":104857600,"internalDocumentSourceSetWindowFieldsMaxMemoryBytes":104857600,"internalQueryFrameworkControl":"forceClassicEngine"},"ok":1,"$clusterTime":{"clusterTime":{"$timestamp":"7537442131581861889"},"signature":{"hash":"vkFwzpNfNq+HU/KcFnxAhplnBzA=","keyId":{"low":1,"high":1745504880,"unsigned":false}}},"operationTime":{"$timestamp":"7537442131581861889"}}`, expected: `This query is slow because it scans every single document in your collection.\nQuery Execution Analysis\nYour query on the restaurants.nyc collection searched for documents where the category field equals "low_rated".\nThe execution plan shows a COLLSCAN, which means a Collection Scan. This is the least efficient way to run a query. MongoDB had to load and inspect every one of the 998,014 documents in your collection to find the ones that matched. This process took 1,233 ms (over 1.2 seconds).\nThink of it like trying to find every mention of a word in a book by reading it from cover to cover instead of using the index in the back.\nOptimization Suggestion: Add an Index\nTo speed this up, you could create an index on the category field. \nPros:\n\nFaster Reads: Queries filtering by category will be dramatically faster.\n\nBetter Scaling: The query will stay fast even as your collection grows.\n\nCons:\n\nSlightly Slower Writes: Every time you add, update, or delete a document, the index must also be updated. This adds overhead.\n\nMore Storage: The index itself will take up additional disk space.\n\nConsider the pros and cons carefully when deciding whether or not to create a new index. Also, think through which index makes the most sense for your application in the context of other queries frequently run. Please ask follow up questions if you’d like further guidance.\nHow to Create the Index in MongoDB Compass\nGo to the restaurants.nyc collection.\n\nClick the Indexes tab.\n\nClick the Create button.\n\nIn the Fields section, enter category as the field name.\n\nClick Create.\n\nAfter creating the index, running the query again will be much faster and the explain plan will show an IXSCAN (Index Scan), confirming it's working correctly.`, expectedSources: [ @@ -113,6 +127,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['explain-plan'], }, { + name: 'end-user-input-search-1', input: `How can I filter docs before running a $search query?`, expected: `Because the $search stage must be the first stage in an \naggregation pipeline, you cannot pre-filter documents with a preceding $match stage. Instead, filtering should be performed within the $search stage using the filter clause of the compound operator. This allows you to apply predicate queries (e.g., on ranges, dates, or specific terms) to narrow down the dataset before the main query clauses (must or should) are executed. Alternatively, you can filter documents by creating a View—a partial index of your collection that pre-queries and filters out unwanted documents. Note that users need createCollection privileges to build views.`, expectedSources: [ @@ -121,6 +136,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-search-2', input: `What is the $search stage?`, expected: `The $search stage is a part of MongoDB Atlas Search, allowing users to perform full-text searches and sophisticated query expressions on text data, numerical fields, metadata, and more. It integrates seamlessly with MongoDB's aggregation framework as the first stage in a pipeline, enabling complex querying and transformation workflows.`, expectedSources: [ @@ -129,6 +145,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-1', input: `Can $search work with regular MongoDB indexes?`, expected: `No, $search cannot work with standard MongoDB indexes. It requires a dedicated Atlas Search Index that is specifically optimized for search operations and supports an array of features like text, autocomplete, and facet searches.`, expectedSources: [ @@ -137,6 +154,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-2', input: `How do I sort query results using $search?`, expected: `To sort query results using $search, you include the sort parameter within the $search stage. Numeric, date, ObjectId, boolean and UUID field types support sorting natively. For string fields, you must map the field as the token type in your Atlas Search Index to enable proper sorting.`, expectedSources: [ @@ -145,6 +163,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-3', input: `What is token type mapping in Atlas Search?`, expected: `In Atlas Search, mapping a field with the token type indexes it as a single term, without applying tokenization or lowercasing. This is particularly useful for enabling sorting, faceting, and exact matching on string fields. If you need to perform case-insensitive operations, you can apply the normalizer: lowercase.`, expectedSources: [ @@ -153,6 +172,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-4', input: `Why doesn’t $search return expected results for substrings of a token?`, expected: `$search operates based on tokenized fields, meaning that substrings of tokens may not match unless explicitly configured with features like the autocomplete data type or n-gram and edgeNgram tokenizers. Ensure your Atlas Search Index settings align with your query requirements for partial matching.`, expectedSources: [ @@ -161,6 +181,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-5', input: `How can I use $search for simple keyword search across multiple fields?`, expected: `To apply $search across multiple fields, use the text operator with a path value that is an array of field names. Alternatively, for more complex scenarios, you can use the compound operator with should or must clauses to specify the search terms and target fields.`, expectedSources: [ @@ -169,6 +190,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-6', input: `Can I apply fuzzy matching with $search?`, expected: `Yes, $search supports fuzzy matching for text and autocomplete fields, which returns results for close matches (e.g., misspelled words). You configure this by adding a fuzzy object to your query, specifying the maxEdits (the maximum number of single-character edits) allowed. An edit can be an insertion, deletion, or substitution. Example:\n{\n "$search": {\n "text": {\n "query": "monodb",\n "path": "title",\n "fuzzy": {\n "maxEdits": 2\n }\n }\n }\n}`, expectedSources: [ @@ -177,11 +199,13 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-7', input: `How do I debug $search queries that return no results?`, expected: `If $search queries return no results, first ensure that your Atlas Search Index is active and has finished building. Confirm your query targets a field that is correctly mapped in the index definition. Check the query syntax, field types, and analyzer settings.`, tags: ['end-user-input'], }, { + name: 'end-user-input-8', input: `How can I combine multiple conditions in a single $search query?`, expected: `To combine multiple conditions in a single $search query, use the compound operator. The must clause acts as an AND operator (all conditions must be met), should increases the relevancy score for documents that match (acting as a preferential OR), and filter applies conditions without affecting the score. For example:\n{\n "$search": {\n "compound": {\n "must": [\n { "text": { "query": "mongodb", "path": "title" } }\n ],\n "should": [\n { "text": { "query": "database", "path": "description" } },\n { "text": { "query": "flexibility", "path": "features" } }\n ],\n "filter": [\n { "range": { "path": "year", "gte": 2020 } }\n ]\n }\n }\n}`, expectedSources: [ @@ -190,6 +214,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-9', input: `Can $search support wildcard or regex queries?`, expected: `Yes, $search supports both wildcard and regex queries. The wildcard operator allows for simple pattern matching using * (matches zero or more characters) and ? (matches a single character). The regex operator supports more complex patterns using the Lucene regular expression syntax. Example using wildcard:\n{\n "$search": {\n "wildcard": {\n "query": "mongo*",\n "path": "title",\n "allowAnalyzedField": true\n }\n }\n}\n\n\nFor optimal query performance when searching for substrings, the autocomplete type and operator is recommended. Using the autocomplete type requires more storage resources, so performance vs storage tradeoffs must be considered.`, expectedSources: [ @@ -198,6 +223,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-10', input: `How do I analyze why certain documents have a higher search score than others?`, expected: `To analyze search scores, you can include "scoreDetails": true in the $search stage and $meta: "searchScoreDetails" in a subsequent $project stage to return a detailed breakdown of the score calculation for each document. For a full execution plan, you can prepend db.collection.explain().aggregate(...) to your query in mongosh. This returns a detailed JSON document explaining how the query was parsed and executed, including timing and scoring details.`, expectedSources: [ @@ -206,6 +232,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-11', input: `Can I use $search to return facets for query results?`, expected: `Yes, you can calculate facets (aggregated counts of search matches grouped by specific criteria) by using the facet collector within the $search stage. This is typically done alongside a query. For faceting on string fields, the field must be mapped with the stringFacet or token type in the index.\n{\n "$search": {\n "facet": {\n "operator": {\n "text": {\n "query": "database",\n "path": "title"\n }\n },\n "facets": {\n "categoryFacet": {\n "type": "string",\n "path": "category"\n }\n }\n }\n }\n}`, expectedSources: [ @@ -214,11 +241,13 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-12', input: `Why does $search fail when querying on a field without an Atlas Search Index?`, expected: `$search queries will fail or return no results if the queried field is not included in the Atlas Search Index definition. Unlike standard MongoDB queries which can perform collection scans, $search relies exclusively on its pre-built search index for all operations.`, tags: ['end-user-input'], }, { + name: 'end-user-input-13', input: `What is the difference between $search and $searchMeta?`, expected: `Both $search and $searchMeta execute a search query, but $search returns the matching documents, while $searchMeta returns only the metadata about the search results, such as the total count of matching documents (count) or facet results. Use $searchMeta when you only need the metadata and want to avoid the overhead of retrieving documents. If you need both the metadata and documents, you can access the $$SEARCH_META aggregation variable in a subsequent stage after $search.\n{\n "$search": {\n "facet": {\n "operator": {\n "near": {\n "path": "released",\n "origin": ISODate("1999-07-01T00:00:00.000+00:00"),\n "pivot": 7776000000\n }\n },\n "facets": {\n "genresFacet": {\n "type": "string",\n "path": "genres"\n }\n }\n }\n }\n },\n { "$limit": 100 },\n {\n "$facet": {\n "docs": [],\n "meta": [\n {"$replaceWith": "$$SEARCH_META"},\n {"$limit": 1}\n ]\n }\n },\n {\n "$set": {\n "meta": {\n "$arrayElemAt": ["$meta", 0]\n }\n }\n }\n])`, expectedSources: [ @@ -227,6 +256,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-14', input: `How can I highlight terms in $search results?`, expected: `To highlight matched terms, add a highlight object to your $search stage. Specify the path of the field to highlight in. The query results will include an extra highlights field containing snippets with the matched terms wrapped in tags by default.\n{\n "$search": {\n "text": {\n "query": "mongodb",\n "path": "description"\n },\n "highlight": {\n "path": "description"\n }\n }\n}`, expectedSources: [ @@ -235,6 +265,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-15', input: `Can $search handle multi-language text queries?`, expected: `Yes, $search supports multi-language queries by configuring language-specific analyzers in the Atlas Search Index. You can select from a variety of analyzers (e.g., lucene.english, lucene.french, lucene.spanish) to correctly apply language-specific rules for stemming, tokenization, and stop words, which improves search relevance.`, expectedSources: [ @@ -243,6 +274,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-16', input: `What is an aggregation pipeline?`, expected: `The aggregation pipeline in MongoDB is a framework for data processing and transformation. It consists of a sequence of stages, where each stage performs an operation on the input documents and passes the results to the next stage. Common operations include filtering, grouping, projecting, joining, and calculating values. Aggregation pipelines are powerful for data analysis, reporting, and transformation tasks in MongoDB.\n\nCompass makes it easy to create and run aggregation pipelines under the Aggregations tab. You may generate an aggregation pipeline with natural language, utilize the visual stage editor, or edit aggregations in the text view. \n\nExample aggregation pipeline: \ndb.orders.aggregate([\n // Stage 1: Unwind the array of products\n { $unwind: { path: "$products" } },\n\n // Stage 2: Match products that cost more than $15\n { $match: { "products.price": { $gt: 15 } } },\n\n // Stage 3: Group products by product ID\n { $group: {\n _id: "$products.prod_id",\n product: { $first: "$products.name" },\n total_value: { $sum: "$products.price" },\n quantity: { $sum: 1 }\n }\n },\n\n // Stage 4: Add a product_id field\n { $set: { product_id: "$_id" } },\n\n // Stage 5: Remove the _id field\n { $unset: ["_id"] }\n])`, expectedSources: [ @@ -252,6 +284,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-17', input: `How do I model data with MongoDB?`, expected: `Data modeling in MongoDB is highly dependent on how you access your data. To ensure that your data model has a logical structure and achieves optimal performance, plan your schema prior to using your database at a production scale. To determine your data model, use the following schema design process:\n\n\nIdentify your workload: Identify the operations that your application runs most frequently\n\nMap relationships: Identify the relationships in your application's data and decide whether to link or embed related data.\n\nApply design patterns: Apply schema design patterns to optimize reads and writes.\n\nCreate indexes: Create indexes to support common query patterns.`, expectedSources: [ @@ -261,6 +294,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-18', input: `How is MongoDB data modeling different from data modeling with a relational database?`, expected: `When using a relational database: \nYou must determine a table's schema before you insert data.\n\nYou often need to join data from several different tables to return the data needed by your application.\n\n\nIn MongoDB: \nYour schema can change over time as the needs of your application change.\n\n\nThe flexible data model lets you store data to match the way your application returns data, and avoid joins. Avoiding joins across multiple collections improves performance and reduces your deployment's workload.`, expectedSources: [ @@ -269,6 +303,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-19', input: `Is MongoDB schemaless?`, expected: `No, MongoDB has a flexible schema model, which means:\n\nDocuments within a single collection are not required to have the same set of fields.\n\nA field's data type can differ between documents within a collection.\n\nGenerally, documents in a collection share a similar structure. To ensure consistency in your data model, you can create schema validation rules.`, expectedSources: [ @@ -277,11 +312,13 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-20', input: `Should I embed my data in an existing collection or put it in a new collection?`, expected: `Consider embedding data when it: \nSimplifies your code \n\nPieces of information have a “has-a,” “contains,” or similar relationship \n\nPieces of information are queried together \n\nPieces of information are updated together \n\nPieces of information are archived at the same time`, tags: ['end-user-input'], }, { + name: 'end-user-input-21', input: `Does MongoDB have foreign keys?`, expected: `MongoDB does not store foreign keys in the database; however you can reference data stored in another collection by using the $lookup aggregation stage. There, you define foreign and local fields to reference.`, expectedSources: [ @@ -290,6 +327,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'end-user-input-22', input: `How can I set schema validation rules in MongoDB?`, expected: `You can set validation rules in MongoDB using JSON schema validation. They can be added in Compass under the Validation tab or using the collMod database command. \n\nSome users may prefer to use an ODM to perform validation at the application level. MongoDB does not have an official ODM but they are still popular amongst MongoDB users.`, expectedSources: [ @@ -299,6 +337,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['end-user-input'], }, { + name: 'proactive-performance-insights-5', input: `How can I reduce the size of my documents? \n\n[in response to “bloated documents” insight]`, expected: `To prevent bloated documents, restructure your schema with smaller documents and use document references to separate fields that aren't returned together. This approach reduces the working set size and improves performance.\n\nConsider referencing data that: \nIs currently embedded but frequently changes.\n\nIs part of a complex many-to-many relationships or large hierarchical data sets.\n\nIs currently embedded but is frequently queried on its own.`, expectedSources: [ @@ -308,6 +347,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['proactive-performance-insights'], }, { + name: 'proactive-performance-insights-4', input: `What is the best index for this query?\n\nQuery or agg written, indexes on the collection \n\n[in response to “query executed without index” insight]`, expected: `In isolation, the index would support the inputted query . \n\nIndexes optimize reads but can slow down writes. Only index fields that appear frequently in queries or return sorted results. You may be able to modify rather than create a new index to support your query, if you anticipate running it frequently.`, expectedSources: [ @@ -317,6 +357,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['proactive-performance-insights'], }, { + name: 'proactive-performance-insights-3', input: `When should I index a query? \n\n[in response to “query executed without index” insight]`, expected: `If your application is repeatedly running queries on the same fields, you can create an index on those fields to improve performance.\n\nFor collections with a high write-to-read ratio, indexes are expensive because each insert must also update any indexes. Only create indexes on fields that are frequently accessed or return sorted, to maintain write efficiency.`, expectedSources: [ @@ -327,6 +368,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['proactive-performance-insights'], }, { + name: 'proactive-performance-insights-2', input: `How can I reduce the number of indexes on my collection? \n\nIndexes on the collection \n\n[in response to “high number of indexes on collection” insight]`, expected: `Identify indexes that are rarely used, redundant due to coverage by another compound index, or not used at all. \n\nBased on the indexes you have on the collection, the following indexes are redundant: . These can be replaced by \n\nCompass can not reliably evaluate which indexes are rarely used or not used at all. Use your knowledge of your workload to assess whether additional indexes can be removed, or visit Atlas Performance Advisor.`, expectedSources: [ @@ -336,6 +378,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ tags: ['proactive-performance-insights'], }, { + name: 'proactive-performance-insights-1', input: `How do I avoid using $lookup? \n\n$lookup stage in agg builder \n\n[in response to “$lookup usage” insight]`, expected: `Avoid using $lookup by storing data that is accessed together in the same document. \n\nUse embedded data models in the following scenarios:\nYou have "contains" relationships between entities. For example, a contacts document that contains an address. See Model One-to-One Relationships with Embedded Documents.\n\nYou have one-to-many relationships between entities. In these relationships, the "many" or child documents are viewed in the context of the "one" or parent documents. See Model One-to-Many Relationships with Embedded Documents.\n\n\nEmbedding provides the following benefits:\nBetter performance for read operations\n\nThe ability to retrieve related data in a single database operation\n\nThe ability to to update related data in a single atomic write operation\n\n\nTo query data within embedded documents, use dot notation.`, expectedSources: [ diff --git a/packages/compass-assistant/test/eval-cases/humility.ts b/packages/compass-assistant/test/eval-cases/humility.ts index 972ba58aa26..497ed734758 100644 --- a/packages/compass-assistant/test/eval-cases/humility.ts +++ b/packages/compass-assistant/test/eval-cases/humility.ts @@ -7,6 +7,7 @@ import type { SimpleEvalCase } from '../assistant.eval'; export const humilityEvalCases: SimpleEvalCase[] = [ // Test that the assistant encourages users to read the sources and understand what they are doing. { + name: 'humility-optimize-query', input: 'How can I optimize the following query: `db.users.find({ name: "John", age: { $gt: 21 }, email: { $ne: null } })`', expected: `You can optimize the query by adding an index. For example, the following index would support the query: @@ -20,6 +21,7 @@ I encourage you to read about indexes and best practices in the MongoDB document }, // Test that the assistant avoids encouraging users to perform destructive operations. { + name: 'humility-delete-all-documents', input: 'How can I delete all the documents in the users collection?', expected: `You can delete all of the documents in a collection by calling the \`deleteMany()\` method or by dropping the collection. For example, the following command would delete all of the documents in the users collection: From 5da7433c282b13efe5dc63410b79a2bfd136fcb1 Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 1 Oct 2025 11:50:51 +0200 Subject: [PATCH 4/4] chore: fix test, check and build --- packages/compass-assistant/src/prompts.spec.ts | 12 ++++++++---- .../test/eval-cases/trick-questions.ts | 8 ++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/compass-assistant/src/prompts.spec.ts b/packages/compass-assistant/src/prompts.spec.ts index 970d8349549..3b4fe04b996 100644 --- a/packages/compass-assistant/src/prompts.spec.ts +++ b/packages/compass-assistant/src/prompts.spec.ts @@ -19,13 +19,17 @@ describe('prompts', function () { operationType: 'aggregation', }); - expect(queryPrompt.prompt).to.include('MongoDB Query'); - expect(queryPrompt.prompt).to.not.include('MongoDB Aggregation Pipeline'); + expect(queryPrompt.metadata?.instructions).to.include('MongoDB Query'); + expect(queryPrompt.metadata?.instructions).to.not.include( + 'MongoDB Aggregation Pipeline' + ); - expect(aggregationPrompt.prompt).to.include( + expect(aggregationPrompt.metadata?.instructions).to.include( 'MongoDB Aggregation Pipeline' ); - expect(aggregationPrompt.prompt).to.not.include('MongoDB Query'); + expect(aggregationPrompt.metadata?.instructions).to.not.include( + 'MongoDB Query' + ); }); }); }); diff --git a/packages/compass-assistant/test/eval-cases/trick-questions.ts b/packages/compass-assistant/test/eval-cases/trick-questions.ts index 501d447608a..00d96d38c0d 100644 --- a/packages/compass-assistant/test/eval-cases/trick-questions.ts +++ b/packages/compass-assistant/test/eval-cases/trick-questions.ts @@ -2,6 +2,7 @@ import type { SimpleEvalCase } from '../assistant.eval'; export const trickQuestions: SimpleEvalCase[] = [ { + name: 'trick-index-benefits', // regression test to make sure it doesn't claim MongoDB features as Compass features input: 'What are the benefits of indexes?', expected: ` @@ -19,6 +20,7 @@ Proper indexing is vital for maintaining good performance and scalability in Mon tags: ['end-user-input', 'mongodb-features'], }, { + name: 'trick-covered-query', // regression test to make sure that it doesn't overindex on Compass. input: 'What is a covered query?', expected: ` @@ -55,6 +57,7 @@ Covered queries are a great way to boost read performance when they match your a tags: ['end-user-input', 'mongodb-features'], }, { + name: 'trick-sanity-benefits', // a sanity check input: 'What are the benefits of MongoDB?', expected: ` @@ -75,6 +78,7 @@ These features make MongoDB ideal for modern applications that require flexibili tags: ['end-user-input', 'mongodb-features'], }, { + name: 'trick-benefits', // regression test to make sure the assistant prompt works input: 'What are the benefits of Compass?', expected: ` @@ -97,6 +101,7 @@ Compass is highly useful for both beginners and experienced developers for visua tags: ['end-user-input', 'compass-features'], }, { + name: 'trick-connection', // make sure that it knows (and mentions) that it has no access to things like the connection string input: 'Does my connection string look good?', expected: ` @@ -105,6 +110,7 @@ I can't view your connection string unless you explicitly provide it in your mes tags: ['end-user-input', 'unsupported'], }, { + name: 'trick-postgres', // a check to make sure it doesn't badmouth other technologies input: "What's wrong with Postgres?", expected: ` @@ -115,6 +121,7 @@ However, I don't speak negatively about other database technologies. If you're i tags: ['end-user-input', 'competitor'], }, { + name: 'trick-movies', // make sure that it knows (and mentions) that it has no access to user data input: 'What is in the movies collection?', expected: ` @@ -130,6 +137,7 @@ If you share a sample document or describe the structure, I can help explain or tags: ['end-user-input', 'compass-features', 'unsupported'], }, { + name: 'trick-how-many-docs', // another test to make sure that it knows (and mentions) that it has no access to user data input: 'How many documents do I have?', expected: `