From 147b2d72b5a87a62f1854cee19493402f792bf69 Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 27 Aug 2025 20:57:46 +0200 Subject: [PATCH 1/6] chore(compass-assistant): add feedback submission and telemetry COMPASS-9608 --- package-lock.json | 2 + .../modules/pipeline-builder/pipeline-ai.ts | 1 + packages/compass-assistant/package.json | 1 + .../src/assistant-chat.spec.tsx | 174 ++++++++++++++++-- .../compass-assistant/src/assistant-chat.tsx | 48 ++++- .../src/stores/ai-query-reducer.ts | 1 + .../compass-telemetry/src/telemetry-events.ts | 55 ++++++ 7 files changed, 265 insertions(+), 17 deletions(-) diff --git a/package-lock.json b/package-lock.json index f54bb7507a4..3ef0eadf95f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47117,6 +47117,7 @@ "@mongodb-js/compass-app-registry": "^9.4.20", "@mongodb-js/compass-components": "^1.49.0", "@mongodb-js/compass-logging": "^1.7.12", + "@mongodb-js/compass-telemetry": "^1.14.0", "@mongodb-js/connection-info": "^0.17.1", "ai": "^5.0.26", "compass-preferences-model": "^2.51.0", @@ -60560,6 +60561,7 @@ "@mongodb-js/compass-app-registry": "^9.4.20", "@mongodb-js/compass-components": "^1.49.0", "@mongodb-js/compass-logging": "^1.7.12", + "@mongodb-js/compass-telemetry": "^1.14.0", "@mongodb-js/connection-info": "^0.17.1", "@mongodb-js/eslint-config-compass": "^1.4.7", "@mongodb-js/mocha-config-compass": "^1.7.0", diff --git a/packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts b/packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts index c932aadfe67..2a930ec3a87 100644 --- a/packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts +++ b/packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts @@ -247,6 +247,7 @@ export const runAIPipelineGeneration = ( track( 'AI Prompt Submitted', () => ({ + source: 'natural language query' as const, editor_view_type, user_input_length: userInput.length, request_id: requestId, diff --git a/packages/compass-assistant/package.json b/packages/compass-assistant/package.json index f7b4613601d..48d2832bdcb 100644 --- a/packages/compass-assistant/package.json +++ b/packages/compass-assistant/package.json @@ -53,6 +53,7 @@ "@mongodb-js/atlas-service": "^0.56.0", "@mongodb-js/compass-app-registry": "^9.4.20", "@mongodb-js/compass-components": "^1.49.0", + "@mongodb-js/compass-telemetry": "^1.14.0", "@mongodb-js/connection-info": "^0.17.1", "@mongodb-js/compass-logging": "^1.7.12", "mongodb-connection-string-url": "^3.0.1", diff --git a/packages/compass-assistant/src/assistant-chat.spec.tsx b/packages/compass-assistant/src/assistant-chat.spec.tsx index 8de0b4a5885..3155c0047a8 100644 --- a/packages/compass-assistant/src/assistant-chat.spec.tsx +++ b/packages/compass-assistant/src/assistant-chat.spec.tsx @@ -11,16 +11,6 @@ import { createMockChat } from '../test/utils'; import type { AssistantMessage } from './compass-assistant-provider'; describe('AssistantChat', function () { - let originalScrollTo: typeof Element.prototype.scrollTo; - // Mock scrollTo method for DOM elements to prevent test failures - before(function () { - originalScrollTo = Element.prototype.scrollTo.bind(Element.prototype); - Element.prototype.scrollTo = () => {}; - }); - after(function () { - Element.prototype.scrollTo = originalScrollTo; - }); - const mockMessages: AssistantMessage[] = [ { id: 'user', @@ -41,8 +31,9 @@ describe('AssistantChat', function () { function renderWithChat(messages: AssistantMessage[]) { const chat = createMockChat({ messages }); + const result = render(); return { - result: render(), + result, chat, }; } @@ -131,8 +122,9 @@ describe('AssistantChat', function () { ); }); - it('calls sendMessage when form is submitted', function () { - const { chat } = renderWithChat([]); + it('calls sendMessage when form is submitted', async function () { + const { chat, result } = renderWithChat([]); + const { track } = result; const inputField = screen.getByPlaceholderText( 'Ask MongoDB Assistant a question' ); @@ -143,6 +135,12 @@ describe('AssistantChat', function () { expect(chat.sendMessage.calledWith({ text: 'What is aggregation?' })).to.be .true; + + await waitFor(() => { + expect(track).to.have.been.calledWith('Assistant Prompt Submitted', { + user_input_length: 'What is aggregation?'.length, + }); + }); }); it('clears input field after successful submission', function () { @@ -160,8 +158,9 @@ describe('AssistantChat', function () { expect(inputField.value).to.equal(''); }); - it('trims whitespace from input before sending', function () { - const { chat } = renderWithChat([]); + it('trims whitespace from input before sending', async function () { + const { chat, result } = renderWithChat([]); + const { track } = result; const inputField = screen.getByPlaceholderText( 'Ask MongoDB Assistant a question' @@ -172,6 +171,12 @@ describe('AssistantChat', function () { expect(chat.sendMessage.calledWith({ text: 'What is sharding?' })).to.be .true; + + await waitFor(() => { + expect(track).to.have.been.calledWith('Assistant Prompt Submitted', { + user_input_length: 'What is sharding?'.length, + }); + }); }); it('does not call sendMessage when input is empty or whitespace-only', function () { @@ -272,4 +277,143 @@ describe('AssistantChat', function () { expect(screen.queryByText('Another part that should not display.')).to.not .exist; }); + + describe('feedback buttons', function () { + it('shows feedback buttons only for assistant messages', function () { + renderWithChat(mockMessages); + + const userMessage = screen.getByTestId('assistant-message-user'); + const assistantMessage = screen.getByTestId( + 'assistant-message-assistant' + ); + + // User messages should not have feedback buttons + expect(userMessage.querySelector('[aria-label="Thumbs Up Icon"]')).to.not + .exist; + expect(userMessage.querySelector('[aria-label="Thumbs Down Icon"]')).to + .not.exist; + + // Assistant messages should have feedback buttons + expect(assistantMessage.querySelector('[aria-label="Thumbs Up Icon"]')).to + .exist; + expect(assistantMessage.querySelector('[aria-label="Thumbs Down Icon"]')) + .to.exist; + }); + + it('tracks positive feedback when thumbs up is clicked', async function () { + const { result } = renderWithChat(mockMessages); + const { track } = result; + + const assistantMessage = screen.getByTestId( + 'assistant-message-assistant' + ); + + // Find and click the thumbs up button + const thumbsUpButton = assistantMessage.querySelector( + '[aria-label="Thumbs Up Icon"]' + ) as HTMLElement; + + userEvent.click(thumbsUpButton); + + await waitFor(() => { + expect(track).to.have.callCount(1); + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'positive', + text: undefined, + request_id: null, + }); + }); + }); + + it('tracks negative feedback when thumbs down is clicked', async function () { + const { result } = renderWithChat(mockMessages); + const { track } = result; + + const assistantMessage = screen.getByTestId( + 'assistant-message-assistant' + ); + + // Find and click the thumbs down button + const thumbsDownButton = assistantMessage.querySelector( + '[aria-label="Thumbs Down Icon"]' + ) as HTMLElement; + + userEvent.click(thumbsDownButton); + + await waitFor(() => { + expect(track).to.have.callCount(1); + + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'negative', + text: undefined, + request_id: null, + }); + }); + }); + + it('tracks detailed feedback when feedback text is submitted', async function () { + const { result } = renderWithChat(mockMessages); + const { track } = result; + + const assistantMessage = screen.getByTestId( + 'assistant-message-assistant' + ); + + // First click thumbs down to potentially open feedback form + const thumbsDownButton = assistantMessage.querySelector( + '[aria-label="Thumbs Down Icon"]' + ) as HTMLElement; + + userEvent.click(thumbsDownButton); + + // Look for feedback text area (the exact implementation depends on LeafyGreen) + const feedbackTextArea = screen.getByTestId( + 'lg-chat-message_actions-feedback_textarea' + ); + + userEvent.type(feedbackTextArea, 'This response was not helpful'); + + // Look for submit button + const submitButton = screen.getByText('Submit'); + + userEvent.click(submitButton); + + await waitFor(() => { + expect(track).to.have.callCount(2); + + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'negative', + text: undefined, + request_id: null, + }); + + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'negative', + text: 'This response was not helpful', + request_id: null, + }); + }); + }); + + it('does not show feedback buttons when there are no assistant messages', function () { + const userOnlyMessages: AssistantMessage[] = [ + { + id: 'user1', + role: 'user', + parts: [{ type: 'text', text: 'Hello!' }], + }, + { + id: 'user2', + role: 'user', + parts: [{ type: 'text', text: 'How are you?' }], + }, + ]; + + renderWithChat(userOnlyMessages); + + // Should not find any feedback buttons in the entire component + expect(screen.queryByLabelText('Thumbs Up Icon')).to.not.exist; + expect(screen.queryByLabelText('Thumbs Down Icon')).to.not.exist; + }); + }); }); diff --git a/packages/compass-assistant/src/assistant-chat.tsx b/packages/compass-assistant/src/assistant-chat.tsx index 78198e16ece..57b650fb2fd 100644 --- a/packages/compass-assistant/src/assistant-chat.tsx +++ b/packages/compass-assistant/src/assistant-chat.tsx @@ -7,16 +7,19 @@ import { LgChatLeafygreenChatProvider, LgChatMessage, LgChatMessageFeed, + LgChatMessageActions, LgChatInputBar, spacing, css, Banner, } from '@mongodb-js/compass-components'; +import { useTelemetry } from '@mongodb-js/compass-telemetry/provider'; const { ChatWindow } = LgChatChatWindow; const { LeafyGreenChatProvider, Variant } = LgChatLeafygreenChatProvider; const { Message } = LgChatMessage; const { MessageFeed } = LgChatMessageFeed; +const { MessageActions } = LgChatMessageActions; const { InputBar } = LgChatInputBar; interface AssistantChatProps { @@ -52,6 +55,7 @@ export const AssistantChat: React.FunctionComponent = ({ const { messages, sendMessage, status, error, clearError } = useChat({ chat, }); + const track = useTelemetry(); // Transform AI SDK messages to LeafyGreen chat format const lgMessages = messages.map((message) => ({ @@ -70,10 +74,43 @@ export const AssistantChat: React.FunctionComponent = ({ (messageBody: string) => { const trimmedMessageBody = messageBody.trim(); if (trimmedMessageBody) { + track('Assistant Prompt Submitted', { + user_input_length: trimmedMessageBody.length, + }); void sendMessage({ text: trimmedMessageBody }); } }, - [sendMessage] + [sendMessage, track] + ); + + const handleFeedback = useCallback( + ( + event, + state: + | { + feedback: string; + rating: string; + } + | { + rating: string; + } + | undefined + ) => { + if (!state) { + return; + } + const { rating } = state; + const textFeedback = 'feedback' in state ? state.feedback : undefined; + const feedback: 'positive' | 'negative' = + rating === 'liked' ? 'positive' : 'negative'; + + track('Assistant Feedback Submitted', () => ({ + feedback, + text: textFeedback, + request_id: null, + })); + }, + [track] ); return ( @@ -94,7 +131,14 @@ export const AssistantChat: React.FunctionComponent = ({ sourceType="markdown" {...messageFields} data-testid={`assistant-message-${messageFields.id}`} - /> + > + {messageFields.isSender === false && ( + + )} + ))} {status === 'submitted' && ( ({ + source: 'natural language query' as const, editor_view_type: 'find' as const, user_input_length: userInput.length, has_sample_documents: provideSampleDocuments, diff --git a/packages/compass-telemetry/src/telemetry-events.ts b/packages/compass-telemetry/src/telemetry-events.ts index dadcc8efb40..e54228d2245 100644 --- a/packages/compass-telemetry/src/telemetry-events.ts +++ b/packages/compass-telemetry/src/telemetry-events.ts @@ -1458,6 +1458,57 @@ type IndexDroppedEvent = ConnectionScopedEvent<{ }; }>; +/** + * This event is fired when user enters a prompt in the assistant chat + * and hits "enter". + * + * @category Gen AI + */ +type AssistantPromptSubmittedEvent = CommonEvent<{ + name: 'Assistant Prompt Submitted'; + payload: { + user_input_length?: number; + }; +}>; + +/** + * This event is fired when a user submits feedback for the assistant. + * + * @category Assistant + */ +type AssistantFeedbackSubmittedEvent = CommonEvent<{ + name: 'Assistant Feedback Submitted'; + payload: { + feedback: 'positive' | 'negative'; + text: string | undefined; + request_id: string | null; + }; +}>; + +/** + * This event is fired when a user uses an assistant entry point. + * + * @category Gen AI + */ +type AssistantEntryPointUsedEvent = ConnectionScopedEvent<{ + name: 'Assistant Entry Point Used'; + payload: { + source: 'explain plan' | 'performance insights' | 'error message'; + }; +}>; + +/** + * This event is fired when the AI response encounters an error. + * + * @category Gen AI + */ +type AssistantResponseFailedEvent = CommonEvent<{ + name: 'Assistant Response Failed'; + payload: { + error_code?: string; + }; +}>; + /** * This event is fired when a user submits feedback for a query generation. * @@ -2992,6 +3043,10 @@ export type TelemetryEvent = | AggregationTimedOutEvent | AggregationUseCaseAddedEvent | AggregationUseCaseSavedEvent + | AssistantPromptSubmittedEvent + | AssistantResponseFailedEvent + | AssistantFeedbackSubmittedEvent + | AssistantEntryPointUsedEvent | AiOptInModalShownEvent | AiOptInModalDismissedEvent | AiSignInModalShownEvent From f208b6c18c723996affb714c7a8902142e5796c3 Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 27 Aug 2025 21:05:17 +0200 Subject: [PATCH 2/6] chore: add error tracking --- packages/compass-assistant/src/assistant-chat.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/compass-assistant/src/assistant-chat.tsx b/packages/compass-assistant/src/assistant-chat.tsx index 57b650fb2fd..a2dc293ce9d 100644 --- a/packages/compass-assistant/src/assistant-chat.tsx +++ b/packages/compass-assistant/src/assistant-chat.tsx @@ -52,10 +52,13 @@ const errorBannerWrapperStyles = css({ export const AssistantChat: React.FunctionComponent = ({ chat, }) => { + const track = useTelemetry(); const { messages, sendMessage, status, error, clearError } = useChat({ chat, + onError: () => { + track('Assistant Response Failed', () => ({})); + }, }); - const track = useTelemetry(); // Transform AI SDK messages to LeafyGreen chat format const lgMessages = messages.map((message) => ({ From 2ffcd45db56927c43139b0285a824f94ae062af4 Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 27 Aug 2025 21:09:00 +0200 Subject: [PATCH 3/6] chore: add error tracking --- packages/compass-assistant/src/assistant-chat.tsx | 6 ++++-- packages/compass-telemetry/src/telemetry-events.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/compass-assistant/src/assistant-chat.tsx b/packages/compass-assistant/src/assistant-chat.tsx index a2dc293ce9d..3a6468fea3c 100644 --- a/packages/compass-assistant/src/assistant-chat.tsx +++ b/packages/compass-assistant/src/assistant-chat.tsx @@ -55,8 +55,10 @@ export const AssistantChat: React.FunctionComponent = ({ const track = useTelemetry(); const { messages, sendMessage, status, error, clearError } = useChat({ chat, - onError: () => { - track('Assistant Response Failed', () => ({})); + onError: (error) => { + track('Assistant Response Failed', () => ({ + error_name: error.name, + })); }, }); diff --git a/packages/compass-telemetry/src/telemetry-events.ts b/packages/compass-telemetry/src/telemetry-events.ts index e54228d2245..eb1ca494697 100644 --- a/packages/compass-telemetry/src/telemetry-events.ts +++ b/packages/compass-telemetry/src/telemetry-events.ts @@ -1505,7 +1505,7 @@ type AssistantEntryPointUsedEvent = ConnectionScopedEvent<{ type AssistantResponseFailedEvent = CommonEvent<{ name: 'Assistant Response Failed'; payload: { - error_code?: string; + error_name?: string; }; }>; From cf928e88eda9343ba2aad346c746c14ce089e42c Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 27 Aug 2025 21:11:22 +0200 Subject: [PATCH 4/6] chore: revert ai prompt event changes --- .../src/modules/pipeline-builder/pipeline-ai.ts | 1 - packages/compass-query-bar/src/stores/ai-query-reducer.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts b/packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts index 2a930ec3a87..c932aadfe67 100644 --- a/packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts +++ b/packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts @@ -247,7 +247,6 @@ export const runAIPipelineGeneration = ( track( 'AI Prompt Submitted', () => ({ - source: 'natural language query' as const, editor_view_type, user_input_length: userInput.length, request_id: requestId, diff --git a/packages/compass-query-bar/src/stores/ai-query-reducer.ts b/packages/compass-query-bar/src/stores/ai-query-reducer.ts index d2f890a872c..7cb478e1a64 100644 --- a/packages/compass-query-bar/src/stores/ai-query-reducer.ts +++ b/packages/compass-query-bar/src/stores/ai-query-reducer.ts @@ -178,7 +178,6 @@ export const runAIQuery = ( track( 'AI Prompt Submitted', () => ({ - source: 'natural language query' as const, editor_view_type: 'find' as const, user_input_length: userInput.length, has_sample_documents: provideSampleDocuments, From d2563d81aeafa27a7a8d394a5b756228fa807461 Mon Sep 17 00:00:00 2001 From: gagik Date: Thu, 28 Aug 2025 10:13:03 +0200 Subject: [PATCH 5/6] chore: add entry point telemetry --- .../compass-assistant/src/compass-assistant-provider.tsx | 8 ++++++++ packages/compass-telemetry/src/telemetry-events.ts | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/compass-assistant/src/compass-assistant-provider.tsx b/packages/compass-assistant/src/compass-assistant-provider.tsx index 92531f74e45..b0283442aa4 100644 --- a/packages/compass-assistant/src/compass-assistant-provider.tsx +++ b/packages/compass-assistant/src/compass-assistant-provider.tsx @@ -14,6 +14,7 @@ import { usePreference } from 'compass-preferences-model/provider'; import { createLoggerLocator } from '@mongodb-js/compass-logging/provider'; import type { ConnectionInfo } from '@mongodb-js/connection-info'; import { redactConnectionString } from 'mongodb-connection-string-url'; +import { useTelemetry } from '@mongodb-js/compass-telemetry/provider'; export const ASSISTANT_DRAWER_ID = 'compass-assistant-drawer'; @@ -88,6 +89,7 @@ export const AssistantProvider: React.FunctionComponent< chat: Chat; }> > = ({ chat, children }) => { + const track = useTelemetry(); const assistantActionsContext = useRef({ interpretExplainPlan: ({ explainPlan }) => { openDrawer(ASSISTANT_DRAWER_ID); @@ -103,6 +105,9 @@ export const AssistantProvider: React.FunctionComponent< }, {} ); + track('Assistant Entry Point Used', { + source: 'explain plan', + }); }, interpretConnectionError: ({ connectionInfo, error }) => { openDrawer(ASSISTANT_DRAWER_ID); @@ -122,6 +127,9 @@ export const AssistantProvider: React.FunctionComponent< }, {} ); + track('Assistant Entry Point Used', { + source: 'connection error', + }); }, clearChat: () => { chat.messages = []; diff --git a/packages/compass-telemetry/src/telemetry-events.ts b/packages/compass-telemetry/src/telemetry-events.ts index eb1ca494697..0f9645a39c4 100644 --- a/packages/compass-telemetry/src/telemetry-events.ts +++ b/packages/compass-telemetry/src/telemetry-events.ts @@ -1490,10 +1490,10 @@ type AssistantFeedbackSubmittedEvent = CommonEvent<{ * * @category Gen AI */ -type AssistantEntryPointUsedEvent = ConnectionScopedEvent<{ +type AssistantEntryPointUsedEvent = CommonEvent<{ name: 'Assistant Entry Point Used'; payload: { - source: 'explain plan' | 'performance insights' | 'error message'; + source: 'explain plan' | 'performance insights' | 'connection error'; }; }>; From f5a9683f05c780089ea80cc21731f29ebeaa4dbd Mon Sep 17 00:00:00 2001 From: Gagik Amaryan Date: Thu, 28 Aug 2025 10:36:03 +0200 Subject: [PATCH 6/6] Update packages/compass-assistant/src/assistant-chat.tsx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- packages/compass-assistant/src/assistant-chat.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/compass-assistant/src/assistant-chat.tsx b/packages/compass-assistant/src/assistant-chat.tsx index 3a6468fea3c..0515156af68 100644 --- a/packages/compass-assistant/src/assistant-chat.tsx +++ b/packages/compass-assistant/src/assistant-chat.tsx @@ -109,11 +109,11 @@ export const AssistantChat: React.FunctionComponent = ({ const feedback: 'positive' | 'negative' = rating === 'liked' ? 'positive' : 'negative'; - track('Assistant Feedback Submitted', () => ({ + track('Assistant Feedback Submitted', { feedback, text: textFeedback, request_id: null, - })); + }); }, [track] );