diff --git a/package-lock.json b/package-lock.json index afd1683a348..792c6c7e49e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47176,6 +47176,7 @@ "@mongodb-js/compass-app-registry": "^9.4.20", "@mongodb-js/compass-components": "^1.49.0", "@mongodb-js/compass-logging": "^1.7.12", + "@mongodb-js/compass-telemetry": "^1.14.0", "@mongodb-js/connection-info": "^0.17.1", "ai": "^5.0.26", "compass-preferences-model": "^2.51.0", @@ -60580,6 +60581,7 @@ "@mongodb-js/compass-app-registry": "^9.4.20", "@mongodb-js/compass-components": "^1.49.0", "@mongodb-js/compass-logging": "^1.7.12", + "@mongodb-js/compass-telemetry": "^1.14.0", "@mongodb-js/connection-info": "^0.17.1", "@mongodb-js/eslint-config-compass": "^1.4.7", "@mongodb-js/mocha-config-compass": "^1.7.0", diff --git a/packages/compass-assistant/package.json b/packages/compass-assistant/package.json index f7b4613601d..48d2832bdcb 100644 --- a/packages/compass-assistant/package.json +++ b/packages/compass-assistant/package.json @@ -53,6 +53,7 @@ "@mongodb-js/atlas-service": "^0.56.0", "@mongodb-js/compass-app-registry": "^9.4.20", "@mongodb-js/compass-components": "^1.49.0", + "@mongodb-js/compass-telemetry": "^1.14.0", "@mongodb-js/connection-info": "^0.17.1", "@mongodb-js/compass-logging": "^1.7.12", "mongodb-connection-string-url": "^3.0.1", diff --git a/packages/compass-assistant/src/assistant-chat.spec.tsx b/packages/compass-assistant/src/assistant-chat.spec.tsx index 97234430d6e..8f0714ef95e 100644 --- a/packages/compass-assistant/src/assistant-chat.spec.tsx +++ b/packages/compass-assistant/src/assistant-chat.spec.tsx @@ -11,16 +11,6 @@ import { createMockChat } from '../test/utils'; import type { AssistantMessage } from './compass-assistant-provider'; describe('AssistantChat', function () { - let originalScrollTo: typeof Element.prototype.scrollTo; - // Mock scrollTo method for DOM elements to prevent test failures - before(function () { - originalScrollTo = Element.prototype.scrollTo.bind(Element.prototype); - Element.prototype.scrollTo = () => {}; - }); - after(function () { - Element.prototype.scrollTo = originalScrollTo; - }); - const mockMessages: AssistantMessage[] = [ { id: 'user', @@ -41,8 +31,9 @@ describe('AssistantChat', function () { function renderWithChat(messages: AssistantMessage[]) { const chat = createMockChat({ messages }); + const result = render(); return { - result: render(), + result, chat, }; } @@ -130,8 +121,9 @@ describe('AssistantChat', function () { ); }); - it('calls sendMessage when form is submitted', function () { - const { chat } = renderWithChat([]); + it('calls sendMessage when form is submitted', async function () { + const { chat, result } = renderWithChat([]); + const { track } = result; const inputField = screen.getByPlaceholderText( 'Ask MongoDB Assistant a question' ); @@ -142,6 +134,12 @@ describe('AssistantChat', function () { expect(chat.sendMessage.calledWith({ text: 'What is aggregation?' })).to.be .true; + + await waitFor(() => { + expect(track).to.have.been.calledWith('Assistant Prompt Submitted', { + user_input_length: 'What is aggregation?'.length, + }); + }); }); it('clears input field after successful submission', function () { @@ -159,8 +157,9 @@ describe('AssistantChat', function () { expect(inputField.value).to.equal(''); }); - it('trims whitespace from input before sending', function () { - const { chat } = renderWithChat([]); + it('trims whitespace from input before sending', async function () { + const { chat, result } = renderWithChat([]); + const { track } = result; const inputField = screen.getByPlaceholderText( 'Ask MongoDB Assistant a question' @@ -171,6 +170,12 @@ describe('AssistantChat', function () { expect(chat.sendMessage.calledWith({ text: 'What is sharding?' })).to.be .true; + + await waitFor(() => { + expect(track).to.have.been.calledWith('Assistant Prompt Submitted', { + user_input_length: 'What is sharding?'.length, + }); + }); }); it('does not call sendMessage when input is empty or whitespace-only', function () { @@ -271,4 +276,143 @@ describe('AssistantChat', function () { expect(screen.queryByText('Another part that should not display.')).to.not .exist; }); + + describe('feedback buttons', function () { + it('shows feedback buttons only for assistant messages', function () { + renderWithChat(mockMessages); + + const userMessage = screen.getByTestId('assistant-message-user'); + const assistantMessage = screen.getByTestId( + 'assistant-message-assistant' + ); + + // User messages should not have feedback buttons + expect(userMessage.querySelector('[aria-label="Thumbs Up Icon"]')).to.not + .exist; + expect(userMessage.querySelector('[aria-label="Thumbs Down Icon"]')).to + .not.exist; + + // Assistant messages should have feedback buttons + expect(assistantMessage.querySelector('[aria-label="Thumbs Up Icon"]')).to + .exist; + expect(assistantMessage.querySelector('[aria-label="Thumbs Down Icon"]')) + .to.exist; + }); + + it('tracks positive feedback when thumbs up is clicked', async function () { + const { result } = renderWithChat(mockMessages); + const { track } = result; + + const assistantMessage = screen.getByTestId( + 'assistant-message-assistant' + ); + + // Find and click the thumbs up button + const thumbsUpButton = assistantMessage.querySelector( + '[aria-label="Thumbs Up Icon"]' + ) as HTMLElement; + + userEvent.click(thumbsUpButton); + + await waitFor(() => { + expect(track).to.have.callCount(1); + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'positive', + text: undefined, + request_id: null, + }); + }); + }); + + it('tracks negative feedback when thumbs down is clicked', async function () { + const { result } = renderWithChat(mockMessages); + const { track } = result; + + const assistantMessage = screen.getByTestId( + 'assistant-message-assistant' + ); + + // Find and click the thumbs down button + const thumbsDownButton = assistantMessage.querySelector( + '[aria-label="Thumbs Down Icon"]' + ) as HTMLElement; + + userEvent.click(thumbsDownButton); + + await waitFor(() => { + expect(track).to.have.callCount(1); + + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'negative', + text: undefined, + request_id: null, + }); + }); + }); + + it('tracks detailed feedback when feedback text is submitted', async function () { + const { result } = renderWithChat(mockMessages); + const { track } = result; + + const assistantMessage = screen.getByTestId( + 'assistant-message-assistant' + ); + + // First click thumbs down to potentially open feedback form + const thumbsDownButton = assistantMessage.querySelector( + '[aria-label="Thumbs Down Icon"]' + ) as HTMLElement; + + userEvent.click(thumbsDownButton); + + // Look for feedback text area (the exact implementation depends on LeafyGreen) + const feedbackTextArea = screen.getByTestId( + 'lg-chat-message_actions-feedback_textarea' + ); + + userEvent.type(feedbackTextArea, 'This response was not helpful'); + + // Look for submit button + const submitButton = screen.getByText('Submit'); + + userEvent.click(submitButton); + + await waitFor(() => { + expect(track).to.have.callCount(2); + + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'negative', + text: undefined, + request_id: null, + }); + + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'negative', + text: 'This response was not helpful', + request_id: null, + }); + }); + }); + + it('does not show feedback buttons when there are no assistant messages', function () { + const userOnlyMessages: AssistantMessage[] = [ + { + id: 'user1', + role: 'user', + parts: [{ type: 'text', text: 'Hello!' }], + }, + { + id: 'user2', + role: 'user', + parts: [{ type: 'text', text: 'How are you?' }], + }, + ]; + + renderWithChat(userOnlyMessages); + + // Should not find any feedback buttons in the entire component + expect(screen.queryByLabelText('Thumbs Up Icon')).to.not.exist; + expect(screen.queryByLabelText('Thumbs Down Icon')).to.not.exist; + }); + }); }); diff --git a/packages/compass-assistant/src/assistant-chat.tsx b/packages/compass-assistant/src/assistant-chat.tsx index 75cee85df13..4f5c67eaf0c 100644 --- a/packages/compass-assistant/src/assistant-chat.tsx +++ b/packages/compass-assistant/src/assistant-chat.tsx @@ -7,6 +7,7 @@ import { LgChatLeafygreenChatProvider, LgChatMessage, LgChatMessageFeed, + LgChatMessageActions, LgChatInputBar, spacing, css, @@ -16,11 +17,13 @@ import { palette, useDarkMode, } from '@mongodb-js/compass-components'; +import { useTelemetry } from '@mongodb-js/compass-telemetry/provider'; const { ChatWindow } = LgChatChatWindow; const { LeafyGreenChatProvider, Variant } = LgChatLeafygreenChatProvider; const { Message } = LgChatMessage; const { MessageFeed } = LgChatMessageFeed; +const { MessageActions } = LgChatMessageActions; const { InputBar } = LgChatInputBar; interface AssistantChatProps { @@ -105,9 +108,15 @@ const errorBannerWrapperStyles = css({ export const AssistantChat: React.FunctionComponent = ({ chat, }) => { + const track = useTelemetry(); const darkMode = useDarkMode(); const { messages, sendMessage, status, error, clearError } = useChat({ chat, + onError: (error) => { + track('Assistant Response Failed', () => ({ + error_name: error.name, + })); + }, }); // Transform AI SDK messages to LeafyGreen chat format @@ -127,10 +136,43 @@ export const AssistantChat: React.FunctionComponent = ({ (messageBody: string) => { const trimmedMessageBody = messageBody.trim(); if (trimmedMessageBody) { + track('Assistant Prompt Submitted', { + user_input_length: trimmedMessageBody.length, + }); void sendMessage({ text: trimmedMessageBody }); } }, - [sendMessage] + [sendMessage, track] + ); + + const handleFeedback = useCallback( + ( + event, + state: + | { + feedback: string; + rating: string; + } + | { + rating: string; + } + | undefined + ) => { + if (!state) { + return; + } + const { rating } = state; + const textFeedback = 'feedback' in state ? state.feedback : undefined; + const feedback: 'positive' | 'negative' = + rating === 'liked' ? 'positive' : 'negative'; + + track('Assistant Feedback Submitted', { + feedback, + text: textFeedback, + request_id: null, + }); + }, + [track] ); return ( @@ -155,7 +197,14 @@ export const AssistantChat: React.FunctionComponent = ({ sourceType="markdown" {...messageFields} data-testid={`assistant-message-${messageFields.id}`} - /> + > + {messageFields.isSender === false && ( + + )} + ))} {status === 'submitted' && ( ; }> > = ({ chat, children }) => { + const track = useTelemetry(); const assistantActionsContext = useRef({ interpretExplainPlan: ({ explainPlan }) => { openDrawer(ASSISTANT_DRAWER_ID); @@ -103,6 +105,9 @@ export const AssistantProvider: React.FunctionComponent< }, {} ); + track('Assistant Entry Point Used', { + source: 'explain plan', + }); }, interpretConnectionError: ({ connectionInfo, error }) => { openDrawer(ASSISTANT_DRAWER_ID); @@ -122,6 +127,9 @@ export const AssistantProvider: React.FunctionComponent< }, {} ); + track('Assistant Entry Point Used', { + source: 'connection error', + }); }, clearChat: () => { chat.messages = []; diff --git a/packages/compass-telemetry/src/telemetry-events.ts b/packages/compass-telemetry/src/telemetry-events.ts index dadcc8efb40..0f9645a39c4 100644 --- a/packages/compass-telemetry/src/telemetry-events.ts +++ b/packages/compass-telemetry/src/telemetry-events.ts @@ -1458,6 +1458,57 @@ type IndexDroppedEvent = ConnectionScopedEvent<{ }; }>; +/** + * This event is fired when user enters a prompt in the assistant chat + * and hits "enter". + * + * @category Gen AI + */ +type AssistantPromptSubmittedEvent = CommonEvent<{ + name: 'Assistant Prompt Submitted'; + payload: { + user_input_length?: number; + }; +}>; + +/** + * This event is fired when a user submits feedback for the assistant. + * + * @category Assistant + */ +type AssistantFeedbackSubmittedEvent = CommonEvent<{ + name: 'Assistant Feedback Submitted'; + payload: { + feedback: 'positive' | 'negative'; + text: string | undefined; + request_id: string | null; + }; +}>; + +/** + * This event is fired when a user uses an assistant entry point. + * + * @category Gen AI + */ +type AssistantEntryPointUsedEvent = CommonEvent<{ + name: 'Assistant Entry Point Used'; + payload: { + source: 'explain plan' | 'performance insights' | 'connection error'; + }; +}>; + +/** + * This event is fired when the AI response encounters an error. + * + * @category Gen AI + */ +type AssistantResponseFailedEvent = CommonEvent<{ + name: 'Assistant Response Failed'; + payload: { + error_name?: string; + }; +}>; + /** * This event is fired when a user submits feedback for a query generation. * @@ -2992,6 +3043,10 @@ export type TelemetryEvent = | AggregationTimedOutEvent | AggregationUseCaseAddedEvent | AggregationUseCaseSavedEvent + | AssistantPromptSubmittedEvent + | AssistantResponseFailedEvent + | AssistantFeedbackSubmittedEvent + | AssistantEntryPointUsedEvent | AiOptInModalShownEvent | AiOptInModalDismissedEvent | AiSignInModalShownEvent