Skip to content

chore(compass-assistant): add feedback submission and telemetry COMPASS-9608 #7243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 29, 2025
Merged

chore(compass-assistant): add feedback submission and telemetry COMPASS-9608 #7243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
147b2d7
chore(compass-assistant): add feedback submission and telemetry COMPA…
gagik Aug 27, 2025
f208b6c
chore: add error tracking
gagik Aug 27, 2025
2ffcd45
chore: add error tracking
gagik Aug 27, 2025
cf928e8
chore: revert ai prompt event changes
gagik Aug 27, 2025
d2563d8
chore: add entry point telemetry
gagik Aug 28, 2025
f5a9683
Update packages/compass-assistant/src/assistant-chat.tsx
gagik Aug 28, 2025
3cdcaa4
Merge branch 'main' of github.com:mongodb-js/compass into gagik/feedb…
gagik Aug 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions packages/compass-assistant/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
"@mongodb-js/atlas-service": "^0.56.0",
"@mongodb-js/compass-app-registry": "^9.4.20",
"@mongodb-js/compass-components": "^1.49.0",
"@mongodb-js/compass-telemetry": "^1.14.0",
"@mongodb-js/connection-info": "^0.17.1",
"@mongodb-js/compass-logging": "^1.7.12",
"mongodb-connection-string-url": "^3.0.1",
Expand Down
174 changes: 159 additions & 15 deletions packages/compass-assistant/src/assistant-chat.spec.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,6 @@ import { createMockChat } from '../test/utils';
import type { AssistantMessage } from './compass-assistant-provider';

describe('AssistantChat', function () {
let originalScrollTo: typeof Element.prototype.scrollTo;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

driveby?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah this was meant to be removed earlier just ended up staying

// Mock scrollTo method for DOM elements to prevent test failures
before(function () {
originalScrollTo = Element.prototype.scrollTo.bind(Element.prototype);
Element.prototype.scrollTo = () => {};
});
after(function () {
Element.prototype.scrollTo = originalScrollTo;
});

const mockMessages: AssistantMessage[] = [
{
id: 'user',
Expand All @@ -41,8 +31,9 @@ describe('AssistantChat', function () {

function renderWithChat(messages: AssistantMessage[]) {
const chat = createMockChat({ messages });
const result = render(<AssistantChat chat={chat} />);
return {
result: render(<AssistantChat chat={chat} />),
result,
chat,
};
}
Expand Down Expand Up @@ -130,8 +121,9 @@ describe('AssistantChat', function () {
);
});

it('calls sendMessage when form is submitted', function () {
const { chat } = renderWithChat([]);
it('calls sendMessage when form is submitted', async function () {
const { chat, result } = renderWithChat([]);
const { track } = result;
const inputField = screen.getByPlaceholderText(
'Ask MongoDB Assistant a question'
);
Expand All @@ -142,6 +134,12 @@ describe('AssistantChat', function () {

expect(chat.sendMessage.calledWith({ text: 'What is aggregation?' })).to.be
.true;

await waitFor(() => {
expect(track).to.have.been.calledWith('Assistant Prompt Submitted', {
user_input_length: 'What is aggregation?'.length,
});
});
});

it('clears input field after successful submission', function () {
Expand All @@ -159,8 +157,9 @@ describe('AssistantChat', function () {
expect(inputField.value).to.equal('');
});

it('trims whitespace from input before sending', function () {
const { chat } = renderWithChat([]);
it('trims whitespace from input before sending', async function () {
const { chat, result } = renderWithChat([]);
const { track } = result;

const inputField = screen.getByPlaceholderText(
'Ask MongoDB Assistant a question'
Expand All @@ -171,6 +170,12 @@ describe('AssistantChat', function () {

expect(chat.sendMessage.calledWith({ text: 'What is sharding?' })).to.be
.true;

await waitFor(() => {
expect(track).to.have.been.calledWith('Assistant Prompt Submitted', {
user_input_length: 'What is sharding?'.length,
});
});
});

it('does not call sendMessage when input is empty or whitespace-only', function () {
Expand Down Expand Up @@ -271,4 +276,143 @@ describe('AssistantChat', function () {
expect(screen.queryByText('Another part that should not display.')).to.not
.exist;
});

describe('feedback buttons', function () {
it('shows feedback buttons only for assistant messages', function () {
renderWithChat(mockMessages);

const userMessage = screen.getByTestId('assistant-message-user');
const assistantMessage = screen.getByTestId(
'assistant-message-assistant'
);

// User messages should not have feedback buttons
expect(userMessage.querySelector('[aria-label="Thumbs Up Icon"]')).to.not
.exist;
expect(userMessage.querySelector('[aria-label="Thumbs Down Icon"]')).to
.not.exist;

// Assistant messages should have feedback buttons
expect(assistantMessage.querySelector('[aria-label="Thumbs Up Icon"]')).to
.exist;
expect(assistantMessage.querySelector('[aria-label="Thumbs Down Icon"]'))
.to.exist;
});

it('tracks positive feedback when thumbs up is clicked', async function () {
const { result } = renderWithChat(mockMessages);
const { track } = result;

const assistantMessage = screen.getByTestId(
'assistant-message-assistant'
);

// Find and click the thumbs up button
const thumbsUpButton = assistantMessage.querySelector(
'[aria-label="Thumbs Up Icon"]'
) as HTMLElement;

userEvent.click(thumbsUpButton);

await waitFor(() => {
expect(track).to.have.callCount(1);
expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
feedback: 'positive',
text: undefined,
request_id: null,
});
});
});

it('tracks negative feedback when thumbs down is clicked', async function () {
const { result } = renderWithChat(mockMessages);
const { track } = result;

const assistantMessage = screen.getByTestId(
'assistant-message-assistant'
);

// Find and click the thumbs down button
const thumbsDownButton = assistantMessage.querySelector(
'[aria-label="Thumbs Down Icon"]'
) as HTMLElement;

userEvent.click(thumbsDownButton);

await waitFor(() => {
expect(track).to.have.callCount(1);

expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
feedback: 'negative',
text: undefined,
request_id: null,
});
});
});

it('tracks detailed feedback when feedback text is submitted', async function () {
const { result } = renderWithChat(mockMessages);
const { track } = result;

const assistantMessage = screen.getByTestId(
'assistant-message-assistant'
);

// First click thumbs down to potentially open feedback form
const thumbsDownButton = assistantMessage.querySelector(
'[aria-label="Thumbs Down Icon"]'
) as HTMLElement;

userEvent.click(thumbsDownButton);

// Look for feedback text area (the exact implementation depends on LeafyGreen)
const feedbackTextArea = screen.getByTestId(
'lg-chat-message_actions-feedback_textarea'
);

userEvent.type(feedbackTextArea, 'This response was not helpful');

// Look for submit button
const submitButton = screen.getByText('Submit');

userEvent.click(submitButton);

await waitFor(() => {
expect(track).to.have.callCount(2);

expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
feedback: 'negative',
text: undefined,
request_id: null,
});

expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
feedback: 'negative',
text: 'This response was not helpful',
request_id: null,
});
});
});

it('does not show feedback buttons when there are no assistant messages', function () {
const userOnlyMessages: AssistantMessage[] = [
{
id: 'user1',
role: 'user',
parts: [{ type: 'text', text: 'Hello!' }],
},
{
id: 'user2',
role: 'user',
parts: [{ type: 'text', text: 'How are you?' }],
},
];

renderWithChat(userOnlyMessages);

// Should not find any feedback buttons in the entire component
expect(screen.queryByLabelText('Thumbs Up Icon')).to.not.exist;
expect(screen.queryByLabelText('Thumbs Down Icon')).to.not.exist;
});
});
});
53 changes: 51 additions & 2 deletions packages/compass-assistant/src/assistant-chat.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
LgChatLeafygreenChatProvider,
LgChatMessage,
LgChatMessageFeed,
LgChatMessageActions,
LgChatInputBar,
spacing,
css,
Expand All @@ -16,11 +17,13 @@ import {
palette,
useDarkMode,
} from '@mongodb-js/compass-components';
import { useTelemetry } from '@mongodb-js/compass-telemetry/provider';

const { ChatWindow } = LgChatChatWindow;
const { LeafyGreenChatProvider, Variant } = LgChatLeafygreenChatProvider;
const { Message } = LgChatMessage;
const { MessageFeed } = LgChatMessageFeed;
const { MessageActions } = LgChatMessageActions;
const { InputBar } = LgChatInputBar;

interface AssistantChatProps {
Expand Down Expand Up @@ -105,9 +108,15 @@ const errorBannerWrapperStyles = css({
export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
chat,
}) => {
const track = useTelemetry();
const darkMode = useDarkMode();
const { messages, sendMessage, status, error, clearError } = useChat({
chat,
onError: (error) => {
track('Assistant Response Failed', () => ({
error_name: error.name,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think error name should be safe from any sensitive info right?

Copy link
Contributor

@lerouxb lerouxb Aug 29, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In my experience I don't think there are error names. The errors I saw from inside ai sdk were from new Error('something'). But I suppose some third party dep might be doing more?

Otherwise this is just:

> (new Error('')).name
'Error'

which is probably fine, but not super helpful either.

(This is probably more a limitation of vercel's ai libraries that anything.)

}));
},
});

// Transform AI SDK messages to LeafyGreen chat format
Expand All @@ -127,10 +136,43 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
(messageBody: string) => {
const trimmedMessageBody = messageBody.trim();
if (trimmedMessageBody) {
track('Assistant Prompt Submitted', {
user_input_length: trimmedMessageBody.length,
});
void sendMessage({ text: trimmedMessageBody });
}
},
[sendMessage]
[sendMessage, track]
);

const handleFeedback = useCallback(
(
event,
state:
| {
feedback: string;
rating: string;
}
| {
rating: string;
}
| undefined
) => {
if (!state) {
return;
}
const { rating } = state;
const textFeedback = 'feedback' in state ? state.feedback : undefined;
const feedback: 'positive' | 'negative' =
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

keeping it this way to align with our NLQ AI Prompt feedback wording

rating === 'liked' ? 'positive' : 'negative';

track('Assistant Feedback Submitted', {
feedback,
text: textFeedback,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe if this is a dev build we can just safely include the prompt and the message?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can probably also do that in some follow-up. Let's get this merged.

request_id: null,
});
},
[track]
);

return (
Expand All @@ -155,7 +197,14 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
sourceType="markdown"
{...messageFields}
data-testid={`assistant-message-${messageFields.id}`}
/>
>
{messageFields.isSender === false && (
<MessageActions
onRatingChange={handleFeedback}
onSubmitFeedback={handleFeedback}
/>
)}
</Message>
))}
{status === 'submitted' && (
<Message
Expand Down
8 changes: 8 additions & 0 deletions packages/compass-assistant/src/compass-assistant-provider.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { usePreference } from 'compass-preferences-model/provider';
import { createLoggerLocator } from '@mongodb-js/compass-logging/provider';
import type { ConnectionInfo } from '@mongodb-js/connection-info';
import { redactConnectionString } from 'mongodb-connection-string-url';
import { useTelemetry } from '@mongodb-js/compass-telemetry/provider';

export const ASSISTANT_DRAWER_ID = 'compass-assistant-drawer';

Expand Down Expand Up @@ -88,6 +89,7 @@ export const AssistantProvider: React.FunctionComponent<
chat: Chat<AssistantMessage>;
}>
> = ({ chat, children }) => {
const track = useTelemetry();
const assistantActionsContext = useRef<AssistantActionsContextType>({
interpretExplainPlan: ({ explainPlan }) => {
openDrawer(ASSISTANT_DRAWER_ID);
Expand All @@ -103,6 +105,9 @@ export const AssistantProvider: React.FunctionComponent<
},
{}
);
track('Assistant Entry Point Used', {
source: 'explain plan',
});
},
interpretConnectionError: ({ connectionInfo, error }) => {
openDrawer(ASSISTANT_DRAWER_ID);
Expand All @@ -122,6 +127,9 @@ export const AssistantProvider: React.FunctionComponent<
},
{}
);
track('Assistant Entry Point Used', {
source: 'connection error',
});
},
clearChat: () => {
chat.messages = [];
Expand Down
Loading
Loading