Simple chat: Bring back chat telemetry (#2291)

Brings back the `chat-question:executied` metric for the new chat. I tried to recreate the metadata so it matches the old payloads. Anything we want to add? cc @chenkc805 ## Test plan <img width="1512" alt="Screenshot 2023-12-12 at 11 05 57" src="https://github.com/sourcegraph/cody/assets/458591/5d450d7c-1610-4438-9545-c69bc19fa3dd"> Open the output console and trigger chat messages. Observe the event being logged
sourcegraph · Dec 12, 2023 · dda38f4 · dda38f4
1 parent 493b87a
commit dda38f4
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 4 deletions.
diff --git a/vscode/CHANGELOG.md b/vscode/CHANGELOG.md
@@ -22,6 +22,7 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a
 ### Fixed
 
 - Fixes an issue where the sidebar would not properly load when not signed in. [pull/2267](https://github.com/sourcegraph/cody/pull/2267)
+- Fixes an issue where telemetry events were not properly logged with the new chat experience. [pull/2291](https://github.com/sourcegraph/cody/pull/2291)
 
 ### Changed
 

diff --git a/vscode/src/chat/chat-view/SimpleChatModel.ts b/vscode/src/chat/chat-view/SimpleChatModel.ts
@@ -5,6 +5,7 @@ import { TranscriptJSON } from '@sourcegraph/cody-shared/src/chat/transcript'
 import { InteractionJSON } from '@sourcegraph/cody-shared/src/chat/transcript/interaction'
 import { errorToChatError, InteractionMessage } from '@sourcegraph/cody-shared/src/chat/transcript/messages'
 import { reformatBotMessageForChat } from '@sourcegraph/cody-shared/src/chat/viewHelpers'
+import { ContextFileSource } from '@sourcegraph/cody-shared/src/codebase-context/messages'
 import { Message } from '@sourcegraph/cody-shared/src/sourcegraph-api'
 
 import { contextItemsToContextFiles } from './chat-helpers'
@@ -166,6 +167,7 @@ export interface ContextItem {
     uri: vscode.Uri
     range?: vscode.Range
     text: string
+    source?: ContextFileSource
 }
 
 export function contextItemId(contextItem: ContextItem): string {

diff --git a/vscode/src/chat/chat-view/SimpleChatPanelProvider.ts b/vscode/src/chat/chat-view/SimpleChatPanelProvider.ts
@@ -507,7 +507,24 @@ export class SimpleChatPanelProvider implements vscode.Disposable, IChatPanelPro
         await this.saveSession(text)
         // trigger the context progress indicator
         this.postViewTranscript({ speaker: 'assistant' })
-        await this.generateAssistantResponse(requestID, userContextFiles, addEnhancedContext)
+        await this.generateAssistantResponse(requestID, userContextFiles, addEnhancedContext, contextSummary => {
+            if (submitType !== 'user') {
+                return
+            }
+
+            const properties = {
+                requestID,
+                chatModel: this.chatModel.modelID,
+                promptText: text,
+                contextSummary,
+            }
+            telemetryService.log('CodyVSCodeExtension:recipe:chat-question:executed', properties, {
+                hasV2Event: true,
+            })
+            telemetryRecorder.recordEvent('cody.recipe.chat-question', 'executed', {
+                metadata: { ...contextSummary },
+            })
+        })
         // Set the title of the webview panel to the current text
         if (this.webviewPanel) {
             this.webviewPanel.title = this.history.getChat(this.sessionID)?.chatTitle || getChatPanelTitle(text)
@@ -536,7 +553,8 @@ export class SimpleChatPanelProvider implements vscode.Disposable, IChatPanelPro
     private async generateAssistantResponse(
         requestID: string,
         userContextFiles?: ContextFile[],
-        addEnhancedContext = true
+        addEnhancedContext = true,
+        sendTelemetry?: (contextSummary: {}) => void
     ): Promise<void> {
         try {
             const contextWindowBytes = 28000 // 7000 tokens * 4 bytes per token
@@ -565,6 +583,23 @@ export class SimpleChatPanelProvider implements vscode.Disposable, IChatPanelPro
                 this.postError(new Error(warningMsg))
             }
 
+            if (sendTelemetry) {
+                // Create a summary of how many code snippets of each context source are being
+                // included in the prompt
+                const contextSummary: { [key: string]: number } = {}
+                for (const { source } of newContextUsed) {
+                    if (!source) {
+                        continue
+                    }
+                    if (contextSummary[source]) {
+                        contextSummary[source] += 1
+                    } else {
+                        contextSummary[source] = 1
+                    }
+                }
+                sendTelemetry(contextSummary)
+            }
+
             this.postViewTranscript({ speaker: 'assistant' })
 
             this.sendLLMRequest(prompt, {
@@ -1059,7 +1094,7 @@ class ContextProvider implements IContextProvider {
             return []
         }
         logDebug('SimpleChatPanelProvider', 'getEnhancedContext > searching local embeddings')
-        const contextItems = []
+        const contextItems: ContextItem[] = []
         const embeddingsResults = await this.localEmbeddings.getContext(text, NUM_CODE_RESULTS + NUM_TEXT_RESULTS)
         for (const result of embeddingsResults) {
             const uri = vscode.Uri.from({
@@ -1075,6 +1110,7 @@ class ContextProvider implements IContextProvider {
                 uri,
                 range,
                 text: result.content,
+                source: 'embeddings',
             })
         }
         return contextItems
@@ -1097,7 +1133,7 @@ class ContextProvider implements IContextProvider {
         }
 
         logDebug('SimpleChatPanelProvider', 'getEnhancedContext > searching remote embeddings')
-        const contextItems = []
+        const contextItems: ContextItem[] = []
         const embeddings = await this.embeddingsClient.search([repoId], text, NUM_CODE_RESULTS, NUM_TEXT_RESULTS)
         if (isError(embeddings)) {
             throw new Error(`Error retrieving embeddings: ${embeddings}`)
@@ -1118,6 +1154,7 @@ class ContextProvider implements IContextProvider {
                 uri,
                 range,
                 text: codeResult.content,
+                source: 'embeddings',
             })
         }
 
@@ -1135,6 +1172,7 @@ class ContextProvider implements IContextProvider {
                 uri,
                 range,
                 text: textResult.content,
+                source: 'embeddings',
             })
         }
 
@@ -1324,6 +1362,7 @@ export function deserializedContextFilesToContextItems(
             uri,
             range,
             text: text || '',
+            source: file.source,
         }
     })
 }