Simple chat: Bring back chat telemetry

vscode/CHANGELOG.md

@@ -22,6 +22,7 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a
 ### Fixed
 
 - Fixes an issue where the sidebar would not properly load when not signed in. [pull/2267](https://github.com/sourcegraph/cody/pull/2267)
+- Fixes an issue where telemetry events were not properly logged with the new chat experience. [pull/2291](https://github.com/sourcegraph/cody/pull/2291)
 
 ### Changed
 

vscode/src/chat/chat-view/SimpleChatModel.ts

@@ -5,6 +5,7 @@ import { TranscriptJSON } from '@sourcegraph/cody-shared/src/chat/transcript'
 import { InteractionJSON } from '@sourcegraph/cody-shared/src/chat/transcript/interaction'
 import { errorToChatError, InteractionMessage } from '@sourcegraph/cody-shared/src/chat/transcript/messages'
 import { reformatBotMessageForChat } from '@sourcegraph/cody-shared/src/chat/viewHelpers'
+import { ContextFileSource } from '@sourcegraph/cody-shared/src/codebase-context/messages'
 import { Message } from '@sourcegraph/cody-shared/src/sourcegraph-api'
 
 import { contextItemsToContextFiles } from './chat-helpers'
@@ -166,6 +167,7 @@ export interface ContextItem {
     uri: vscode.Uri
     range?: vscode.Range
     text: string
+    source?: ContextFileSource
 }
 
 export function contextItemId(contextItem: ContextItem): string {

vscode/src/chat/chat-view/SimpleChatPanelProvider.ts

@@ -507,7 +507,12 @@ export class SimpleChatPanelProvider implements vscode.Disposable, IChatPanelPro
         await this.saveSession(text)
         // trigger the context progress indicator
         this.postViewTranscript({ speaker: 'assistant' })
-        await this.generateAssistantResponse(requestID, userContextFiles, addEnhancedContext)
+        await this.generateAssistantResponse(
+            requestID,
+            userContextFiles,
+            addEnhancedContext,
+            submitType === 'user' ? text : undefined
+        )
         // Set the title of the webview panel to the current text
         if (this.webviewPanel) {
             this.webviewPanel.title = this.history.getChat(this.sessionID)?.chatTitle || getChatPanelTitle(text)
@@ -536,7 +541,8 @@ export class SimpleChatPanelProvider implements vscode.Disposable, IChatPanelPro
     private async generateAssistantResponse(
         requestID: string,
         userContextFiles?: ContextFile[],
-        addEnhancedContext = true
+        addEnhancedContext = true,
+        userPrompt: string | undefined = undefined
     ): Promise<void> {
         try {
             const contextWindowBytes = 28000 // 7000 tokens * 4 bytes per token
@@ -565,6 +571,35 @@ export class SimpleChatPanelProvider implements vscode.Disposable, IChatPanelPro
                 this.postError(new Error(warningMsg))
             }
 
+            if (userPrompt) {
+                // Create a summary of how many code snippets of each context source are being
+                // included in the prompt
+                const contextSummary: { [key: string]: number } = {}
+                for (const { source } of newContextUsed) {
+                    if (!source) {
+                        continue
+                    }
+                    if (contextSummary[source]) {
+                        contextSummary[source] += 1
+                    } else {
+                        contextSummary[source] = 1
+                    }
+                }
+
+                const properties = {
+                    requestID,
+                    chatModel: this.chatModel.modelID,
+                    promptText: userPrompt,
+                    contextSummary,
+                }
+                telemetryService.log('CodyVSCodeExtension:recipe:chat-question:executed', properties, {
+                    hasV2Event: true,
+                })
+                telemetryRecorder.recordEvent('cody.recipe.chat-question', 'executed', {
+                    metadata: { ...contextSummary },
+                })
+            }
+
             this.postViewTranscript({ speaker: 'assistant' })
 
             this.sendLLMRequest(prompt, {
@@ -1059,7 +1094,7 @@ class ContextProvider implements IContextProvider {
             return []
         }
         logDebug('SimpleChatPanelProvider', 'getEnhancedContext > searching local embeddings')
-        const contextItems = []
+        const contextItems: ContextItem[] = []
         const embeddingsResults = await this.localEmbeddings.getContext(text, NUM_CODE_RESULTS + NUM_TEXT_RESULTS)
         for (const result of embeddingsResults) {
             const uri = vscode.Uri.from({
@@ -1075,6 +1110,7 @@ class ContextProvider implements IContextProvider {
                 uri,
                 range,
                 text: result.content,
+                source: 'embeddings',
             })
         }
         return contextItems
@@ -1097,7 +1133,7 @@ class ContextProvider implements IContextProvider {
         }
 
         logDebug('SimpleChatPanelProvider', 'getEnhancedContext > searching remote embeddings')
-        const contextItems = []
+        const contextItems: ContextItem[] = []
         const embeddings = await this.embeddingsClient.search([repoId], text, NUM_CODE_RESULTS, NUM_TEXT_RESULTS)
         if (isError(embeddings)) {
             throw new Error(`Error retrieving embeddings: ${embeddings}`)
@@ -1118,6 +1154,7 @@ class ContextProvider implements IContextProvider {
                 uri,
                 range,
                 text: codeResult.content,
+                source: 'embeddings',
             })
         }
 
@@ -1135,6 +1172,7 @@ class ContextProvider implements IContextProvider {
                 uri,
                 range,
                 text: textResult.content,
+                source: 'embeddings',
             })
         }
 
@@ -1324,6 +1362,7 @@ export function deserializedContextFilesToContextItems(
             uri,
             range,
             text: text || '',
+            source: file.source,
         }
     })
 }

vscode/src/chat/chat-view/prompt.ts

@@ -82,6 +82,7 @@ export class DefaultPrompter implements IPrompter {
                 (item: ContextItem) => this.renderContextItem(item)
             )
             newContextUsed.push(...used)
+
             if (limitReached) {
                 warnings.push('Ignored current user-specified context items due to context limit')
                 return { prompt: promptBuilder.build(), warnings, newContextUsed }