update: log token count for code generated and button click events across the extension

lib/ui/src/Chat.tsx

@@ -103,7 +103,7 @@ export interface FeedbackButtonsProps {
 
 // TODO: Rename to CodeBlockActionsProps
 export interface CopyButtonProps {
-    copyButtonOnSubmit: (text: string, insert?: boolean) => void
+    copyButtonOnSubmit: (text: string, insert?: boolean, event?: 'Keydown' | 'Button') => void
 }
 
 export interface ChatCommandsProps {

lib/ui/src/chat/CodeBlocks.tsx

@@ -67,7 +67,7 @@ function createCopyButton(
         button.textContent = 'Copied'
         setTimeout(() => (button.textContent = 'Copy'), 3000)
         if (copyButtonOnSubmit) {
-            copyButtonOnSubmit('copyButton')
+            copyButtonOnSubmit(text, false)
         }
     })
     return button
@@ -115,6 +115,12 @@ export const CodeBlocks: React.FunctionComponent<CodeBlocksProps> = React.memo(f
                     preElement,
                     createButtons(preText, copyButtonClassName, CopyButtonProps, insertButtonClassName)
                 )
+                // capture copy events (right click or keydown) on code block
+                preElement.addEventListener('copy', () => {
+                    if (CopyButtonProps) {
+                        CopyButtonProps(preText, false, 'Keydown')
+                    }
+                })
             }
         }
     }, [displayText, CopyButtonProps, copyButtonClassName, insertButtonClassName, rootRef])

vscode/CHANGELOG.md

@@ -20,6 +20,8 @@ Starting from `0.2.0`, Cody is using `major.EVEN_NUMBER.patch` for release versi
 - `Explain Code` command now includes visible content of the current file when no code is selected. [pull/602](https://github.com/sourcegraph/cody/pull/602)
 - Cody Commands: Show errors in chat view instead of notification windows. [pull/602](https://github.com/sourcegraph/cody/pull/602)
 - Include the number of accepted characters per autocomplete suggestion. [pull/674](https://github.com/sourcegraph/cody/pull/674)
+- Log all button click events. [pull/675](https://github.com/sourcegraph/cody/pull/675)
+- Log token count for code generated by Cody. [pull/675](https://github.com/sourcegraph/cody/pull/675)
 
 ## [0.6.6]
 

vscode/src/chat/ChatViewProvider.ts

@@ -5,6 +5,7 @@ import { ChatMessage, UserLocalHistory } from '@sourcegraph/cody-shared/src/chat
 
 import { View } from '../../webviews/NavBar'
 import { debug } from '../log'
+import { countCode, matchCodeSnippets } from '../services/InlineAssist'
 
 import { MessageProvider, MessageProviderOptions } from './MessageProvider'
 import { ExtensionMessage, WebviewMessage } from './protocol'
@@ -43,9 +44,11 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
             case 'edit':
                 this.transcript.removeLastInteraction()
                 await this.onHumanMessageSubmitted(message.text, 'user')
+                this.telemetryService.log('CodyVSCodeExtension:editChatButton:clicked')
                 break
             case 'abort':
                 await this.abortCompletion()
+                this.telemetryService.log('CodyVSCodeExtension:abortButton:clicked', { source: 'sidebar' })
                 break
             case 'executeRecipe':
                 await this.setWebviewView('chat')
@@ -67,7 +70,10 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
                 await vscode.commands.executeCommand(`cody.auth.${message.type}`)
                 break
             case 'insert':
-                await this.insertAtCursor(message.text)
+                await this.handleInsertAtCursor(message.text)
+                break
+            case 'copy':
+                await this.handleCopiedCode(message.text, message.eventType)
                 break
             case 'event':
                 this.telemetryService.log(message.eventName, message.properties)
@@ -89,6 +95,7 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
                 break
             case 'reload':
                 await this.authProvider.reloadAuthStatus()
+                this.telemetryService.log('CodyVSCodeExtension:authReloadButton:clicked')
                 break
             case 'openFile':
                 await this.openFilePath(message.filePath)
@@ -104,11 +111,11 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
 
     private async onHumanMessageSubmitted(text: string, submitType: 'user' | 'suggestion' | 'example'): Promise<void> {
         debug('ChatViewProvider:onHumanMessageSubmitted', '', { verbose: { text, submitType } })
+        this.telemetryService.log('CodyVSCodeExtension:chat:submitted', { source: 'sidebar' })
         if (submitType === 'suggestion') {
             this.telemetryService.log('CodyVSCodeExtension:chatPredictions:used')
         }
         if (text === '/') {
-            this.telemetryService.log('CodyVSCodeExtension:custom-command-menu:clicked')
             void vscode.commands.executeCommand('cody.action.commands.menu', true)
             return
         }
@@ -123,7 +130,7 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
      * Process custom command click
      */
     private async onCustomPromptClicked(title: string, commandType: CodyPromptType = 'user'): Promise<void> {
-        this.telemetryService.log('CodyVSCodeExtension:custom-command:clicked')
+        this.telemetryService.log('CodyVSCodeExtension:command:customMenu:clicked')
         debug('ChatViewProvider:onCustomPromptClicked', title)
         if (!this.isCustomCommandAction(title)) {
             await this.setWebviewView('chat')
@@ -175,20 +182,67 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
     }
 
     /**
-     * Insert text at cursor position
-     * Replace selection if there is one
+     * Prevent logging insert events as paste events on doc change
+     */
+    private isInsertEvent = false
+
+    /**
+     * Handles insert event to insert text from code block at cursor position
+     * Replace selection if there is one and then log insert event
      * Note: Using workspaceEdit instead of 'editor.action.insertSnippet' as the later reformats the text incorrectly
      */
-    private async insertAtCursor(text: string): Promise<void> {
+    private async handleInsertAtCursor(text: string): Promise<void> {
+        this.isInsertEvent = true
         const selectionRange = vscode.window.activeTextEditor?.selection
         const editor = vscode.window.activeTextEditor
         if (!editor || !selectionRange) {
             return
         }
+
         const edit = new vscode.WorkspaceEdit()
         // trimEnd() to remove new line added by Cody
         edit.replace(editor.document.uri, selectionRange, text.trimEnd())
         await vscode.workspace.applyEdit(edit)
+
+        // Log insert event
+        const op = 'insert'
+        const { lineCount, charCount } = countCode(text)
+        const eventName = op + 'Button'
+        const args = { op, charCount, lineCount }
+        this.telemetryService.log(`CodyVSCodeExtension:${eventName}:clicked`, args)
+        this.isInsertEvent = false
+    }
+
+    /**
+     * Handles copying code and detecting a paste event.
+     *
+     * @param text - The text from code block when copy event is triggered
+     * @param eventType - Either 'Button' or 'Keydown'
+     */
+    private async handleCopiedCode(text: string, eventType: 'Button' | 'Keydown'): Promise<void> {
+        // If it's a Button event, then the text is already passed in from the whole code block
+        const copiedCode = eventType === 'Button' ? text : await vscode.env.clipboard.readText()
+
+        // Log Copy event
+        const op = 'copy'
+        const { lineCount, charCount } = countCode(copiedCode)
+        const eventName = op + eventType
+        const args = { op, charCount, lineCount }
+        this.telemetryService.log(`CodyVSCodeExtension:${eventName}:clicked`, args)
+
+        // Create listener for changes to the active text editor for paste event
+        vscode.workspace.onDidChangeTextDocument(e => {
+            const changedText = e.contentChanges[0]?.text
+            // check if the copied code is the same as the changed text without spaces
+            const isMatched = matchCodeSnippets(copiedCode, changedText)
+            // Log paste event when the copied code is pasted
+            if (!this.isInsertEvent && isMatched) {
+                this.telemetryService.log('CodyVSCodeExtension:pasteKeydown:clicked', {
+                    ...args,
+                    op: 'paste',
+                })
+            }
+        })
     }
 
     protected handleEnabledPlugins(plugins: string[]): void {

vscode/src/chat/MessageProvider.ts

@@ -28,6 +28,7 @@ import { LocalStorage } from '../services/LocalStorageProvider'
 import { TestSupport } from '../test-support'
 
 import { ContextProvider } from './ContextProvider'
+import { countGeneratedCode } from './utils'
 
 /**
  * The problem with a token limit for the prompt is that we can only
@@ -138,12 +139,14 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
         this.handleSuggestions([])
         this.sendTranscript()
         this.sendHistory()
+        this.telemetryService.log('CodyVSCodeExtension:chatReset:executed')
     }
 
     public async clearHistory(): Promise<void> {
         MessageProvider.chatHistory = {}
         MessageProvider.inputHistory = []
         await this.localStorage.removeChatHistory()
+        this.telemetryService.log('CodyVSCodeExtension:clearChatHistoryButton:clicked')
     }
 
     /**
@@ -157,6 +160,7 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
         await this.transcript.toJSON()
         this.sendTranscript()
         this.sendHistory()
+        this.telemetryService.log('CodyVSCodeExtension:restoreChatHistoryButton:clicked')
     }
 
     private sendEnabledPlugins(plugins: string[]): void {
@@ -194,6 +198,10 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
                     this.transcript.addAssistantResponse(text || '', displayText)
                 }
                 await this.onCompletionEnd()
+                // Count code generated from response
+                const codeCount = countGeneratedCode(text)
+                const op = codeCount ? 'hasCode' : 'noCode'
+                this.telemetryService.log('CodyVSCodeExtension:chatResponse:' + op, codeCount || {})
             },
         })
 
@@ -511,14 +519,13 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
             case 'menu':
                 await this.editor.controllers.command?.menu('custom')
                 await this.sendCodyCommands()
-                this.telemetryService.log('CodyVSCodeExtension:command:menu:opened')
                 break
             case 'add':
                 if (!type) {
                     break
                 }
                 await this.editor.controllers.command?.config('add', type)
-                this.telemetryService.log('CodyVSCodeExtension:command:addCommand')
+                this.telemetryService.log('CodyVSCodeExtension:addCommandButton:clicked')
                 break
         }
         // Get prompt details from controller by title then execute prompt's command
@@ -529,7 +536,7 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
             return
         }
         await this.executeRecipe('custom-prompt', promptText)
-        this.telemetryService.log('CodyVSCodeExtension:command:executedFromMenu')
+        this.telemetryService.log('CodyVSCodeExtension:command:started', { source: 'menu' })
         const starter = (await this.editor.controllers.command?.getCustomConfig())?.starter
         if (starter) {
             this.telemetryService.log('CodyVSCodeExtension:command:customStarter:applied')
@@ -546,25 +553,29 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
             return { text, recipeId }
         }
         const commandKey = text.split(' ')[0].replace('/', '')
-        this.telemetryService.log(`CodyVSCodeExtension:command:${commandKey}:filtering`)
         switch (true) {
             case text === '/':
-                return vscode.commands.executeCommand('cody.action.commands.menu')
+                return vscode.commands.executeCommand('cody.action.commands.menu', 'sidebar')
             case text === '/commands-settings':
+                this.telemetryService.log('CodyVSCodeExtension:command:configMenuButton:clicked', { source: 'sidebar' })
                 return vscode.commands.executeCommand('cody.settings.commands')
             case /^\/o(pen)?\s/.test(text) && this.editor.controllers.command !== undefined:
                 // open the user's ~/.vscode/cody.json file
                 await this.editor.controllers.command?.open(text.split(' ')[1])
+                this.telemetryService.log('CodyVSCodeExtension:command:openFile:executed')
                 return null
             case /^\/r(eset)?$/.test(text):
                 await this.clearAndRestartSession()
+                this.telemetryService.log('CodyVSCodeExtension:command:resetChat:executed')
                 return null
             case /^\/s(earch)?\s/.test(text):
                 return { text, recipeId: 'context-search' }
             case /^\/f(ix)?\s.*$/.test(text):
                 return { text, recipeId: 'fixup' }
             case /^\/(explain|doc|test|smell)$/.test(text):
-                this.telemetryService.log(`CodyVSCodeExtension:command:${text.replace('/', '')}:executed`)
+                this.telemetryService.log(`CodyVSCodeExtension:command:${commandKey}:called`, {
+                    source: 'chat',
+                })
             default: {
                 if (!this.editor.getActiveTextEditor()?.filePath) {
                     await this.addCustomInteraction('Command failed. Please open a file and try again.', text)
@@ -573,7 +584,6 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
                 const promptText = this.editor.controllers.command?.find(text, true)
                 await this.editor.controllers.command?.get('command')
                 if (promptText) {
-                    this.telemetryService.log(`CodyVSCodeExtension:command:${commandKey}:executing`)
                     return { text: promptText, recipeId: 'custom-prompt' }
                 }
                 // Inline chat has its own filter for slash commands
@@ -647,6 +657,7 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
         delete MessageProvider.chatHistory[chatID]
         await this.localStorage.deleteChatHistory(chatID)
         this.sendHistory()
+        this.telemetryService.log('CodyVSCodeExtension:deleteChatHistoryButton:clicked')
     }
 
     /**

vscode/src/chat/protocol.ts

@@ -23,7 +23,8 @@ export type WebviewMessage =
     | { command: 'links'; value: string }
     | { command: 'openFile'; filePath: string }
     | { command: 'edit'; text: string }
-    | { command: 'insert'; text: string }
+    | { command: 'insert'; eventType: 'Button' | 'Keydown'; text: string }
+    | { command: 'copy'; eventType: 'Button' | 'Keydown'; text: string }
     | {
           command: 'auth'
           type: 'signin' | 'signout' | 'support' | 'app' | 'callback'

vscode/src/chat/utils.ts

@@ -82,3 +82,31 @@ export function newAuthStatus(
     authStatus.isLoggedIn = isLoggedIn && isAllowed
     return authStatus
 }
+
+/**
+ * Counts the number of lines and characters in code blocks in a given string.
+ *
+ * @param text - The string to search for code blocks.
+ * @returns An object with the total lineCount and charCount of code in code blocks,
+ * or null if no code blocks are found.
+ */
+export const countGeneratedCode = (text: string): { lineCount: number; charCount: number } | null => {
+    const codeBlockRegex = /```[\S\s]*?```/g
+    const codeBlocks = text.match(codeBlockRegex)
+    if (!codeBlocks) {
+        return null
+    }
+    const count = { lineCount: 0, charCount: 0 }
+    const backticks = '```'
+    for (const block of codeBlocks) {
+        const lines = block.split('\n')
+        const codeLines = lines.filter(line => !line.startsWith(backticks))
+        const lineCount = codeLines.length
+        const language = lines[0].replace(backticks, '')
+        // 2 backticks + 2 newline
+        const charCount = block.length - language.length - backticks.length * 2 - 2
+        count.charCount += charCount
+        count.lineCount += lineCount
+    }
+    return count
+}

vscode/src/completions/logger.ts

@@ -114,7 +114,7 @@ export function suggested(id: string, source: string): void {
     }
 }
 
-export function accept(id: string, lines: number, chars: number): void {
+export function accept(id: string, lineCount: number, charCount: number): void {
     const completionEvent = displayedCompletions.get(id)
     if (!completionEvent || completionEvent.acceptedAt) {
         // Log a debug event, this case should not happen in production
@@ -139,8 +139,8 @@ export function accept(id: string, lines: number, chars: number): void {
     logSuggestionEvents()
     logCompletionEvent('accepted', {
         ...completionEvent.params,
-        lines,
-        chars,
+        lineCount,
+        charCount,
         otherCompletionProviderEnabled: otherCompletionProviderEnabled(),
     })
 }

vscode/src/custom-prompts/CommandsController.ts

@@ -118,10 +118,8 @@ export class CommandsController implements VsCodeCommandsController, vscode.Disp
             this.lastUsedCommands.add(id)
         }
 
-        if (myPrompt?.type === 'default') {
-            this.telemetryService.log(`CodyVSCodeExtension:command:${myPrompt.slashCommand}:executed`)
-        }
-
+        const commandName = myPrompt?.type === 'default' ? myPrompt.slashCommand?.replace('/', '') : 'custom'
+        this.telemetryService.log(`CodyVSCodeExtension:command:${commandName}:called`)
         return myPrompt?.prompt || ''
     }
 
@@ -166,8 +164,8 @@ export class CommandsController implements VsCodeCommandsController, vscode.Disp
     /**
      * Menu Controller
      */
-    public async menu(type: 'custom' | 'config' | 'default', showDesc?: boolean): Promise<void> {
-        this.telemetryService.log(`CodyVSCodeExtension:command:menu:${type}`)
+    public async menu(type: 'custom' | 'config' | 'default', showDesc = true): Promise<void> {
+        this.telemetryService.log('CodyVSCodeExtension:command:menu:opened', { type })
         await this.refresh()
         switch (type) {
             case 'custom':
@@ -198,7 +196,7 @@ export class CommandsController implements VsCodeCommandsController, vscode.Disp
     /**
      * Main Menu: Cody Commands
      */
-    public async mainCommandMenu(showDesc = false): Promise<void> {
+    public async mainCommandMenu(showDesc = true): Promise<void> {
         try {
             const commandItems = [menu_separators.inline, menu_options.chat, menu_options.fix, menu_separators.commands]
             const allCommands = this.default.getGroupedCommands(true)

vscode/src/editor/EditorCodeLenses.ts

@@ -70,7 +70,7 @@ export class EditorCodeLenses implements vscode.CodeLensProvider {
         if (activeEditor) {
             activeEditor.selection = lens.selection
         }
-        await vscode.commands.executeCommand(lens.name, false)
+        await vscode.commands.executeCommand(lens.name, 'codeLens')
     }
     /**
      * Gets the code lenses for the specified document.