update: log token count for code generated and button click events ac…

…ross the extension (#675) RE: https://docs.google.com/document/d/1UcjUdon1XO5GoLDIJJCGBq1zyhR6VfL2FE_apSPeLQo/edit#heading=h.m8ojuhhojtjm Close #672 Add loggings for following events | Description | Type | Example Log Entry | |----------------------------------------------------------------------------------|------------|-------------------------------------------------------------------------------------------| | Chat submit from sidebar | submit | CodyVSCodeExtension:chat:submitted {"source":"chat"} | | Chat submit from inline | submit | CodyVSCodeExtension:chat:submitted {"source":"inline"} | | Chat submit from command menu | submit | CodyVSCodeExtension:chat:submitted {"source":"menu"} | | Code generated for chat | response | CodyVSCodeExtension:chatResponse:noCode {} | | Code generated for chat | response | CodyVSCodeExtension:chatResponse:hasCode {"lineCount":20,"charCount":575} | | Fixup code lens button: apply | click | CodyVSCodeExtension:fixup:codeLens:clicked', { op: apply) | | Fixup code lens button: show diff | click | CodyVSCodeExtension:fixup:codeLens:clicked', { op: diff) | | Fixup code lens button: cancel | click | CodyVSCodeExtension:fixup:codeLens:clicked', { op: cancel) | | Fixup token count for code applied | applied | CodyVSCodeExtension:fixup:applied {"lineCount":20,"charCount":575} | | Chat title bar button: new convo | click | CodyVSCodeExtension:chatTitleButton:clicked {"name":"reset"} | | Chat title bar button: history | click | CodyVSCodeExtension:chatTitleButton:clicked {"name":"history"} | | Copy event on code generated for Inline Chat | keydown | CodyVSCodeExtension:inlineChat:event:detected {"op":"copy","lineCount":5,"charCount":76} | | Paste event on code generated for Inline Chat | keydown | CodyVSCodeExtension:inlineChat:event:detected {"op":"paste","lineCount":5,"charCount":76} | | Copy event on code generated for side bar chat triggered by copy button click: | click | CodyVSCodeExtension:copyButton:clicked {"lineCount":2,"charCount":27,"op":"copy"} | | Paste event on code generated for side bar chat triggered by copy button click: | keydown | CodyVSCodeExtension:pasteButton:clicked {"lineCount":2,"charCount":27,"op":"paste"} | | Copy event on code generated for side bar chat, but not triggered by buttons | keydown | CodyVSCodeExtension:copyKeydown:clicked {"lineCount":2,"charCount":27,"op":"copy"} | | Paste event on code generated for side bar chat, but not triggered by buttons | keydown | CodyVSCodeExtension:pasteKeydown:clicked {"lineCount":2,"charCount":27,"op":"paste"} | | Insert code block button | click | CodyVSCodeExtension:insertButton:clicked {“op”:“insert”,“charCount”:194,“lineCount”:11} | | On “Stop Generating” button click in sidebar | click | CodyVSCodeExtension:abortButton:clicked' { source: 'sidebar' } | | On “Stop Generating” button click in Inline Chat | click | CodyVSCodeExtension:abortButton:clicked' { source: inline } | | Code generated for fixup task | response | CodyVSCodeExtension:fixupResponse:hasCode {"lineCount":12,"charCount":354} | `CodyVSCodeExtension:completion:accepted` was added by @philipp-spiess in #674 `CodyVSCodeExtension:completion:suggested` is not added. Will work on this in a separated PR ## Test plan  Added test for token count methods. To test the newly added token events Start Cody from this branch in debug mode, and check the Output channel for logged events after performing an action: ![image](https://github.com/sourcegraph/cody/assets/68532117/f71dc5e9-0217-43bf-be64-1a783dcae5cc)
sourcegraph · Aug 14, 2023 · 690c9c5 · 690c9c5
1 parent b018cda
commit 690c9c5
Show file tree

Hide file tree

Showing 16 changed files with 333 additions and 58 deletions.
diff --git a/lib/ui/src/Chat.tsx b/lib/ui/src/Chat.tsx
@@ -103,7 +103,7 @@ export interface FeedbackButtonsProps {
 
 // TODO: Rename to CodeBlockActionsProps
 export interface CopyButtonProps {
-    copyButtonOnSubmit: (text: string, insert?: boolean) => void
+    copyButtonOnSubmit: (text: string, insert?: boolean, event?: 'Keydown' | 'Button') => void
 }
 
 export interface ChatCommandsProps {

diff --git a/lib/ui/src/chat/CodeBlocks.tsx b/lib/ui/src/chat/CodeBlocks.tsx
@@ -67,7 +67,7 @@ function createCopyButton(
         button.textContent = 'Copied'
         setTimeout(() => (button.textContent = 'Copy'), 3000)
         if (copyButtonOnSubmit) {
-            copyButtonOnSubmit('copyButton')
+            copyButtonOnSubmit(text, false)
         }
     })
     return button
@@ -115,6 +115,12 @@ export const CodeBlocks: React.FunctionComponent<CodeBlocksProps> = React.memo(f
                     preElement,
                     createButtons(preText, copyButtonClassName, CopyButtonProps, insertButtonClassName)
                 )
+                // capture copy events (right click or keydown) on code block
+                preElement.addEventListener('copy', () => {
+                    if (CopyButtonProps) {
+                        CopyButtonProps(preText, false, 'Keydown')
+                    }
+                })
             }
         }
     }, [displayText, CopyButtonProps, copyButtonClassName, insertButtonClassName, rootRef])

diff --git a/vscode/CHANGELOG.md b/vscode/CHANGELOG.md
@@ -20,6 +20,8 @@ Starting from `0.2.0`, Cody is using `major.EVEN_NUMBER.patch` for release versi
 - `Explain Code` command now includes visible content of the current file when no code is selected. [pull/602](https://github.com/sourcegraph/cody/pull/602)
 - Cody Commands: Show errors in chat view instead of notification windows. [pull/602](https://github.com/sourcegraph/cody/pull/602)
 - Include the number of accepted characters per autocomplete suggestion. [pull/674](https://github.com/sourcegraph/cody/pull/674)
+- Log all button click events. [pull/675](https://github.com/sourcegraph/cody/pull/675)
+- Log token count for code generated by Cody. [pull/675](https://github.com/sourcegraph/cody/pull/675)
 
 ## [0.6.6]
 

diff --git a/vscode/src/chat/ChatViewProvider.ts b/vscode/src/chat/ChatViewProvider.ts
@@ -5,6 +5,7 @@ import { ChatMessage, UserLocalHistory } from '@sourcegraph/cody-shared/src/chat
 
 import { View } from '../../webviews/NavBar'
 import { debug } from '../log'
+import { countCode, matchCodeSnippets } from '../services/InlineAssist'
 
 import { MessageProvider, MessageProviderOptions } from './MessageProvider'
 import { ExtensionMessage, WebviewMessage } from './protocol'
@@ -43,9 +44,11 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
             case 'edit':
                 this.transcript.removeLastInteraction()
                 await this.onHumanMessageSubmitted(message.text, 'user')
+                this.telemetryService.log('CodyVSCodeExtension:editChatButton:clicked')
                 break
             case 'abort':
                 await this.abortCompletion()
+                this.telemetryService.log('CodyVSCodeExtension:abortButton:clicked', { source: 'sidebar' })
                 break
             case 'executeRecipe':
                 await this.setWebviewView('chat')
@@ -67,7 +70,10 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
                 await vscode.commands.executeCommand(`cody.auth.${message.type}`)
                 break
             case 'insert':
-                await this.insertAtCursor(message.text)
+                await this.handleInsertAtCursor(message.text)
+                break
+            case 'copy':
+                await this.handleCopiedCode(message.text, message.eventType)
                 break
             case 'event':
                 this.telemetryService.log(message.eventName, message.properties)
@@ -89,6 +95,7 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
                 break
             case 'reload':
                 await this.authProvider.reloadAuthStatus()
+                this.telemetryService.log('CodyVSCodeExtension:authReloadButton:clicked')
                 break
             case 'openFile':
                 await this.openFilePath(message.filePath)
@@ -104,11 +111,11 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
 
     private async onHumanMessageSubmitted(text: string, submitType: 'user' | 'suggestion' | 'example'): Promise<void> {
         debug('ChatViewProvider:onHumanMessageSubmitted', '', { verbose: { text, submitType } })
+        this.telemetryService.log('CodyVSCodeExtension:chat:submitted', { source: 'sidebar' })
         if (submitType === 'suggestion') {
             this.telemetryService.log('CodyVSCodeExtension:chatPredictions:used')
         }
         if (text === '/') {
-            this.telemetryService.log('CodyVSCodeExtension:custom-command-menu:clicked')
             void vscode.commands.executeCommand('cody.action.commands.menu', true)
             return
         }
@@ -123,7 +130,7 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
      * Process custom command click
      */
     private async onCustomPromptClicked(title: string, commandType: CodyPromptType = 'user'): Promise<void> {
-        this.telemetryService.log('CodyVSCodeExtension:custom-command:clicked')
+        this.telemetryService.log('CodyVSCodeExtension:command:customMenu:clicked')
         debug('ChatViewProvider:onCustomPromptClicked', title)
         if (!this.isCustomCommandAction(title)) {
             await this.setWebviewView('chat')
@@ -175,20 +182,67 @@ export class ChatViewProvider extends MessageProvider implements vscode.WebviewV
     }
 
     /**
-     * Insert text at cursor position
-     * Replace selection if there is one
+     * Prevent logging insert events as paste events on doc change
+     */
+    private isInsertEvent = false
+
+    /**
+     * Handles insert event to insert text from code block at cursor position
+     * Replace selection if there is one and then log insert event
      * Note: Using workspaceEdit instead of 'editor.action.insertSnippet' as the later reformats the text incorrectly
      */
-    private async insertAtCursor(text: string): Promise<void> {
+    private async handleInsertAtCursor(text: string): Promise<void> {
+        this.isInsertEvent = true
         const selectionRange = vscode.window.activeTextEditor?.selection
         const editor = vscode.window.activeTextEditor
         if (!editor || !selectionRange) {
             return
         }
+
         const edit = new vscode.WorkspaceEdit()
         // trimEnd() to remove new line added by Cody
         edit.replace(editor.document.uri, selectionRange, text.trimEnd())
         await vscode.workspace.applyEdit(edit)
+
+        // Log insert event
+        const op = 'insert'
+        const { lineCount, charCount } = countCode(text)
+        const eventName = op + 'Button'
+        const args = { op, charCount, lineCount }
+        this.telemetryService.log(`CodyVSCodeExtension:${eventName}:clicked`, args)
+        this.isInsertEvent = false
+    }
+
+    /**
+     * Handles copying code and detecting a paste event.
+     *
+     * @param text - The text from code block when copy event is triggered
+     * @param eventType - Either 'Button' or 'Keydown'
+     */
+    private async handleCopiedCode(text: string, eventType: 'Button' | 'Keydown'): Promise<void> {
+        // If it's a Button event, then the text is already passed in from the whole code block
+        const copiedCode = eventType === 'Button' ? text : await vscode.env.clipboard.readText()
+
+        // Log Copy event
+        const op = 'copy'
+        const { lineCount, charCount } = countCode(copiedCode)
+        const eventName = op + eventType
+        const args = { op, charCount, lineCount }
+        this.telemetryService.log(`CodyVSCodeExtension:${eventName}:clicked`, args)
+
+        // Create listener for changes to the active text editor for paste event
+        vscode.workspace.onDidChangeTextDocument(e => {
+            const changedText = e.contentChanges[0]?.text
+            // check if the copied code is the same as the changed text without spaces
+            const isMatched = matchCodeSnippets(copiedCode, changedText)
+            // Log paste event when the copied code is pasted
+            if (!this.isInsertEvent && isMatched) {
+                this.telemetryService.log('CodyVSCodeExtension:pasteKeydown:clicked', {
+                    ...args,
+                    op: 'paste',
+                })
+            }
+        })
     }
 
     protected handleEnabledPlugins(plugins: string[]): void {

diff --git a/vscode/src/chat/MessageProvider.ts b/vscode/src/chat/MessageProvider.ts
@@ -28,6 +28,7 @@ import { LocalStorage } from '../services/LocalStorageProvider'
 import { TestSupport } from '../test-support'
 
 import { ContextProvider } from './ContextProvider'
+import { countGeneratedCode } from './utils'
 
 /**
  * The problem with a token limit for the prompt is that we can only
@@ -138,12 +139,14 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
         this.handleSuggestions([])
         this.sendTranscript()
         this.sendHistory()
+        this.telemetryService.log('CodyVSCodeExtension:chatReset:executed')
     }
 
     public async clearHistory(): Promise<void> {
         MessageProvider.chatHistory = {}
         MessageProvider.inputHistory = []
         await this.localStorage.removeChatHistory()
+        this.telemetryService.log('CodyVSCodeExtension:clearChatHistoryButton:clicked')
     }
 
     /**
@@ -157,6 +160,7 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
         await this.transcript.toJSON()
         this.sendTranscript()
         this.sendHistory()
+        this.telemetryService.log('CodyVSCodeExtension:restoreChatHistoryButton:clicked')
     }
 
     private sendEnabledPlugins(plugins: string[]): void {
@@ -194,6 +198,10 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
                     this.transcript.addAssistantResponse(text || '', displayText)
                 }
                 await this.onCompletionEnd()
+                // Count code generated from response
+                const codeCount = countGeneratedCode(text)
+                const op = codeCount ? 'hasCode' : 'noCode'
+                this.telemetryService.log('CodyVSCodeExtension:chatResponse:' + op, codeCount || {})
             },
         })
 
@@ -511,14 +519,13 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
             case 'menu':
                 await this.editor.controllers.command?.menu('custom')
                 await this.sendCodyCommands()
-                this.telemetryService.log('CodyVSCodeExtension:command:menu:opened')
                 break
             case 'add':
                 if (!type) {
                     break
                 }
                 await this.editor.controllers.command?.config('add', type)
-                this.telemetryService.log('CodyVSCodeExtension:command:addCommand')
+                this.telemetryService.log('CodyVSCodeExtension:addCommandButton:clicked')
                 break
         }
         // Get prompt details from controller by title then execute prompt's command
@@ -529,7 +536,7 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
             return
         }
         await this.executeRecipe('custom-prompt', promptText)
-        this.telemetryService.log('CodyVSCodeExtension:command:executedFromMenu')
+        this.telemetryService.log('CodyVSCodeExtension:command:started', { source: 'menu' })
         const starter = (await this.editor.controllers.command?.getCustomConfig())?.starter
         if (starter) {
             this.telemetryService.log('CodyVSCodeExtension:command:customStarter:applied')
@@ -546,25 +553,29 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
             return { text, recipeId }
         }
         const commandKey = text.split(' ')[0].replace('/', '')
-        this.telemetryService.log(`CodyVSCodeExtension:command:${commandKey}:filtering`)
         switch (true) {
             case text === '/':
-                return vscode.commands.executeCommand('cody.action.commands.menu')
+                return vscode.commands.executeCommand('cody.action.commands.menu', 'sidebar')
             case text === '/commands-settings':
+                this.telemetryService.log('CodyVSCodeExtension:command:configMenuButton:clicked', { source: 'sidebar' })
                 return vscode.commands.executeCommand('cody.settings.commands')
             case /^\/o(pen)?\s/.test(text) && this.editor.controllers.command !== undefined:
                 // open the user's ~/.vscode/cody.json file
                 await this.editor.controllers.command?.open(text.split(' ')[1])
+                this.telemetryService.log('CodyVSCodeExtension:command:openFile:executed')
                 return null
             case /^\/r(eset)?$/.test(text):
                 await this.clearAndRestartSession()
+                this.telemetryService.log('CodyVSCodeExtension:command:resetChat:executed')
                 return null
             case /^\/s(earch)?\s/.test(text):
                 return { text, recipeId: 'context-search' }
             case /^\/f(ix)?\s.*$/.test(text):
                 return { text, recipeId: 'fixup' }
             case /^\/(explain|doc|test|smell)$/.test(text):
-                this.telemetryService.log(`CodyVSCodeExtension:command:${text.replace('/', '')}:executed`)
+                this.telemetryService.log(`CodyVSCodeExtension:command:${commandKey}:called`, {
+                    source: 'chat',
+                })
             default: {
                 if (!this.editor.getActiveTextEditor()?.filePath) {
                     await this.addCustomInteraction('Command failed. Please open a file and try again.', text)
@@ -573,7 +584,6 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
                 const promptText = this.editor.controllers.command?.find(text, true)
                 await this.editor.controllers.command?.get('command')
                 if (promptText) {
-                    this.telemetryService.log(`CodyVSCodeExtension:command:${commandKey}:executing`)
                     return { text: promptText, recipeId: 'custom-prompt' }
                 }
                 // Inline chat has its own filter for slash commands
@@ -647,6 +657,7 @@ export abstract class MessageProvider extends MessageHandler implements vscode.D
         delete MessageProvider.chatHistory[chatID]
         await this.localStorage.deleteChatHistory(chatID)
         this.sendHistory()
+        this.telemetryService.log('CodyVSCodeExtension:deleteChatHistoryButton:clicked')
     }
 
     /**

diff --git a/vscode/src/chat/protocol.ts b/vscode/src/chat/protocol.ts
@@ -23,7 +23,8 @@ export type WebviewMessage =
     | { command: 'links'; value: string }
     | { command: 'openFile'; filePath: string }
     | { command: 'edit'; text: string }
-    | { command: 'insert'; text: string }
+    | { command: 'insert'; eventType: 'Button' | 'Keydown'; text: string }
+    | { command: 'copy'; eventType: 'Button' | 'Keydown'; text: string }
     | {
           command: 'auth'
           type: 'signin' | 'signout' | 'support' | 'app' | 'callback'

diff --git a/vscode/src/chat/utils.ts b/vscode/src/chat/utils.ts
@@ -82,3 +82,31 @@ export function newAuthStatus(
     authStatus.isLoggedIn = isLoggedIn && isAllowed
     return authStatus
 }
+
+/**
+ * Counts the number of lines and characters in code blocks in a given string.
+ *
+ * @param text - The string to search for code blocks.
+ * @returns An object with the total lineCount and charCount of code in code blocks,
+ * or null if no code blocks are found.
+ */
+export const countGeneratedCode = (text: string): { lineCount: number; charCount: number } | null => {
+    const codeBlockRegex = /```[\S\s]*?```/g
+    const codeBlocks = text.match(codeBlockRegex)
+    if (!codeBlocks) {
+        return null
+    }
+    const count = { lineCount: 0, charCount: 0 }
+    const backticks = '```'
+    for (const block of codeBlocks) {
+        const lines = block.split('\n')
+        const codeLines = lines.filter(line => !line.startsWith(backticks))
+        const lineCount = codeLines.length
+        const language = lines[0].replace(backticks, '')
+        // 2 backticks + 2 newline
+        const charCount = block.length - language.length - backticks.length * 2 - 2
+        count.charCount += charCount
+        count.lineCount += lineCount
+    }
+    return count
+}
diff --git a/vscode/src/completions/logger.ts b/vscode/src/completions/logger.ts
@@ -114,7 +114,7 @@ export function suggested(id: string, source: string): void {
     }
 }
 
-export function accept(id: string, lines: number, chars: number): void {
+export function accept(id: string, lineCount: number, charCount: number): void {
     const completionEvent = displayedCompletions.get(id)
     if (!completionEvent || completionEvent.acceptedAt) {
         // Log a debug event, this case should not happen in production
@@ -139,8 +139,8 @@ export function accept(id: string, lines: number, chars: number): void {
     logSuggestionEvents()
     logCompletionEvent('accepted', {
         ...completionEvent.params,
-        lines,
-        chars,
+        lineCount,
+        charCount,
         otherCompletionProviderEnabled: otherCompletionProviderEnabled(),
     })
 }

diff --git a/vscode/src/custom-prompts/CommandsController.ts b/vscode/src/custom-prompts/CommandsController.ts
@@ -118,10 +118,8 @@ export class CommandsController implements VsCodeCommandsController, vscode.Disp
             this.lastUsedCommands.add(id)
         }
 
-        if (myPrompt?.type === 'default') {
-            this.telemetryService.log(`CodyVSCodeExtension:command:${myPrompt.slashCommand}:executed`)
-        }
-
+        const commandName = myPrompt?.type === 'default' ? myPrompt.slashCommand?.replace('/', '') : 'custom'
+        this.telemetryService.log(`CodyVSCodeExtension:command:${commandName}:called`)
         return myPrompt?.prompt || ''
     }
 
@@ -166,8 +164,8 @@ export class CommandsController implements VsCodeCommandsController, vscode.Disp
     /**
      * Menu Controller
      */
-    public async menu(type: 'custom' | 'config' | 'default', showDesc?: boolean): Promise<void> {
-        this.telemetryService.log(`CodyVSCodeExtension:command:menu:${type}`)
+    public async menu(type: 'custom' | 'config' | 'default', showDesc = true): Promise<void> {
+        this.telemetryService.log('CodyVSCodeExtension:command:menu:opened', { type })
         await this.refresh()
         switch (type) {
             case 'custom':
@@ -198,7 +196,7 @@ export class CommandsController implements VsCodeCommandsController, vscode.Disp
     /**
      * Main Menu: Cody Commands
      */
-    public async mainCommandMenu(showDesc = false): Promise<void> {
+    public async mainCommandMenu(showDesc = true): Promise<void> {
         try {
             const commandItems = [menu_separators.inline, menu_options.chat, menu_options.fix, menu_separators.commands]
             const allCommands = this.default.getGroupedCommands(true)

diff --git a/vscode/src/editor/EditorCodeLenses.ts b/vscode/src/editor/EditorCodeLenses.ts
@@ -70,7 +70,7 @@ export class EditorCodeLenses implements vscode.CodeLensProvider {
         if (activeEditor) {
             activeEditor.selection = lens.selection
         }
-        await vscode.commands.executeCommand(lens.name, false)
+        await vscode.commands.executeCommand(lens.name, 'codeLens')
     }
     /**
      * Gets the code lenses for the specified document.