Merge pull request #4478 from leigaol/percent_copy

feat(cwspr): Improve percentage code written metrics to include some user multi character input

.changes/next-release/Feature-742cb7ff-2139-4c0f-9b9e-22d72c960f60.json

@@ -0,0 +1,4 @@
+{
+	"type": "Feature",
+	"description": "CodeWhisperer: Improve percentage code written metrics to include user multi character input"
+}

packages/toolkit/src/codewhisperer/service/inlineCompletionItemProvider.ts

@@ -104,6 +104,7 @@ export class CWInlineCompletionItemProvider implements vscode.InlineCompletionIt
             }
             return undefined
         }
+        TelemetryHelper.instance.lastSuggestionInDisplay = truncatedSuggestion
         return {
             insertText: truncatedSuggestion,
             range: new vscode.Range(start, end),

packages/toolkit/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts

@@ -24,6 +24,23 @@ interface CodeWhispererToken {
     accepted: number
 }
 
+interface UserInputDetail {
+    count: number
+    total: number
+}
+
+interface UserInputDetails {
+    lt1: UserInputDetail
+    lt50: UserInputDetail
+    lt100: UserInputDetail
+    lt200: UserInputDetail
+    lt300: UserInputDetail
+    lt400: UserInputDetail
+    lt500: UserInputDetail
+    lt1000: UserInputDetail
+    gt1000: UserInputDetail
+}
+
 const autoClosingKeystrokeInputs = ['[]', '{}', '()', '""', "''"]
 
 /**
@@ -37,12 +54,25 @@ export class CodeWhispererCodeCoverageTracker {
     private _language: CodewhispererLanguage
     private _serviceInvocationCount: number
 
+    private _userInputDetails: UserInputDetails
+
     private constructor(language: CodewhispererLanguage) {
         this._acceptedTokens = {}
         this._totalTokens = {}
         this._startTime = 0
         this._language = language
         this._serviceInvocationCount = 0
+        this._userInputDetails = {
+            lt1: { count: 0, total: 0 },
+            lt50: { count: 0, total: 0 },
+            lt100: { count: 0, total: 0 },
+            lt200: { count: 0, total: 0 },
+            lt300: { count: 0, total: 0 },
+            lt400: { count: 0, total: 0 },
+            lt500: { count: 0, total: 0 },
+            lt1000: { count: 0, total: 0 },
+            gt1000: { count: 0, total: 0 },
+        }
     }
 
     public get serviceInvocationCount(): number {
@@ -110,7 +140,7 @@ export class CodeWhispererCodeCoverageTracker {
         }
         // the accepted characters without counting user modification
         let acceptedTokens = 0
-        // the accepted characters after calculating user modificaiton
+        // the accepted characters after calculating user modification
         let unmodifiedAcceptedTokens = 0
         for (const filename in this._acceptedTokens) {
             this._acceptedTokens[filename].forEach(v => {
@@ -136,6 +166,7 @@ export class CodeWhispererCodeCoverageTracker {
             codewhispererUserGroup: CodeWhispererUserGroupSettings.getUserGroup().toString(),
             codewhispererCustomizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn,
             credentialStartUrl: AuthUtil.instance.startUrl,
+            codewhispererUserInputDetails: JSON.stringify(this._userInputDetails),
         })
 
         client
@@ -206,6 +237,17 @@ export class CodeWhispererCodeCoverageTracker {
         this._acceptedTokens = {}
         this._startTime = 0
         this._serviceInvocationCount = 0
+        this._userInputDetails = {
+            lt1: { count: 0, total: 0 },
+            lt50: { count: 0, total: 0 },
+            lt100: { count: 0, total: 0 },
+            lt200: { count: 0, total: 0 },
+            lt300: { count: 0, total: 0 },
+            lt400: { count: 0, total: 0 },
+            lt500: { count: 0, total: 0 },
+            lt1000: { count: 0, total: 0 },
+            gt1000: { count: 0, total: 0 },
+        }
     }
 
     private closeTimer() {
@@ -287,9 +329,54 @@ export class CodeWhispererCodeCoverageTracker {
         if (this.isFromUserKeystroke(e)) {
             this.tryStartTimer()
             this.addTotalTokens(e.document.fileName, 1)
+            this._userInputDetails.lt1.count += 1
+            this._userInputDetails.lt1.total += 1
         } else if (this.getCharacterCountFromComplexEvent(e) !== 0) {
             this.tryStartTimer()
-            this.addTotalTokens(e.document.fileName, this.getCharacterCountFromComplexEvent(e))
+            const characterIncrease = this.getCharacterCountFromComplexEvent(e)
+            this.addTotalTokens(e.document.fileName, characterIncrease)
+            this._userInputDetails.lt1.count += 1
+            this._userInputDetails.lt1.total += characterIncrease
+        }
+        // also include multi character input within 500 characters (not from CWSPR)
+        else if (
+            e.contentChanges.length === 1 &&
+            e.contentChanges[0].text.length > 1 &&
+            TelemetryHelper.instance.lastSuggestionInDisplay !== e.contentChanges[0].text
+        ) {
+            const multiCharInputSize = e.contentChanges[0].text.length
+
+            // select 500 as the cut-off threshold for counting user input.
+            if (multiCharInputSize < 500) {
+                this.addTotalTokens(e.document.fileName, multiCharInputSize)
+            }
+
+            // report multiple user input patterns for adjusting the threshold
+            if (multiCharInputSize < 50) {
+                this._userInputDetails.lt50.total += multiCharInputSize
+                this._userInputDetails.lt50.count += 1
+            } else if (multiCharInputSize < 100) {
+                this._userInputDetails.lt100.total += multiCharInputSize
+                this._userInputDetails.lt100.count += 1
+            } else if (multiCharInputSize < 200) {
+                this._userInputDetails.lt200.total += multiCharInputSize
+                this._userInputDetails.lt200.count += 1
+            } else if (multiCharInputSize < 300) {
+                this._userInputDetails.lt300.total += multiCharInputSize
+                this._userInputDetails.lt300.count += 1
+            } else if (multiCharInputSize < 400) {
+                this._userInputDetails.lt400.total += multiCharInputSize
+                this._userInputDetails.lt400.count += 1
+            } else if (multiCharInputSize < 500) {
+                this._userInputDetails.lt500.total += multiCharInputSize
+                this._userInputDetails.lt500.count += 1
+            } else if (multiCharInputSize < 1000) {
+                this._userInputDetails.lt1000.total += multiCharInputSize
+                this._userInputDetails.lt1000.count += 1
+            } else {
+                this._userInputDetails.gt1000.total += multiCharInputSize
+                this._userInputDetails.gt1000.count += 1
+            }
         }
     }
 

packages/toolkit/src/codewhisperer/util/telemetryHelper.ts

@@ -48,6 +48,9 @@ export class TelemetryHelper {
     private classifierResult?: number = undefined
     private classifierThreshold?: number = undefined
 
+    // use this to distinguish DocumentChangeEvent from CWSPR or from other sources
+    public lastSuggestionInDisplay = ''
+
     constructor() {}
 
     static #instance: TelemetryHelper

packages/toolkit/src/shared/telemetry/vscodeTelemetry.json

@@ -271,6 +271,11 @@
             "name": "cwsprChatCommandName",
             "type": "string",
             "description": "Type of chat command name executed"
+        },
+        {
+            "name": "codewhispererUserInputDetails",
+            "type": "string",
+            "description": "A JSON serialized string of user input details."
         }
     ],
     "metrics": [
@@ -989,6 +994,47 @@
                     "required": false
                 }
             ]
+        },
+        {
+            "name": "codewhisperer_codePercentage",
+            "description": "Percentage of user tokens against suggestions until 5 mins of time",
+            "metadata": [
+                {
+                    "type": "codewhispererAcceptedTokens"
+                },
+                {
+                    "type": "codewhispererLanguage"
+                },
+                {
+                    "type": "codewhispererPercentage"
+                },
+                {
+                    "type": "codewhispererTotalTokens"
+                },
+                {
+                    "type": "codewhispererUserInputDetails",
+                    "required": false
+                },
+                {
+                    "type": "codewhispererUserGroup"
+                },
+                {
+                    "type": "reason",
+                    "required": false
+                },
+                {
+                    "type": "successCount"
+                },
+                {
+                    "type": "codewhispererCustomizationArn",
+                    "required": false
+                },
+                {
+                    "type": "credentialStartUrl",
+                    "required": false
+                }
+            ],
+            "passive": true
         }
     ]
 }

packages/toolkit/src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts

@@ -232,7 +232,7 @@ describe('codewhispererCodecoverageTracker', function () {
             CodeWhispererCodeCoverageTracker.instances.clear()
         })
 
-        it('Should skip when content change size is not 1', function () {
+        it('Should skip when content change size is more than 500', function () {
             if (!tracker) {
                 assert.fail()
             }
@@ -241,16 +241,34 @@ describe('codewhispererCodecoverageTracker', function () {
                 document: createMockDocument(),
                 contentChanges: [
                     {
-                        range: new vscode.Range(0, 0, 0, 30),
+                        range: new vscode.Range(0, 0, 0, 600),
                         rangeOffset: 0,
-                        rangeLength: 30,
-                        text: 'def twoSum(nums, target):\nfor',
+                        rangeLength: 600,
+                        text: 'def twoSum(nums, target):\nfor '.repeat(20),
                     },
                 ],
             })
+            assert.strictEqual(Object.keys(tracker.totalTokens).length, 0)
+        })
 
-            const startedSpy = sinon.spy(CodeWhispererCodeCoverageTracker.prototype, 'addTotalTokens')
-            assert.ok(!startedSpy.called)
+        it('Should not skip when content change size is less than 500', function () {
+            if (!tracker) {
+                assert.fail()
+            }
+            tracker.countTotalTokens({
+                reason: undefined,
+                document: createMockDocument(),
+                contentChanges: [
+                    {
+                        range: new vscode.Range(0, 0, 0, 300),
+                        rangeOffset: 0,
+                        rangeLength: 300,
+                        text: 'def twoSum(nums, target): for '.repeat(10),
+                    },
+                ],
+            })
+            assert.strictEqual(Object.keys(tracker.totalTokens).length, 1)
+            assert.strictEqual(Object.values(tracker.totalTokens)[0], 300)
         })
 
         it('Should skip when CodeWhisperer is editing', function () {