Autocomplete: Make string distance filters more lenient (#1320)

We have gotten reports for cases where Cody didn't suggest a useful
completion. One such case can be reproed with this example:

```
interface DoInteractionRequest {
    chatId: string
    prompt: string
    systemPrompt: string
}

interface Fuzz {
    string: () => string
}

const z: Fuzz = {} as any

const interactionRequest: DoInteractionRequest = {
    chatId█
    prompt: z.string(),
    systemPrompt: z.string(),
}
```

This was caused by the string distance between the inserted line
`chatId: z.string(),` and the next line `systemPrompt: z.string(),` (or
the one above, for that matter) to be too close. We use to guard heavily
against it since our initial Claude Instant implementation could not do
infilling well and was repeating the suffix in many cases.

Since we have infilling now, we can make this overlap much more lenient.
This PR changes the threshold from 0.33% to 0.2%.

## Test plan



https://github.com/sourcegraph/cody/assets/458591/e8c4cd05-e33d-4821-91ee-63b23af19a78



<!-- Required. See
https://docs.sourcegraph.com/dev/background-information/testing_principles.
-->

vscode/CHANGELOG.md

@@ -10,6 +10,8 @@ Starting from `0.2.0`, Cody is using `major.EVEN_NUMBER.patch` for release versi
 
 ### Fixed
 
+- Fixes an issue where autocomplete suggestions where sometimes not shown when the overlap with the next line was too large. [pull/1320](https://github.com/sourcegraph/cody/pull/1320
+
 ### Changed
 
 ## [0.14.0]

vscode/src/completions/text-processing/process-inline-completions.test.ts

@@ -1,5 +1,5 @@
 import dedent from 'dedent'
-import { beforeAll, describe, expect, test } from 'vitest'
+import { beforeAll, describe, expect, it } from 'vitest'
 import Parser from 'web-tree-sitter'
 
 import { range } from '../../testutils/textDocument'
@@ -11,7 +11,7 @@ import { InlineCompletionItem } from '../types'
 import { adjustRangeToOverwriteOverlappingCharacters, processInlineCompletions } from './process-inline-completions'
 
 describe('adjustRangeToOverwriteOverlappingCharacters', () => {
-    test('no adjustment at end of line', () => {
+    it('no adjustment at end of line', () => {
         const item: InlineCompletionItem = { insertText: 'array) {' }
         const { position } = documentAndPosition('function sort(█')
         expect(
@@ -22,7 +22,7 @@ describe('adjustRangeToOverwriteOverlappingCharacters', () => {
         ).toEqual<InlineCompletionItem>(item)
     })
 
-    test('handles non-empty currentLineSuffix', () => {
+    it('handles non-empty currentLineSuffix', () => {
         const item: InlineCompletionItem = { insertText: 'array) {' }
         const { position } = documentAndPosition('function sort(█)')
         expect(
@@ -36,7 +36,7 @@ describe('adjustRangeToOverwriteOverlappingCharacters', () => {
         })
     })
 
-    test('handles whitespace in currentLineSuffix', () => {
+    it('handles whitespace in currentLineSuffix', () => {
         const item: InlineCompletionItem = { insertText: 'array) {' }
         const { position } = documentAndPosition('function sort(█)')
         expect(
@@ -80,19 +80,19 @@ describe('process completion item', () => {
         )
     }
 
-    test('adds parse info to single-line completions', () => {
+    it('adds parse info to single-line completions', () => {
         const completions = processCompletions('function sort(█', ['array) {}', 'array) new'])
 
         expect(completions.map(c => Boolean(c.parseErrorCount))).toEqual([false, true])
     })
 
-    test('respects completion insert ranges', () => {
+    it('respects completion insert ranges', () => {
         const completions = processCompletions('function sort(█)', ['array) {}', 'array) new'])
 
         expect(completions.map(c => Boolean(c.parseErrorCount))).toEqual([false, true])
     })
 
-    test('adds parse info to multi-line completions', () => {
+    it('adds parse info to multi-line completions', () => {
         const completions = processCompletions(
             `
             function hello() {
@@ -152,7 +152,7 @@ describe('process completion item', () => {
         `)
     })
 
-    test('truncates multi-line if statements correctly', () => {
+    it('truncates multi-line if statements correctly', () => {
         const completions = processCompletions(
             `
             function whatever() {

vscode/src/completions/text-processing/string-comparator.test.ts

@@ -14,10 +14,11 @@ describe('isAlmostTheSameString', () => {
         ],
         [
             false,
-            "console.log('Hello world', getSumAandB(a, b))",
+            "console.log('Hello world', getSumAAndB(a, b))",
             "console.error('Error log', getDBConnection(context))",
         ],
-    ])('should return %b for strings %s and %s', (expected, stringA, stringB) => {
+        [false, '    chatId: z.string(),', '    prompt: z.string(),'],
+    ])('should return %s for strings %j and %j', (expected, stringA, stringB) => {
         expect(isAlmostTheSameString(stringA, stringB)).toBe(expected)
     })
 })

vscode/src/completions/text-processing/string-comparator.ts

@@ -9,7 +9,7 @@
  *
  * For more details see https://en.wikipedia.org/wiki/Levenshtein_distance
  */
-export const isAlmostTheSameString = (stringA: string, stringB: string, percentage: number = 0.33): boolean => {
+export const isAlmostTheSameString = (stringA: string, stringB: string, percentage: number = 0.2): boolean => {
     const maxLength = Math.max(stringA.length, stringB.length)
     const editOperations = LevenshteinCompare(stringA, stringB)