Autocomplete: Split debounce into two chunks, race the second part with the context retrieving

vscode/CHANGELOG.md

@@ -14,6 +14,7 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a
 
 - Autocomplete: Removes the latency for cached completions. [https://github.com/sourcegraph/cody/pull/3138](https://github.com/sourcegraph/cody/pull/3138)
 - Autocomplete: Enable the recent jaccard similarity improvements by default. [pull/3135](https://github.com/sourcegraph/cody/pull/3135)
+- Autocomplete: Start retrieval phase earlier to improve latency. [pull/3149](https://github.com/sourcegraph/cody/pull/3149)
 
 ## [1.4.3]
 

vscode/src/completions/get-inline-completions-tests/helpers.ts

@@ -32,10 +32,11 @@ import {
 } from '../providers/anthropic'
 import type { ProviderOptions } from '../providers/provider'
 import { RequestManager } from '../request-manager'
-import { documentAndPosition, sleep } from '../test-helpers'
+import { documentAndPosition } from '../test-helpers'
 import { pressEnterAndGetIndentString } from '../providers/hot-streak'
 import { completionProviderConfig } from '../completion-provider-config'
 import { emptyMockFeatureFlagProvider } from '../../testutils/mocks'
+import { sleep } from '../utils'
 
 // The dedent package seems to replace `\t` with `\\t` so in order to insert a tab character, we
 // have to use interpolation. We abbreviate this to `T` because ${T} is exactly 4 characters,

vscode/src/completions/get-inline-completions.ts

@@ -28,6 +28,7 @@ import type { InlineCompletionItemWithAnalytics } from './text-processing/proces
 import type { ProvideInlineCompletionsItemTraceData } from './tracer'
 import { isValidTestFile } from '../commands/utils/test-commands'
 import { completionProviderConfig } from './completion-provider-config'
+import { sleep } from './utils'
 
 export interface InlineCompletionsParams {
     // Context
@@ -306,6 +307,21 @@ async function doGetInlineCompletions(
         }
     }
 
+    const debounceTime =
+        triggerKind !== TriggerKind.Automatic
+            ? 0
+            : ((multiline ? debounceInterval?.multiLine : debounceInterval?.singleLine) ?? 0) +
+              (artificialDelay ?? 0)
+
+    // We split the desired debounceTime into two chunks. One that is at most 25ms where every
+    // further execution is halted...
+    const waitInterval = Math.min(debounceTime, 25)
+    // ...and one for the remaining time where we can already start retrieving context in parallel.
+    const remainingInterval = debounceTime - waitInterval
+    if (waitInterval > 0) {
+        await wrapInActiveSpan('autocomplete.debounce.wait', () => sleep(waitInterval))
+    }
+
     // Debounce to avoid firing off too many network requests as the user is still typing.
     await wrapInActiveSpan('autocomplete.debounce', async () => {
         const interval =
@@ -324,19 +340,26 @@ async function doGetInlineCompletions(
     setIsLoading?.(true)
     CompletionLogger.start(logId)
 
-    // Fetch context
-    const contextResult = await wrapInActiveSpan('autocomplete.retrieve', async () => {
-        return contextMixer.getContext({
-            document,
-            position,
-            docContext,
-            abortSignal,
-            maxChars: providerConfig.contextSizeHints.totalChars,
-        })
-    })
+    // Fetch context and apply remaining debounce time
+    const [contextResult] = await Promise.all([
+        wrapInActiveSpan('autocomplete.retrieve', () =>
+            contextMixer.getContext({
+                document,
+                position,
+                docContext,
+                abortSignal,
+                maxChars: providerConfig.contextSizeHints.totalChars,
+            })
+        ),
+        remainingInterval > 0
+            ? wrapInActiveSpan('autocomplete.debounce.remaining', () => sleep(remainingInterval))
+            : null,
+    ])
+
     if (abortSignal?.aborted) {
         return null
     }
+
     tracer?.({ context: contextResult })
 
     const completionProvider = getCompletionProvider({

vscode/src/completions/test-helpers.ts

@@ -60,7 +60,3 @@ export function documentAndPosition(
 export function nextTick(): Promise<void> {
     return new Promise(resolve => setTimeout(resolve, 0))
 }
-
-export function sleep(ms: number): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, ms))
-}

vscode/src/completions/utils.ts

@@ -131,3 +131,7 @@ function createTimeout(timeoutMs: number): Promise<never> {
         setTimeout(() => reject(new TimeoutError('The request timed out')), timeoutMs)
     )
 }
+
+export function sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms))
+}