Increase GPT4o context size budget (#4180)

Co-authored-by: Dominic Cooney <dominic.cooney@sourcegraph.com>

lib/shared/src/models/dotcom.ts

@@ -2,8 +2,8 @@ import type { ModelProvider } from '.'
 import {
     CHAT_INPUT_TOKEN_BUDGET,
     CHAT_OUTPUT_TOKEN_BUDGET,
-    CLAUDE3_CHAT_INPUT_TOKEN_BUDGET,
-    CLAUDE3_USER_CONTEXT_TOKEN_BUDGET,
+    EXTENDED_CHAT_INPUT_TOKEN_BUDGET,
+    EXTENDED_USER_CONTEXT_TOKEN_BUDGET,
 } from '../token/constants'
 
 import { ModelUsage } from './types'
@@ -20,11 +20,11 @@ const DEFAULT_DOT_COM_MODELS: ModelProvider[] = [
         initialDefault: true,
         codyProOnly: false,
         usage: [ModelUsage.Chat, ModelUsage.Edit],
-        // Has a higher context window with a seperated limit for user-context.
+        // Has a higher context window with a separate limit for user-context.
         contextWindow: {
-            input: CLAUDE3_CHAT_INPUT_TOKEN_BUDGET,
+            input: EXTENDED_CHAT_INPUT_TOKEN_BUDGET,
             output: CHAT_OUTPUT_TOKEN_BUDGET,
-            context: { user: CLAUDE3_USER_CONTEXT_TOKEN_BUDGET },
+            context: { user: EXTENDED_USER_CONTEXT_TOKEN_BUDGET },
         },
         deprecated: false,
         uiGroup: ModelUIGroup.Balanced,
@@ -36,11 +36,11 @@ const DEFAULT_DOT_COM_MODELS: ModelProvider[] = [
         default: false,
         codyProOnly: true,
         usage: [ModelUsage.Chat, ModelUsage.Edit],
-        // Has a higher context window with a seperated limit for user-context.
+        // Has a higher context window with a separate limit for user-context.
         contextWindow: {
-            input: CLAUDE3_CHAT_INPUT_TOKEN_BUDGET,
+            input: EXTENDED_CHAT_INPUT_TOKEN_BUDGET,
             output: CHAT_OUTPUT_TOKEN_BUDGET,
-            context: { user: CLAUDE3_USER_CONTEXT_TOKEN_BUDGET },
+            context: { user: EXTENDED_USER_CONTEXT_TOKEN_BUDGET },
         },
         deprecated: false,
         uiGroup: ModelUIGroup.Accuracy,
@@ -63,7 +63,12 @@ const DEFAULT_DOT_COM_MODELS: ModelProvider[] = [
         default: false,
         codyProOnly: true,
         usage: [ModelUsage.Chat, ModelUsage.Edit],
-        contextWindow: { input: CHAT_INPUT_TOKEN_BUDGET, output: CHAT_OUTPUT_TOKEN_BUDGET },
+        // Has a higher context window with a separate limit for user-context.
+        contextWindow: {
+            input: EXTENDED_CHAT_INPUT_TOKEN_BUDGET,
+            output: CHAT_OUTPUT_TOKEN_BUDGET,
+            context: { user: EXTENDED_USER_CONTEXT_TOKEN_BUDGET },
+        },
         deprecated: false,
         uiGroup: ModelUIGroup.Accuracy,
     },

lib/shared/src/token/constants.ts

@@ -20,15 +20,15 @@ export const CHAT_OUTPUT_TOKEN_BUDGET = 4000
 export const ENHANCED_CONTEXT_ALLOCATION = 0.6
 
 /**
- * NOTE: Reserved for Claude-3 Sonnet and Opus only.
+ * NOTE: Reserved for models with large context windows and good recall.
  *
  * The total context window reserved for user added context (@-mention, right-click, etc.)
  */
-export const CLAUDE3_USER_CONTEXT_TOKEN_BUDGET = 30000
+export const EXTENDED_USER_CONTEXT_TOKEN_BUDGET = 30000
 
 /**
- * NOTE: Reserved for Claude-3 Sonnet and Opus only.
+ * NOTE: Reserved for models with large context windows and good recall.
  *
  * The total context window reserved for chat input.
  */
-export const CLAUDE3_CHAT_INPUT_TOKEN_BUDGET = 15000
+export const EXTENDED_CHAT_INPUT_TOKEN_BUDGET = 15000

lib/shared/src/token/counter.test.ts

@@ -3,8 +3,8 @@ import { ps } from '../prompt/prompt-string'
 import type { Message } from '../sourcegraph-api'
 import {
     CHAT_INPUT_TOKEN_BUDGET,
-    CLAUDE3_USER_CONTEXT_TOKEN_BUDGET,
     ENHANCED_CONTEXT_ALLOCATION,
+    EXTENDED_USER_CONTEXT_TOKEN_BUDGET,
 } from './constants'
 import { TokenCounter } from './counter'
 
@@ -35,10 +35,10 @@ describe('TokenCounter class', () => {
         const counter = new TokenCounter({
             input: CHAT_INPUT_TOKEN_BUDGET,
             output: 0,
-            context: { user: CLAUDE3_USER_CONTEXT_TOKEN_BUDGET },
+            context: { user: EXTENDED_USER_CONTEXT_TOKEN_BUDGET },
         })
         expect(counter.maxChatTokens).toBe(CHAT_INPUT_TOKEN_BUDGET)
-        expect(counter.maxContextTokens.user).toBe(CLAUDE3_USER_CONTEXT_TOKEN_BUDGET)
+        expect(counter.maxContextTokens.user).toBe(EXTENDED_USER_CONTEXT_TOKEN_BUDGET)
         expect(counter.maxContextTokens.enhanced).toBe(
             CHAT_INPUT_TOKEN_BUDGET * ENHANCED_CONTEXT_ALLOCATION
         )
@@ -68,7 +68,7 @@ describe('TokenCounter class', () => {
         const counter = new TokenCounter({
             input: CHAT_INPUT_TOKEN_BUDGET,
             output: 0,
-            context: { user: CLAUDE3_USER_CONTEXT_TOKEN_BUDGET },
+            context: { user: EXTENDED_USER_CONTEXT_TOKEN_BUDGET },
         })
         expect(counter.updateUsage('preamble', preamble)).toBe(true) // 3 chat tokens used
         expect(

vscode/CHANGELOG.md

@@ -15,6 +15,8 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a
 - Chat: Pressing <kbd>Space</kbd> no longer accepts an @-mention item. Press <kbd>Tab</kbd> or <kbd>Enter</kbd> instead. [pull/4154](https://github.com/sourcegraph/cody/pull/4154)
 - Chat: You can now change the model after you send a chat message. Subsequent messages will be sent using your selected model. [pull/4189](https://github.com/sourcegraph/cody/pull/4189)
 - Chat: The `@`-mention menu now shows the types of context you can include. [pull/4188](https://github.com/sourcegraph/cody/pull/4188)
+- Increases the context window for the new `GPT-4o` model. [pull/4180](https://github.com/sourcegraph/cody/pull/4180)
+
 
 ## [1.18.0]
 

vscode/src/editor/utils/editor-context.test.ts

@@ -3,9 +3,9 @@ import * as vscode from 'vscode'
 import { URI } from 'vscode-uri'
 
 import {
-    CLAUDE3_USER_CONTEXT_TOKEN_BUDGET,
     type ContextItem,
     type ContextItemFile,
+    EXTENDED_USER_CONTEXT_TOKEN_BUDGET,
     type Editor,
     contextFiltersProvider,
     ignores,
@@ -163,7 +163,7 @@ describe('filterContextItemFiles', () => {
             uri: vscode.Uri.file('/large-text.txt'),
             type: 'file',
         }
-        const fsSizeInBytes = CLAUDE3_USER_CONTEXT_TOKEN_BUDGET * 4 + 100
+        const fsSizeInBytes = EXTENDED_USER_CONTEXT_TOKEN_BUDGET * 4 + 100
         vscode.workspace.fs.stat = vi.fn().mockResolvedValueOnce({
             size: fsSizeInBytes,
             type: vscode.FileType.File,