Add pruning fallback and character truncation for large context

extensions/cli/src/stream/streamChatResponse.ts

@@ -1,7 +1,11 @@
 import { ModelConfig } from "@continuedev/config-yaml";
 import { BaseLlmApi } from "@continuedev/openai-adapters";
 import type { ChatHistoryItem } from "core/index.js";
-import { convertFromUnifiedHistoryWithSystemMessage } from "core/util/messageConversion.js";
+import { compileChatMessages } from "core/llm/countTokens.js";
+import {
+  convertFromUnifiedHistoryWithSystemMessage,
+  convertToUnifiedHistory,
+} from "core/util/messageConversion.js";
 import * as dotenv from "dotenv";
 import type {
   ChatCompletionMessageParam,
@@ -150,18 +154,90 @@ export async function processStreamingResponse(
 
   // Validate context length before making the request
   const validation = validateContextLength(chatHistory, model);
-  if (!validation.isValid) {
-    throw new Error(`Context length validation failed: ${validation.error}`);
-  }
 
   // Get fresh system message and inject it
   const systemMessage = await services.systemMessage.getSystemMessage(
     services.toolPermissions.getState().currentMode,
   );
-  const openaiChatHistory = convertFromUnifiedHistoryWithSystemMessage(
-    chatHistory,
+
+  let openaiChatHistory: ChatCompletionMessageParam[];
+  let chatHistoryToUse = chatHistory;
+
+  // If validation fails, try to prune using compileChatMessages
+  if (!validation.isValid) {
+    logger.warn(
+      "Context length validation failed, attempting to prune messages",
+      {
+        error: validation.error,
+        historyLength: chatHistory.length,
+      },
+    );
+
+    try {
+      // Convert to ChatMessage format for pruning
+      const openaiMessages = convertFromUnifiedHistoryWithSystemMessage(
+        chatHistory,
+        systemMessage,
+      ) as ChatCompletionMessageParam[];
+
+      // Use compileChatMessages to prune
+      const contextLength = model.contextLength || 4096;
+      const maxTokens = model.defaultCompletionOptions?.maxTokens || 1024;
+
+      const result = compileChatMessages({
+        modelName: model.model,
+        msgs: openaiMessages.map((msg) => ({
+          role: msg.role,
+          content: msg.content || "",
+          ...("tool_calls" in msg && msg.tool_calls
+            ? { toolCalls: msg.tool_calls }
+            : {}),
+          ...("tool_call_id" in msg && msg.tool_call_id
+            ? { toolCallId: msg.tool_call_id }
+            : {}),
+        })),
+        knownContextLength: contextLength,
+        maxTokens,
+        supportsImages: false,
+        tools,
+      });
+
+      if (result.didPrune) {
+        logger.info("Successfully pruned chat history to fit context length", {
+          originalLength: chatHistory.length,
+          prunedLength: result.compiledChatMessages.length,
+          contextPercentage: `${(result.contextPercentage * 100).toFixed(1)}%`,
+        });
+
+        // Convert pruned messages back to ChatHistoryItem format
+        const prunedOpenaiMessages = result.compiledChatMessages.map(
+          (msg: any) => ({
+            role: msg.role,
+            content: msg.content,
+            ...(msg.toolCalls ? { tool_calls: msg.toolCalls } : {}),
+            ...(msg.toolCallId ? { tool_call_id: msg.toolCallId } : {}),
+          }),
+        ) as ChatCompletionMessageParam[];
+
+        // Remove system message from the pruned messages to avoid duplication
+        const messagesWithoutSystem = prunedOpenaiMessages.filter(
+          (msg) => msg.role !== "system",
+        );
+        chatHistoryToUse = convertToUnifiedHistory(messagesWithoutSystem);
+      }
+    } catch (pruneError: any) {
+      logger.error("Failed to prune chat history", { error: pruneError });
+      throw new Error(
+        `Context length validation failed and pruning failed: ${pruneError.message}`,
+      );
+    }
+  }
+
+  openaiChatHistory = convertFromUnifiedHistoryWithSystemMessage(
+    chatHistoryToUse,
     systemMessage,
   ) as ChatCompletionMessageParam[];
+
   const requestStartTime = Date.now();
 
   const streamFactory = async (retryAbortSignal: AbortSignal) => {

extensions/cli/src/tools/runTerminalCommand.ts

@@ -13,6 +13,30 @@ import {
 
 import { Tool } from "./types.js";
 
+// Maximum number of lines and characters to return from command output
+const MAX_OUTPUT_LINES = 5000;
+const MAX_OUTPUT_CHARS = 200000;
+
+// Helper function to truncate command output by both lines and characters
+function truncateOutput(output: string): string {
+  const lines = output.split("\n");
+  let truncated = output;
+  let truncationMsg = "";
+
+  // First check character limit
+  if (output.length > MAX_OUTPUT_CHARS) {
+    truncated = output.substring(0, MAX_OUTPUT_CHARS);
+    truncationMsg = `\n\n[Output truncated to first ${MAX_OUTPUT_CHARS} characters of ${output.length} total]`;
+  }
+  // Then check line limit (only if not already truncated by characters)
+  else if (lines.length > MAX_OUTPUT_LINES) {
+    truncated = lines.slice(0, MAX_OUTPUT_LINES).join("\n");
+    truncationMsg = `\n\n[Output truncated to first ${MAX_OUTPUT_LINES} lines of ${lines.length} total]`;
+  }
+
+  return truncationMsg ? truncated + truncationMsg : truncated;
+}
+
 // Helper function to use login shell on Unix/macOS and PowerShell on Windows
 function getShellCommand(command: string): { shell: string; args: string[] } {
   if (process.platform === "win32") {
@@ -99,18 +123,9 @@ Commands are automatically executed from the current working directory (${proces
           let output = stdout + (stderr ? `\nStderr: ${stderr}` : "");
           output += `\n\n[Command timed out after ${TIMEOUT_MS / 1000} seconds of no output]`;
 
-          // Truncate output if it has too many lines
-          const lines = output.split("\n");
-          if (lines.length > 5000) {
-            const truncatedOutput = lines.slice(0, 5000).join("\n");
-            resolve(
-              truncatedOutput +
-                `\n\n[Output truncated to first 5000 lines of ${lines.length} total]`,
-            );
-            return;
-          }
-
-          resolve(output);
+          // Truncate output by both lines and characters
+          const truncatedOutput = truncateOutput(output);
+          resolve(truncatedOutput);
         }, TIMEOUT_MS);
       };
 
@@ -155,18 +170,9 @@ Commands are automatically executed from the current working directory (${proces
           output = stdout + `\nStderr: ${stderr}`;
         }
 
-        // Truncate output if it has too many lines
-        const lines = output.split("\n");
-        if (lines.length > 5000) {
-          const truncatedOutput = lines.slice(0, 5000).join("\n");
-          resolve(
-            truncatedOutput +
-              `\n\n[Output truncated to first 5000 lines of ${lines.length} total]`,
-          );
-          return;
-        }
-
-        resolve(output);
+        // Truncate output by both lines and characters
+        const truncatedOutput = truncateOutput(output);
+        resolve(truncatedOutput);
       });
 
       child.on("error", (error) => {