diff --git a/frontend/src/components/UnifiedChat.tsx b/frontend/src/components/UnifiedChat.tsx index 176db243..5e1c2040 100644 --- a/frontend/src/components/UnifiedChat.tsx +++ b/frontend/src/components/UnifiedChat.tsx @@ -4506,6 +4506,64 @@ export function UnifiedChat({ isVisible = true }: { isVisible?: boolean }) { ) { throw new Error("Response stream ended before completion"); } + + // Check if response completed but produced no useful output. + // This can happen when the model tries to use a tool that was available in + // previous conversation turns but is not in the current request's tools array. + if (terminalState === "completed") { + const finalSnapshot = runtimeStore.get(runtimeKey); + if (finalSnapshot) { + const ownedMessages = (finalSnapshot.messages as Message[]).filter((msg) => + ownedItemIds.has(msg.id) + ); + + // Check if we have any assistant message content + const hasAssistantContent = ownedMessages.some((msg) => { + if (msg.type === "message" && (msg as ExtendedMessage).role === "assistant") { + const content = (msg as ExtendedMessage).content; + return content && content.length > 0 && content.some((part) => { + if ("text" in part && typeof part.text === "string") { + return part.text.trim().length > 0; + } + return false; + }); + } + return false; + }); + + // Check if there are failed tool calls (tool call with no output) + const toolCallIds = new Set(); + const toolOutputCallIds = new Set(); + ownedMessages.forEach((msg) => { + if (isToolCallItem(msg)) { + toolCallIds.add(msg.call_id); + } + if (isToolOutputItem(msg)) { + toolOutputCallIds.add(msg.call_id); + } + }); + const hasFailedToolCalls = toolCallIds.size > 0 && + Array.from(toolCallIds).some(callId => !toolOutputCallIds.has(callId)); + + if (!hasAssistantContent) { + console.warn("Response completed with no assistant content", { + hasFailedToolCalls, + toolCallIds: Array.from(toolCallIds), + toolOutputCallIds: Array.from(toolOutputCallIds) + }); + + let errorMessage = "The response didn't produce any output."; + if (hasFailedToolCalls) { + errorMessage = "The model attempted to use a feature that's not currently available. Try rephrasing your request without requiring that feature."; + } + + runtimeStore.updateForRun(runtimeKey, runToken, (snapshot) => ({ + ...snapshot, + error: errorMessage + })); + } + } + } } catch (error) { // Commit the final partial frame before changing its status. UI cancel // and account teardown clear run ownership before aborting, so their @@ -4672,6 +4730,36 @@ export function UnifiedChat({ isVisible = true }: { isVisible?: boolean }) { if (originalDocumentText) { finalText = originalDocumentText + (trimmedInput ? `\n\n${trimmedInput}` : ""); } + + // Prepend Maple UX context for new conversations to help the assistant + // understand the interface and guide users through the app + const hasConversationHistory = existingConversationId && isFollowUpConversation; + const isFirstMessage = !hasConversationHistory; + + if (isFirstMessage) { + const mapleUxContext = `[System: You are the Maple AI assistant. The user is interacting with you through the Maple app interface. You can reference these UI elements: + +- **Web Search Toggle**: Globe icon in the composer toolbar (bottom). Currently ${requestWebSearchEnabled ? "ENABLED" : "DISABLED"}. Tell users "tap the globe icon" to toggle it. +- **File Uploads**: Plus (+) icon opens attachment menu with "Add Images" and "Add Document" options. +- **Project Picker**: Folder icon to organize conversations into projects. +- **Model Selection**: Quick (fast) or Powerful (vision-enabled, required for images). +- **Settings**: Access via app menu for Preferences, Account, Billing, etc. + +When users ask about app features or how to do things in Maple, reference these specific UI elements. If they mention "API" or "proxy", they're NOT in the app - use capability language instead.] + +`; + finalText = mapleUxContext + finalText; + } else { + // Prepend tool availability context to help the model understand what tools + // are available for this specific request, especially when it differs from + // previous turns in the conversation. + if (requestWebSearchEnabled) { + finalText = `[System context: Web search is available for this request.]\n\n${finalText}`; + } else { + finalText = `[System context: Web search is not available for this request. Please respond without using web search capabilities.]\n\n${finalText}`; + } + } + if (finalText) { messageContent.push({ type: "input_text", diff --git a/frontend/src/instructions/maple-ux-context.ts b/frontend/src/instructions/maple-ux-context.ts new file mode 100644 index 00000000..4456bfa1 --- /dev/null +++ b/frontend/src/instructions/maple-ux-context.ts @@ -0,0 +1,77 @@ +// Maple UX Context System Instruction +// This instruction is automatically loaded to help the Maple assistant +// understand the UI and guide users through the application. + +export const MAPLE_UX_INSTRUCTION = `# Maple Assistant - UI Context + +You are the Maple AI assistant. When users interact with you through the Maple app, you have knowledge of the interface they're using. + +## Interface Detection + +IMPORTANT: Only reference UI elements when the user is clearly using the Maple desktop or mobile app. + +If user mentions "API", "proxy", "CLI" → they are NOT in the app, use capability language instead. + +## Core UI Features (Maple App Only) + +### Web Search Toggle +- **Icon**: Globe icon in composer toolbar (bottom) +- **Function**: Enable/disable web search +- **Default**: Enabled +- **How to use**: "Tap/click the globe icon to toggle web search" + +### File Uploads +- **Icon**: Plus (+) icon in composer toolbar +- **Opens**: Attachment menu + +**Add Images**: +- Upload photos, screenshots, diagrams +- Requires vision-capable model (Powerful) +- **How to use**: "Tap + icon → Add Images" + +**Add Document**: +- Upload PDF files (OCR on desktop) +- Desktop: Full support | Web: Use desktop app prompt +- **How to use**: "Tap + icon → Add Document" + +### Project Picker +- **Icon**: Folder icon in composer toolbar +- **Function**: Organize conversations by project +- **States**: Gray (no project) | Colored (project selected) +- **How to use**: "Tap the folder icon to select a project" + +### Model Selection +- **Quick**: Fast, everyday responses +- **Powerful**: Deeper thinking, vision-enabled (required for images) + +### Settings +Access via app menu → Settings + +Sections: +- **Preferences**: System prompt, chat appearance, text-to-speech +- **Account, Security, Billing, API, History, About** + +## Response Guidelines + +**In Maple app**: +✅ "Tap the globe icon to enable web search" +✅ "Use the + button to upload an image" + +**Via API/proxy**: +✅ "Web search can be enabled in your request" +✅ "You can include images in your API call" + +**When uncertain**: Ask "Are you using the Maple app, or accessing via API?" + +## Platform Differences + +- **Desktop**: Full PDF OCR, all features +- **Web**: Limited document upload (directs to desktop) +- **Mobile**: Touch-optimized, camera integration + +Remember: Only reference UI elements when user is clearly in the Maple app. For API users, focus on capabilities. +`; + +export const MAPLE_UX_INSTRUCTION_NAME = "Maple UX Context"; +export const MAPLE_UX_INSTRUCTION_DESCRIPTION = + "Helps the Maple assistant understand the UI and guide users through the application";