diff --git a/src/components/ChatInput.tsx b/src/components/ChatInput.tsx index a5f0523d7..ff9d1c26a 100644 --- a/src/components/ChatInput.tsx +++ b/src/components/ChatInput.tsx @@ -2,10 +2,9 @@ import React, { useState, useRef, useCallback, useEffect, useId } from "react"; import styled from "@emotion/styled"; import { CommandSuggestions, COMMAND_SUGGESTION_KEYS } from "./CommandSuggestions"; import type { Toast } from "./ChatInputToast"; -import { ChatInputToast, SolutionLabel } from "./ChatInputToast"; -import type { ParsedCommand } from "@/utils/slashCommands/types"; +import { ChatInputToast } from "./ChatInputToast"; +import { createCommandToast, createErrorToast } from "./ChatInputToasts"; import { parseCommand } from "@/utils/slashCommands/parser"; -import type { SendMessageError as SendMessageErrorType } from "@/types/errors"; import { usePersistedState, updatePersistedState } from "@/hooks/usePersistedState"; import { useMode } from "@/contexts/ModeContext"; import { ChatToggles } from "./ChatToggles"; @@ -25,6 +24,11 @@ import { ModelSelector, type ModelSelectorRef } from "./ModelSelector"; import { useModelLRU } from "@/hooks/useModelLRU"; import { VimTextArea } from "./VimTextArea"; import { ImageAttachments, type ImageAttachment } from "./ImageAttachments"; +import { + extractImagesFromClipboard, + extractImagesFromDrop, + processImageFiles, +} from "@/utils/imageHandling"; import type { ThinkingLevel } from "@/types/thinking"; import type { CmuxFrontendMetadata, CompactionRequestData } from "@/types/message"; @@ -138,196 +142,6 @@ export interface ChatInputProps { } // Helper function to convert parsed command to display toast -const createCommandToast = (parsed: ParsedCommand): Toast | null => { - if (!parsed) return null; - - switch (parsed.type) { - case "providers-help": - return { - id: Date.now().toString(), - type: "error", - title: "Providers Command", - message: "Configure AI provider settings", - solution: ( - <> - Usage: - /providers set <provider> <key> <value> -
-
- Example: - /providers set anthropic apiKey YOUR_API_KEY - - ), - }; - - case "providers-missing-args": { - const missing = - parsed.argCount === 0 - ? "provider, key, and value" - : parsed.argCount === 1 - ? "key and value" - : parsed.argCount === 2 - ? "value" - : ""; - - return { - id: Date.now().toString(), - type: "error", - title: "Missing Arguments", - message: `Missing ${missing} for /providers set`, - solution: ( - <> - Usage: - /providers set <provider> <key> <value> - - ), - }; - } - - case "providers-invalid-subcommand": - return { - id: Date.now().toString(), - type: "error", - title: "Invalid Subcommand", - message: `Invalid subcommand '${parsed.subcommand}'`, - solution: ( - <> - Available Commands: - /providers set - Configure provider settings - - ), - }; - - case "model-help": - return { - id: Date.now().toString(), - type: "error", - title: "Model Command", - message: "Select AI model for this session", - solution: ( - <> - Usage: - /model <abbreviation> or /model <provider:model> -
-
- Examples: - /model sonnet -
- /model anthropic:opus-4-1 - - ), - }; - - case "telemetry-help": - return { - id: Date.now().toString(), - type: "error", - title: "Telemetry Command", - message: "Enable or disable usage telemetry", - solution: ( - <> - Usage: - /telemetry <on|off> -
-
- Examples: - /telemetry off -
- /telemetry on - - ), - }; - - case "fork-help": - return { - id: Date.now().toString(), - type: "error", - title: "Fork Command", - message: "Fork current workspace with a new name", - solution: ( - <> - Usage: - /fork <new-name> [optional start message] -
-
- Examples: - /fork experiment-branch -
- /fork refactor Continue with refactoring approach - - ), - }; - - case "unknown-command": { - const cmd = "/" + parsed.command + (parsed.subcommand ? " " + parsed.subcommand : ""); - return { - id: Date.now().toString(), - type: "error", - message: `Unknown command: ${cmd}`, - }; - } - - default: - return null; - } -}; - -// Helper function to convert SendMessageError to Toast -const createErrorToast = (error: SendMessageErrorType): Toast => { - switch (error.type) { - case "api_key_not_found": - return { - id: Date.now().toString(), - type: "error", - title: "API Key Not Found", - message: `The ${error.provider} provider requires an API key to function.`, - solution: ( - <> - Quick Fix: - /providers set {error.provider} apiKey YOUR_API_KEY - - ), - }; - - case "provider_not_supported": - return { - id: Date.now().toString(), - type: "error", - title: "Provider Not Supported", - message: `The ${error.provider} provider is not supported yet.`, - solution: ( - <> - Try This: - Use an available provider from /providers list - - ), - }; - - case "invalid_model_string": - return { - id: Date.now().toString(), - type: "error", - title: "Invalid Model Format", - message: error.message, - solution: ( - <> - Expected Format: - provider:model-name (e.g., anthropic:claude-opus-4-1) - - ), - }; - - case "unknown": - default: - return { - id: Date.now().toString(), - type: "error", - title: "Message Send Failed", - message: error.raw || "An unexpected error occurred while sending your message.", - }; - } -}; - /** * Prepare compaction message from /compact command * Returns the actual message text (summarization request), metadata, and options @@ -572,30 +386,14 @@ export const ChatInput: React.FC = ({ const items = e.clipboardData?.items; if (!items) return; - // Look for image items in clipboard - for (const item of Array.from(items)) { - if (!item?.type.startsWith("image/")) continue; - - e.preventDefault(); // Prevent default paste behavior for images + const imageFiles = extractImagesFromClipboard(items); + if (imageFiles.length === 0) return; - const file = item.getAsFile(); - if (!file) continue; + e.preventDefault(); // Prevent default paste behavior for images - // Convert to base64 data URL - const reader = new FileReader(); - reader.onload = (event) => { - const dataUrl = event.target?.result as string; - if (dataUrl) { - const attachment: ImageAttachment = { - id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, - dataUrl, - mimeType: file.type, - }; - setImageAttachments((prev) => [...prev, attachment]); - } - }; - reader.readAsDataURL(file); - } + void processImageFiles(imageFiles).then((attachments) => { + setImageAttachments((prev) => [...prev, ...attachments]); + }); }, []); // Handle removing an image attachment @@ -603,6 +401,27 @@ export const ChatInput: React.FC = ({ setImageAttachments((prev) => prev.filter((img) => img.id !== id)); }, []); + // Handle drag over to allow drop + const handleDragOver = useCallback((e: React.DragEvent) => { + // Check if drag contains files + if (e.dataTransfer.types.includes("Files")) { + e.preventDefault(); + e.dataTransfer.dropEffect = "copy"; + } + }, []); + + // Handle drop to extract images + const handleDrop = useCallback((e: React.DragEvent) => { + e.preventDefault(); + + const imageFiles = extractImagesFromDrop(e.dataTransfer); + if (imageFiles.length === 0) return; + + void processImageFiles(imageFiles).then((attachments) => { + setImageAttachments((prev) => [...prev, ...attachments]); + }); + }, []); + // Handle command selection const handleCommandSelect = useCallback( (suggestion: SlashSuggestion) => { @@ -818,10 +637,33 @@ export const ChatInput: React.FC = ({ try { // Prepare image parts if any - const imageParts = imageAttachments.map((img) => ({ - image: img.dataUrl, - mimeType: img.mimeType, - })); + const imageParts = imageAttachments.map((img, index) => { + // Validate before sending to help with debugging + if (!img.url || typeof img.url !== "string") { + console.error( + `Image attachment [${index}] has invalid url:`, + typeof img.url, + img.url?.slice(0, 50) + ); + } + if (!img.url?.startsWith("data:")) { + console.error( + `Image attachment [${index}] url is not a data URL:`, + img.url?.slice(0, 100) + ); + } + if (!img.mediaType || typeof img.mediaType !== "string") { + console.error( + `Image attachment [${index}] has invalid mediaType:`, + typeof img.mediaType, + img.mediaType + ); + } + return { + url: img.url, + mediaType: img.mediaType, + }; + }); // When editing a /compact command, regenerate the actual summarization request let actualMessageText = messageText; @@ -982,6 +824,8 @@ export const ChatInput: React.FC = ({ onChange={setInput} onKeyDown={handleKeyDown} onPaste={handlePaste} + onDragOver={handleDragOver} + onDrop={handleDrop} suppressKeys={showCommandSuggestions ? COMMAND_SUGGESTION_KEYS : undefined} placeholder={placeholder} disabled={!editingMessage && (disabled || isSending || isCompacting)} diff --git a/src/components/ChatInputToasts.tsx b/src/components/ChatInputToasts.tsx new file mode 100644 index 000000000..88f6b8abc --- /dev/null +++ b/src/components/ChatInputToasts.tsx @@ -0,0 +1,200 @@ +import React from "react"; +import type { Toast } from "./ChatInputToast"; +import { SolutionLabel } from "./ChatInputToast"; +import type { ParsedCommand } from "@/utils/slashCommands/types"; +import type { SendMessageError as SendMessageErrorType } from "@/types/errors"; + +/** + * Creates a toast message for command-related errors and help messages + */ +export const createCommandToast = (parsed: ParsedCommand): Toast | null => { + if (!parsed) return null; + + switch (parsed.type) { + case "providers-help": + return { + id: Date.now().toString(), + type: "error", + title: "Providers Command", + message: "Configure AI provider settings", + solution: ( + <> + Usage: + /providers set <provider> <key> <value> +
+
+ Example: + /providers set anthropic apiKey YOUR_API_KEY + + ), + }; + + case "providers-missing-args": { + const missing = + parsed.argCount === 0 + ? "provider, key, and value" + : parsed.argCount === 1 + ? "key and value" + : parsed.argCount === 2 + ? "value" + : ""; + + return { + id: Date.now().toString(), + type: "error", + title: "Missing Arguments", + message: `Missing ${missing} for /providers set`, + solution: ( + <> + Usage: + /providers set <provider> <key> <value> + + ), + }; + } + + case "providers-invalid-subcommand": + return { + id: Date.now().toString(), + type: "error", + title: "Invalid Subcommand", + message: `Invalid subcommand '${parsed.subcommand}'`, + solution: ( + <> + Available Commands: + /providers set - Configure provider settings + + ), + }; + + case "model-help": + return { + id: Date.now().toString(), + type: "error", + title: "Model Command", + message: "Select AI model for this session", + solution: ( + <> + Usage: + /model <abbreviation> or /model <provider:model> +
+
+ Examples: + /model sonnet +
+ /model anthropic:opus-4-1 + + ), + }; + + case "telemetry-help": + return { + id: Date.now().toString(), + type: "error", + title: "Telemetry Command", + message: "Enable or disable usage telemetry", + solution: ( + <> + Usage: + /telemetry <on|off> +
+
+ Examples: + /telemetry off +
+ /telemetry on + + ), + }; + + case "fork-help": + return { + id: Date.now().toString(), + type: "error", + title: "Fork Command", + message: "Fork current workspace with a new name", + solution: ( + <> + Usage: + /fork <new-name> [optional start message] +
+
+ Examples: + /fork experiment-branch +
+ /fork refactor Continue with refactoring approach + + ), + }; + + case "unknown-command": { + const cmd = "/" + parsed.command + (parsed.subcommand ? " " + parsed.subcommand : ""); + return { + id: Date.now().toString(), + type: "error", + message: `Unknown command: ${cmd}`, + }; + } + + default: + return null; + } +}; + +/** + * Converts a SendMessageError to a Toast for display + */ +export const createErrorToast = (error: SendMessageErrorType): Toast => { + switch (error.type) { + case "api_key_not_found": + return { + id: Date.now().toString(), + type: "error", + title: "API Key Not Found", + message: `The ${error.provider} provider requires an API key to function.`, + solution: ( + <> + Quick Fix: + /providers set {error.provider} apiKey YOUR_API_KEY + + ), + }; + + case "provider_not_supported": + return { + id: Date.now().toString(), + type: "error", + title: "Provider Not Supported", + message: `The ${error.provider} provider is not supported yet.`, + solution: ( + <> + Try This: + Use an available provider from /providers list + + ), + }; + + case "invalid_model_string": + return { + id: Date.now().toString(), + type: "error", + title: "Invalid Model Format", + message: error.message, + solution: ( + <> + Expected Format: + provider:model-name (e.g., anthropic:claude-opus-4-1) + + ), + }; + + case "unknown": + default: + return { + id: Date.now().toString(), + type: "error", + title: "Message Send Failed", + message: error.raw || "An unexpected error occurred while sending your message.", + }; + } +}; diff --git a/src/components/ImageAttachments.tsx b/src/components/ImageAttachments.tsx index d9d2d1176..bd7efd32c 100644 --- a/src/components/ImageAttachments.tsx +++ b/src/components/ImageAttachments.tsx @@ -49,8 +49,8 @@ const RemoveButton = styled.button` export interface ImageAttachment { id: string; - dataUrl: string; - mimeType: string; + url: string; + mediaType: string; } interface ImageAttachmentsProps { @@ -65,7 +65,7 @@ export const ImageAttachments: React.FC = ({ images, onRe {images.map((image) => ( - + onRemove(image.id)} title="Remove image"> × diff --git a/src/components/Messages/UserMessage.stories.tsx b/src/components/Messages/UserMessage.stories.tsx index 7ca0f28bc..ff8217485 100644 --- a/src/components/Messages/UserMessage.stories.tsx +++ b/src/components/Messages/UserMessage.stories.tsx @@ -71,8 +71,8 @@ export const WithSingleImage: Story = { message: createUserMessage("What's in this image?", { imageParts: [ { - image: "https://placehold.co/600x400", - mimeType: "image/png", + url: "https://placehold.co/600x400", + mediaType: "image/png", }, ], }), @@ -84,16 +84,16 @@ export const WithMultipleImages: Story = { message: createUserMessage("Compare these screenshots:", { imageParts: [ { - image: "https://placehold.co/600x400?text=Before", - mimeType: "image/png", + url: "https://placehold.co/600x400?text=Before", + mediaType: "image/png", }, { - image: "https://placehold.co/600x400?text=After", - mimeType: "image/png", + url: "https://placehold.co/600x400?text=After", + mediaType: "image/png", }, { - image: "https://placehold.co/600x400?text=Expected", - mimeType: "image/png", + url: "https://placehold.co/600x400?text=Expected", + mediaType: "image/png", }, ], }), @@ -118,8 +118,8 @@ export const EmptyContent: Story = { message: createUserMessage("", { imageParts: [ { - image: "https://placehold.co/300x400?text=Image+Only", - mimeType: "image/png", + url: "https://placehold.co/300x400?text=Image+Only", + mediaType: "image/png", }, ], }), diff --git a/src/components/Messages/UserMessage.tsx b/src/components/Messages/UserMessage.tsx index c23739987..c86cf801c 100644 --- a/src/components/Messages/UserMessage.tsx +++ b/src/components/Messages/UserMessage.tsx @@ -141,7 +141,7 @@ export const UserMessage: React.FC = ({ {message.imageParts && message.imageParts.length > 0 && ( {message.imageParts.map((img, idx) => ( - + ))} )} diff --git a/src/services/agentSession.ts b/src/services/agentSession.ts index abb776564..23b2c653a 100644 --- a/src/services/agentSession.ts +++ b/src/services/agentSession.ts @@ -15,8 +15,8 @@ import { Ok, Err } from "@/types/result"; import { enforceThinkingPolicy } from "@/utils/thinking/policy"; interface ImagePart { - image: string; - mimeType: string; + url: string; + mediaType: string; } export interface AgentSessionChatEvent { @@ -220,16 +220,23 @@ export class AgentSession { const messageId = `user-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`; const additionalParts = imageParts && imageParts.length > 0 - ? imageParts.map((img) => { - assert(typeof img.image === "string", "image part must include base64 string content"); + ? imageParts.map((img, index) => { assert( - typeof img.mimeType === "string" && img.mimeType.trim().length > 0, - "image part must include a mimeType" + typeof img.url === "string", + `image part [${index}] must include url string content (got ${typeof img.url}): ${JSON.stringify(img).slice(0, 200)}` + ); + assert( + img.url.startsWith("data:"), + `image part [${index}] url must be a data URL (got: ${img.url.slice(0, 50)}...)` + ); + assert( + typeof img.mediaType === "string" && img.mediaType.trim().length > 0, + `image part [${index}] must include a mediaType (got ${typeof img.mediaType}): ${JSON.stringify(img).slice(0, 200)}` ); return { - type: "image" as const, - image: img.image, - mimeType: img.mimeType, + type: "file" as const, + url: img.url, + mediaType: img.mediaType, }; }) : undefined; diff --git a/src/services/ipcMain.ts b/src/services/ipcMain.ts index 01fd1b4c1..ccd6f4893 100644 --- a/src/services/ipcMain.ts +++ b/src/services/ipcMain.ts @@ -528,7 +528,7 @@ export class IpcMain { _event, workspaceId: string, message: string, - options?: SendMessageOptions & { imageParts?: Array<{ image: string; mimeType: string }> } + options?: SendMessageOptions & { imageParts?: Array<{ url: string; mediaType: string }> } ) => { log.debug("sendMessage handler: Received", { workspaceId, diff --git a/src/types/ipc.ts b/src/types/ipc.ts index 128875a0a..e0490c6d5 100644 --- a/src/types/ipc.ts +++ b/src/types/ipc.ts @@ -203,7 +203,7 @@ export interface IPCApi { sendMessage( workspaceId: string, message: string, - options?: SendMessageOptions & { imageParts?: Array<{ image: string; mimeType: string }> } + options?: SendMessageOptions & { imageParts?: Array<{ url: string; mediaType: string }> } ): Promise>; resumeStream( workspaceId: string, diff --git a/src/types/message.ts b/src/types/message.ts index 04de8ee51..a307766a4 100644 --- a/src/types/message.ts +++ b/src/types/message.ts @@ -71,11 +71,13 @@ export interface CmuxReasoningPart { timestamp?: number; } -// Image part type for multimodal messages +// File/Image part type for multimodal messages (matches AI SDK FileUIPart) +// Images are represented as files with image/* mediaType export interface CmuxImagePart { - type: "image"; - image: string | Uint8Array | ArrayBuffer | URL; // base64 string or binary data or URL - mimeType?: string; // e.g., "image/png", "image/jpeg" + type: "file"; + mediaType: string; // IANA media type, e.g., "image/png", "image/jpeg" + url: string; // Data URL (e.g., "data:image/png;base64,...") or hosted URL + filename?: string; // Optional filename } // CmuxMessage extends UIMessage with our metadata and custom parts @@ -92,7 +94,7 @@ export type DisplayedMessage = id: string; // Display ID for UI/React keys historyId: string; // Original CmuxMessage ID for history operations content: string; - imageParts?: Array<{ image: string; mimeType?: string }>; // Optional image attachments + imageParts?: Array<{ url: string; mediaType?: string }>; // Optional image attachments historySequence: number; // Global ordering across all messages timestamp?: number; compactionRequest?: { diff --git a/src/utils/imageHandling.test.ts b/src/utils/imageHandling.test.ts new file mode 100644 index 000000000..832277e1d --- /dev/null +++ b/src/utils/imageHandling.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, test } from "@jest/globals"; +import { + generateImageId, + fileToImageAttachment, + extractImagesFromClipboard, + extractImagesFromDrop, + processImageFiles, +} from "./imageHandling"; + +// Mock FileReader for Node.js environment +class MockFileReader { + onload: ((event: { target: { result: string } }) => void) | null = null; + onerror: (() => void) | null = null; + + readAsDataURL(blob: Blob) { + // Simulate async read with setTimeout + setTimeout(() => { + // Create a fake base64 data URL based on the blob type + const fakeDataUrl = `data:${blob.type};base64,ZmFrZWRhdGE=`; + if (this.onload) { + this.onload({ target: { result: fakeDataUrl } }); + } + }, 0); + } +} + +global.FileReader = MockFileReader as unknown as typeof FileReader; + +describe("imageHandling", () => { + describe("generateImageId", () => { + test("generates unique IDs", () => { + const id1 = generateImageId(); + const id2 = generateImageId(); + + expect(id1).not.toBe(id2); + expect(id1).toMatch(/^\d+-[a-z0-9]+$/); + expect(id2).toMatch(/^\d+-[a-z0-9]+$/); + }); + }); + + describe("fileToImageAttachment", () => { + test("converts a File to ImageAttachment", async () => { + // Create a mock image file + const blob = new Blob(["fake image data"], { type: "image/png" }); + const file = new File([blob], "test.png", { type: "image/png" }); + + const attachment = await fileToImageAttachment(file); + + expect(attachment).toMatchObject({ + id: expect.stringMatching(/^\d+-[a-z0-9]+$/), + url: expect.stringContaining("data:image/png;base64,"), + mediaType: "image/png", + }); + }); + + test("handles JPEG images", async () => { + const blob = new Blob(["fake jpeg data"], { type: "image/jpeg" }); + const file = new File([blob], "test.jpg", { type: "image/jpeg" }); + + const attachment = await fileToImageAttachment(file); + + expect(attachment.mediaType).toBe("image/jpeg"); + expect(attachment.url).toContain("data:image/jpeg;base64,"); + }); + }); + + describe("extractImagesFromClipboard", () => { + test("extracts image files from clipboard items", () => { + // Mock clipboard items + const mockFile = new File(["fake image"], "test.png", { type: "image/png" }); + + const mockItems = [ + { + type: "image/png", + getAsFile: () => mockFile, + }, + { + type: "text/plain", + getAsFile: () => null, + }, + ] as unknown as DataTransferItemList; + + const files = extractImagesFromClipboard(mockItems); + + expect(files).toHaveLength(1); + expect(files[0]).toBe(mockFile); + }); + + test("ignores non-image items", () => { + const mockItems: DataTransferItemList = [ + { + type: "text/plain", + getAsFile: () => new File(["text"], "test.txt", { type: "text/plain" }), + }, + { + type: "text/html", + getAsFile: () => new File(["

html

"], "test.html", { type: "text/html" }), + }, + ] as unknown as DataTransferItemList; + + const files = extractImagesFromClipboard(mockItems); + + expect(files).toHaveLength(0); + }); + + test("handles multiple images", () => { + const mockFile1 = new File(["fake image 1"], "test1.png", { type: "image/png" }); + const mockFile2 = new File(["fake image 2"], "test2.jpg", { type: "image/jpeg" }); + + const mockItems = [ + { + type: "image/png", + getAsFile: () => mockFile1, + }, + { + type: "image/jpeg", + getAsFile: () => mockFile2, + }, + ] as unknown as DataTransferItemList; + + const files = extractImagesFromClipboard(mockItems); + + expect(files).toHaveLength(2); + expect(files).toContain(mockFile1); + expect(files).toContain(mockFile2); + }); + }); + + describe("extractImagesFromDrop", () => { + test("extracts image files from DataTransfer", () => { + const mockFile1 = new File(["image 1"], "test1.png", { type: "image/png" }); + const mockFile2 = new File(["text"], "test.txt", { type: "text/plain" }); + const mockFile3 = new File(["image 2"], "test2.jpg", { type: "image/jpeg" }); + + const mockDataTransfer = { + files: [mockFile1, mockFile2, mockFile3], + }; + + const files = extractImagesFromDrop(mockDataTransfer as unknown as DataTransfer); + + expect(files).toHaveLength(2); + expect(files).toContain(mockFile1); + expect(files).toContain(mockFile3); + expect(files).not.toContain(mockFile2); + }); + + test("returns empty array when no images", () => { + const mockFile = new File(["text"], "test.txt", { type: "text/plain" }); + + const mockDataTransfer = { + files: [mockFile], + }; + + const files = extractImagesFromDrop(mockDataTransfer as unknown as DataTransfer); + + expect(files).toHaveLength(0); + }); + + test("accepts files with image extensions when MIME type is empty (macOS drag-drop)", () => { + const mockFile1 = new File(["image"], "photo.png", { type: "" }); // Empty type + const mockFile2 = new File(["image"], "picture.jpg", { type: "" }); // Empty type + const mockFile3 = new File(["text"], "document.txt", { type: "" }); // Empty type, not an image + + const mockDataTransfer = { + files: [mockFile1, mockFile2, mockFile3], + }; + + const files = extractImagesFromDrop(mockDataTransfer as unknown as DataTransfer); + + expect(files).toHaveLength(2); + expect(files).toContain(mockFile1); + expect(files).toContain(mockFile2); + expect(files).not.toContain(mockFile3); + }); + }); + + describe("processImageFiles", () => { + test("converts multiple files to attachments", async () => { + const file1 = new File(["image 1"], "test1.png", { type: "image/png" }); + const file2 = new File(["image 2"], "test2.jpg", { type: "image/jpeg" }); + + const attachments = await processImageFiles([file1, file2]); + + expect(attachments).toHaveLength(2); + expect(attachments[0].mediaType).toBe("image/png"); + expect(attachments[1].mediaType).toBe("image/jpeg"); + expect(attachments[0].url).toContain("data:image/png;base64,"); + expect(attachments[1].url).toContain("data:image/jpeg;base64,"); + }); + + test("handles empty array", async () => { + const attachments = await processImageFiles([]); + + expect(attachments).toHaveLength(0); + }); + }); +}); diff --git a/src/utils/imageHandling.ts b/src/utils/imageHandling.ts new file mode 100644 index 000000000..77b6af611 --- /dev/null +++ b/src/utils/imageHandling.ts @@ -0,0 +1,103 @@ +import type { ImageAttachment } from "@/components/ImageAttachments"; + +/** + * Generates a unique ID for an image attachment + */ +export function generateImageId(): string { + return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; +} + +/** + * Detects MIME type from file extension as fallback + */ +function getMimeTypeFromExtension(filename: string): string { + const ext = filename.toLowerCase().split(".").pop(); + const mimeTypes: Record = { + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", + webp: "image/webp", + bmp: "image/bmp", + svg: "image/svg+xml", + }; + return mimeTypes[ext ?? ""] ?? "image/png"; +} + +/** + * Converts a File to an ImageAttachment with a base64 data URL + */ +export async function fileToImageAttachment(file: File): Promise { + const dataUrl = await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = (event) => { + const result = event.target?.result as string; + if (result) { + resolve(result); + } else { + reject(new Error("Failed to read file")); + } + }; + reader.onerror = () => reject(new Error("Failed to read file")); + reader.readAsDataURL(file); + }); + + // Use file.type if available, otherwise infer from extension + const mediaType = file.type !== "" ? file.type : getMimeTypeFromExtension(file.name); + + return { + id: generateImageId(), + url: dataUrl, + mediaType, + }; +} + +/** + * Extracts image files from clipboard items + */ +export function extractImagesFromClipboard(items: DataTransferItemList): File[] { + const imageFiles: File[] = []; + + for (const item of Array.from(items)) { + if (item?.type.startsWith("image/")) { + const file = item.getAsFile(); + if (file) { + imageFiles.push(file); + } + } + } + + return imageFiles; +} + +/** + * Checks if a file is likely an image based on extension + */ +function hasImageExtension(filename: string): boolean { + const ext = filename.toLowerCase().split(".").pop(); + return ["png", "jpg", "jpeg", "gif", "webp", "bmp", "svg"].includes(ext ?? ""); +} + +/** + * Extracts image files from drag and drop DataTransfer + * Accepts files with image MIME type OR image file extensions (for macOS drag-and-drop) + */ +export function extractImagesFromDrop(dataTransfer: DataTransfer): File[] { + const imageFiles: File[] = []; + + for (const file of Array.from(dataTransfer.files)) { + // Accept files with image MIME type, or files with image extensions (macOS drag-drop has empty type) + if (file.type.startsWith("image/") || (file.type === "" && hasImageExtension(file.name))) { + imageFiles.push(file); + } + } + + return imageFiles; +} + +/** + * Processes multiple image files and converts them to attachments + */ +export async function processImageFiles(files: File[]): Promise { + return await Promise.all(files.map(fileToImageAttachment)); +} diff --git a/src/utils/messages/StreamingMessageAggregator.ts b/src/utils/messages/StreamingMessageAggregator.ts index 7959ba8da..4bfc1e67c 100644 --- a/src/utils/messages/StreamingMessageAggregator.ts +++ b/src/utils/messages/StreamingMessageAggregator.ts @@ -1,4 +1,4 @@ -import type { CmuxMessage, CmuxMetadata, DisplayedMessage } from "@/types/message"; +import type { CmuxMessage, CmuxMetadata, CmuxImagePart, DisplayedMessage } from "@/types/message"; import { createCmuxMessage } from "@/types/message"; import type { StreamStartEvent, @@ -511,10 +511,14 @@ export class StreamingMessageAggregator { .join(""); const imageParts = message.parts - .filter((p) => p.type === "image") + .filter((p): p is CmuxImagePart => { + // Accept both new "file" type and legacy "image" type (from before PR #308) + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + return p.type === "file" || (p as any).type === "image"; + }) .map((p) => ({ - image: typeof p.image === "string" ? p.image : "", - mimeType: p.mimeType, + url: typeof p.url === "string" ? p.url : "", + mediaType: p.mediaType, })); // Check if this is a compaction request message diff --git a/src/utils/messages/modelMessageTransform.ts b/src/utils/messages/modelMessageTransform.ts index e84c91db8..2451cfe45 100644 --- a/src/utils/messages/modelMessageTransform.ts +++ b/src/utils/messages/modelMessageTransform.ts @@ -486,12 +486,12 @@ function mergeConsecutiveUserMessages(messages: ModelMessage[]): ModelMessage[] // Merge with newline prefix const mergedText = prevText + "\n" + currentText; - // Collect image parts from both messages + // Collect file/image parts from both messages const prevImageParts = Array.isArray(prevMsg.content) - ? prevMsg.content.filter((c) => c.type === "image") + ? prevMsg.content.filter((c) => c.type === "file") : []; const currentImageParts = Array.isArray(msg.content) - ? msg.content.filter((c) => c.type === "image") + ? msg.content.filter((c) => c.type === "file") : []; // Update the previous message with merged text and all image parts diff --git a/tests/ipcMain/helpers.ts b/tests/ipcMain/helpers.ts index bff1647e0..475a7d8d4 100644 --- a/tests/ipcMain/helpers.ts +++ b/tests/ipcMain/helpers.ts @@ -32,7 +32,7 @@ export async function sendMessage( mockIpcRenderer: IpcRenderer, workspaceId: string, message: string, - options?: SendMessageOptions + options?: SendMessageOptions & { imageParts?: Array<{ url: string; mediaType: string }> } ): Promise> { return (await mockIpcRenderer.invoke( IPC_CHANNELS.WORKSPACE_SEND_MESSAGE, @@ -299,6 +299,47 @@ export async function waitForFileExists(filePath: string, timeoutMs = 5000): Pro }, timeoutMs); } +/** + * Wait for stream to complete successfully + * Common pattern: create collector, wait for end, assert success + */ +export async function waitForStreamSuccess( + sentEvents: Array<{ channel: string; data: unknown }>, + workspaceId: string, + timeoutMs = 30000 +): Promise { + const collector = createEventCollector(sentEvents, workspaceId); + await collector.waitForEvent("stream-end", timeoutMs); + assertStreamSuccess(collector); + return collector; +} + +/** + * Read and parse chat history from disk + */ +export async function readChatHistory( + tempDir: string, + workspaceId: string +): Promise }>> { + const fsPromises = await import("fs/promises"); + const historyPath = path.join(tempDir, "sessions", workspaceId, "chat.jsonl"); + const historyContent = await fsPromises.readFile(historyPath, "utf-8"); + return historyContent + .trim() + .split("\n") + .map((line: string) => JSON.parse(line)); +} + +/** + * Test image fixtures (1x1 pixel PNGs) + */ +export const TEST_IMAGES = { + RED_PIXEL: + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==", + BLUE_PIXEL: + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M/wHwAEBgIApD5fRAAAAABJRU5ErkJggg==", +} as const; + /** * Wait for a file to NOT exist with retry logic */ diff --git a/tests/ipcMain/sendMessage.test.ts b/tests/ipcMain/sendMessage.test.ts index 26d67473c..5edd61b0c 100644 --- a/tests/ipcMain/sendMessage.test.ts +++ b/tests/ipcMain/sendMessage.test.ts @@ -15,6 +15,10 @@ import { assertError, waitFor, buildLargeHistory, + waitForStreamSuccess, + readChatHistory, + TEST_IMAGES, + modelString, } from "./helpers"; import type { StreamDeltaEvent } from "../../src/types/stream"; import { IPC_CHANNELS } from "../../src/constants/ipc-constants"; @@ -543,9 +547,11 @@ describeIntegration("IpcMain sendMessage integration tests", () => { expect(result.success).toBe(true); // Wait for stream to complete - const collector = createEventCollector(env.sentEvents, workspaceId); - await collector.waitForEvent("stream-end", provider === "openai" ? 30000 : 10000); - assertStreamSuccess(collector); + const collector = await waitForStreamSuccess( + env.sentEvents, + workspaceId, + provider === "openai" ? 30000 : 10000 + ); // Get the final assistant message const finalMessage = collector.getFinalMessage(); @@ -790,8 +796,7 @@ These are general instructions that apply to all modes. ); // Collect response - const collector = createEventCollector(env.sentEvents, workspaceId); - await collector.waitForEvent("stream-end", 10000); + const collector = await waitForStreamSuccess(env.sentEvents, workspaceId, 10000); results[provider] = { success: result.success, @@ -1190,9 +1195,7 @@ These are general instructions that apply to all modes. expect(result.success).toBe(true); // Wait for stream to complete - const collector = createEventCollector(env.sentEvents, workspaceId); - await collector.waitForEvent("stream-end", 10000); - assertStreamSuccess(collector); + const collector = await waitForStreamSuccess(env.sentEvents, workspaceId, 10000); // Get the final assistant message const finalMessage = collector.getFinalMessage(); @@ -1449,3 +1452,82 @@ These are general instructions that apply to all modes. 5000 ); }); + +// Test image support across providers +describe.each(PROVIDER_CONFIGS)("%s:%s image support", (provider, model) => { + test.concurrent( + "should send images to AI model and get response", + async () => { + const { env, workspaceId, cleanup } = await setupWorkspace(provider); + try { + // Send message with image attachment + const result = await sendMessage(env.mockIpcRenderer, workspaceId, "What color is this?", { + model: modelString(provider, model), + imageParts: [{ url: TEST_IMAGES.RED_PIXEL, mediaType: "image/png" }], + }); + + expect(result.success).toBe(true); + + // Wait for stream to complete + const collector = await waitForStreamSuccess(env.sentEvents, workspaceId, 30000); + + // Verify we got a response about the image + const deltas = collector.getDeltas(); + expect(deltas.length).toBeGreaterThan(0); + + // Combine all text deltas + const fullResponse = deltas + .map((d) => (d as StreamDeltaEvent).delta) + .join("") + .toLowerCase(); + + // Should mention red color in some form + expect(fullResponse.length).toBeGreaterThan(0); + // Red pixel should be detected (flexible matching as different models may phrase differently) + expect(fullResponse).toMatch(/red|color/i); + } finally { + await cleanup(); + } + }, + 40000 // Vision models can be slower + ); + + test.concurrent( + "should preserve image parts through history", + async () => { + const { env, workspaceId, cleanup } = await setupWorkspace(provider); + try { + // Send message with image + const result = await sendMessage(env.mockIpcRenderer, workspaceId, "Describe this", { + model: modelString(provider, model), + imageParts: [{ url: TEST_IMAGES.BLUE_PIXEL, mediaType: "image/png" }], + }); + + expect(result.success).toBe(true); + + // Wait for stream to complete + await waitForStreamSuccess(env.sentEvents, workspaceId, 30000); + + // Read history from disk + const messages = await readChatHistory(env.tempDir, workspaceId); + + // Find the user message + const userMessage = messages.find((m: { role: string }) => m.role === "user"); + expect(userMessage).toBeDefined(); + + // Verify image part is preserved with correct format + if (userMessage) { + const imagePart = userMessage.parts.find((p: { type: string }) => p.type === "file"); + expect(imagePart).toBeDefined(); + if (imagePart) { + expect(imagePart.url).toBe(TEST_IMAGES.BLUE_PIXEL); + expect(imagePart.mediaType).toBe("image/png"); + } + } + } finally { + await cleanup(); + } + }, + 40000 + ); +});