From dada99a6d3145a3b53500e00a6d8dfc5ff67cda6 Mon Sep 17 00:00:00 2001 From: Jeremy Rose Date: Mon, 29 Sep 2025 16:11:42 -0700 Subject: [PATCH 1/3] show "Viewed Image" when the model views an image --- codex-rs/core/src/codex.rs | 11 +++++++++- codex-rs/core/src/rollout/policy.rs | 1 + .../src/event_processor_with_human_output.rs | 8 ++++++++ codex-rs/mcp-server/src/codex_tool_runner.rs | 1 + codex-rs/protocol/src/protocol.rs | 11 ++++++++++ codex-rs/tui/src/chatwidget.rs | 11 ++++++++++ ...cal_image_attachment_history_snapshot.snap | 6 ++++++ codex-rs/tui/src/chatwidget/tests.rs | 20 +++++++++++++++++++ codex-rs/tui/src/diff_render.rs | 4 +++- codex-rs/tui/src/history_cell.rs | 13 ++++++++++++ 10 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__local_image_attachment_history_snapshot.snap diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 7a2b2c8da0..1a90161bbd 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -110,6 +110,7 @@ use crate::protocol::Submission; use crate::protocol::TokenCountEvent; use crate::protocol::TokenUsage; use crate::protocol::TurnDiffEvent; +use crate::protocol::ViewImageToolCallEvent; use crate::protocol::WebSearchBeginEvent; use crate::rollout::RolloutRecorder; use crate::rollout::RolloutRecorderParams; @@ -2470,13 +2471,21 @@ async fn handle_function_call( )) })?; let abs = turn_context.resolve_path(Some(args.path)); - sess.inject_input(vec![InputItem::LocalImage { path: abs }]) + sess.inject_input(vec![InputItem::LocalImage { path: abs.clone() }]) .await .map_err(|_| { FunctionCallError::RespondToModel( "unable to attach image (no active task)".to_string(), ) })?; + sess.send_event(Event { + id: sub_id.clone(), + msg: EventMsg::ViewImageToolCall(ViewImageToolCallEvent { + call_id: call_id.clone(), + path: abs, + }), + }) + .await; Ok("attached local image path".to_string()) } diff --git a/codex-rs/core/src/rollout/policy.rs b/codex-rs/core/src/rollout/policy.rs index 2fd0efb0dc..8fc39e79f5 100644 --- a/codex-rs/core/src/rollout/policy.rs +++ b/codex-rs/core/src/rollout/policy.rs @@ -70,6 +70,7 @@ pub(crate) fn should_persist_event_msg(ev: &EventMsg) -> bool { | EventMsg::ListCustomPromptsResponse(_) | EventMsg::PlanUpdate(_) | EventMsg::ShutdownComplete + | EventMsg::ViewImageToolCall(_) | EventMsg::ConversationPath(_) => false, } } diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 4f231bab84..72c31c2979 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -580,6 +580,14 @@ impl EventProcessor for EventProcessorWithHumanOutput { EventMsg::ListCustomPromptsResponse(_) => { // Currently ignored in exec output. } + EventMsg::ViewImageToolCall(view) => { + ts_println!( + self, + "{} {}", + "viewed image".style(self.magenta), + view.path.display() + ); + } EventMsg::TurnAborted(abort_reason) => match abort_reason.reason { TurnAbortReason::Interrupted => { ts_println!(self, "task interrupted"); diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index db48da28e2..a83578c808 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -280,6 +280,7 @@ async fn run_codex_tool_session_inner( | EventMsg::ConversationPath(_) | EventMsg::UserMessage(_) | EventMsg::ShutdownComplete + | EventMsg::ViewImageToolCall(_) | EventMsg::EnteredReviewMode(_) | EventMsg::ExitedReviewMode(_) => { // For now, we do not do anything extra for these diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 01b4eb3a5a..b76b2a30da 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -477,6 +477,9 @@ pub enum EventMsg { ExecCommandEnd(ExecCommandEndEvent), + /// Notification that the agent attached a local image via the view_image tool. + ViewImageToolCall(ViewImageToolCallEvent), + ExecApprovalRequest(ExecApprovalRequestEvent), ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent), @@ -1074,6 +1077,14 @@ pub struct ExecCommandEndEvent { pub formatted_output: String, } +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +pub struct ViewImageToolCallEvent { + /// Identifier for the originating tool call. + pub call_id: String, + /// Local filesystem path provided to the tool. + pub path: PathBuf, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, TS)] #[serde(rename_all = "snake_case")] pub enum ExecOutputStream { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 724feb29f2..6bce9263de 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -39,6 +39,7 @@ use codex_core::protocol::TokenUsageInfo; use codex_core::protocol::TurnAbortReason; use codex_core::protocol::TurnDiffEvent; use codex_core::protocol::UserMessageEvent; +use codex_core::protocol::ViewImageToolCallEvent; use codex_core::protocol::WebSearchBeginEvent; use codex_core::protocol::WebSearchEndEvent; use codex_protocol::mcp_protocol::ConversationId; @@ -533,6 +534,15 @@ impl ChatWidget { )); } + fn on_view_image_tool_call(&mut self, event: ViewImageToolCallEvent) { + self.flush_answer_stream_with_separator(); + self.add_to_history(history_cell::new_view_image_tool_call( + event.path, + &self.config.cwd, + )); + self.request_redraw(); + } + fn on_patch_apply_end(&mut self, event: codex_core::protocol::PatchApplyEndEvent) { let ev2 = event.clone(); self.defer_or_handle( @@ -1394,6 +1404,7 @@ impl ChatWidget { EventMsg::PatchApplyBegin(ev) => self.on_patch_apply_begin(ev), EventMsg::PatchApplyEnd(ev) => self.on_patch_apply_end(ev), EventMsg::ExecCommandEnd(ev) => self.on_exec_command_end(ev), + EventMsg::ViewImageToolCall(ev) => self.on_view_image_tool_call(ev), EventMsg::McpToolCallBegin(ev) => self.on_mcp_tool_call_begin(ev), EventMsg::McpToolCallEnd(ev) => self.on_mcp_tool_call_end(ev), EventMsg::WebSearchBegin(ev) => self.on_web_search_begin(ev), diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__local_image_attachment_history_snapshot.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__local_image_attachment_history_snapshot.snap new file mode 100644 index 0000000000..da9c6d68fc --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__local_image_attachment_history_snapshot.snap @@ -0,0 +1,6 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: combined +--- +• Viewed Image + └ screenshots/example.png diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index f3799ad336..c41a0fda6f 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -34,6 +34,7 @@ use codex_core::protocol::ReviewRequest; use codex_core::protocol::StreamErrorEvent; use codex_core::protocol::TaskCompleteEvent; use codex_core::protocol::TaskStartedEvent; +use codex_core::protocol::ViewImageToolCallEvent; use codex_protocol::mcp_protocol::ConversationId; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; @@ -828,6 +829,25 @@ fn custom_prompt_enter_empty_does_not_send() { assert!(rx.try_recv().is_err(), "no app event should be sent"); } +#[test] +fn view_image_tool_call_adds_history_cell() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(); + let image_path = chat.config.cwd.join("screenshots").join("example.png"); + + chat.handle_codex_event(Event { + id: "sub-image".into(), + msg: EventMsg::ViewImageToolCall(ViewImageToolCallEvent { + call_id: "call-image".into(), + path: image_path, + }), + }); + + let cells = drain_insert_history(&mut rx); + assert_eq!(cells.len(), 1, "expected a single history cell"); + let combined = lines_to_single_string(&cells[0]); + assert_snapshot!("local_image_attachment_history_snapshot", combined); +} + // Snapshot test: interrupting a running exec finalizes the active cell with a red ✗ // marker (replacing the spinner) and flushes it into history. #[test] diff --git a/codex-rs/tui/src/diff_render.rs b/codex-rs/tui/src/diff_render.rs index 1e0c222bbd..7883509975 100644 --- a/codex-rs/tui/src/diff_render.rs +++ b/codex-rs/tui/src/diff_render.rs @@ -274,7 +274,9 @@ pub(crate) fn display_path_for(path: &Path, cwd: &Path) -> String { let chosen = if path_in_same_repo { pathdiff::diff_paths(path, cwd).unwrap_or_else(|| path.to_path_buf()) } else { - relativize_to_home(path).unwrap_or_else(|| path.to_path_buf()) + relativize_to_home(path) + .map(|p| PathBuf::from_iter([Path::new("~"), p.as_path()])) + .unwrap_or_else(|| path.to_path_buf()) }; chosen.display().to_string() } diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index d0e13003ff..df60ef7f9b 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -1,4 +1,5 @@ use crate::diff_render::create_diff_summary; +use crate::diff_render::display_path_for; use crate::exec_cell::CommandOutput; use crate::exec_cell::OutputLinesParams; use crate::exec_cell::TOOL_CALL_MAX_LINES; @@ -31,6 +32,7 @@ use image::DynamicImage; use image::ImageReader; use mcp_types::EmbeddedResourceResource; use mcp_types::ResourceLink; +use pathdiff::diff_paths; use ratatui::prelude::*; use ratatui::style::Modifier; use ratatui::style::Style; @@ -1050,6 +1052,17 @@ pub(crate) fn new_proposed_command(command: &[String]) -> PlainHistoryCell { PlainHistoryCell { lines } } +pub(crate) fn new_view_image_tool_call(path: PathBuf, cwd: &Path) -> PlainHistoryCell { + let display_path = display_path_for(&path, cwd); + + let lines: Vec> = vec![ + vec!["• ".dim(), "Viewed Image".bold()].into(), + vec![" └ ".dim(), display_path.dim()].into(), + ]; + + PlainHistoryCell { lines } +} + pub(crate) fn new_reasoning_block( full_reasoning_buffer: String, config: &Config, From 3be39812bde79c31b521a36d018ee01809023219 Mon Sep 17 00:00:00 2001 From: Jeremy Rose Date: Wed, 1 Oct 2025 15:52:15 -0700 Subject: [PATCH 2/3] fix test --- codex-rs/tui/src/history_cell.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index df60ef7f9b..d20fd331ec 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -32,7 +32,6 @@ use image::DynamicImage; use image::ImageReader; use mcp_types::EmbeddedResourceResource; use mcp_types::ResourceLink; -use pathdiff::diff_paths; use ratatui::prelude::*; use ratatui::style::Modifier; use ratatui::style::Style; From 91aafab3ad13298d7ad26c01d30ffa1bd992a641 Mon Sep 17 00:00:00 2001 From: Jeremy Rose Date: Thu, 2 Oct 2025 11:24:30 -0700 Subject: [PATCH 3/3] platform-independent test --- ...twidget__tests__local_image_attachment_history_snapshot.snap | 2 +- codex-rs/tui/src/chatwidget/tests.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__local_image_attachment_history_snapshot.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__local_image_attachment_history_snapshot.snap index da9c6d68fc..cf4c6943fd 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__local_image_attachment_history_snapshot.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__local_image_attachment_history_snapshot.snap @@ -3,4 +3,4 @@ source: tui/src/chatwidget/tests.rs expression: combined --- • Viewed Image - └ screenshots/example.png + └ example.png diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 19ac2c13ac..e175e180b9 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -798,7 +798,7 @@ fn custom_prompt_enter_empty_does_not_send() { #[test] fn view_image_tool_call_adds_history_cell() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(); - let image_path = chat.config.cwd.join("screenshots").join("example.png"); + let image_path = chat.config.cwd.join("example.png"); chat.handle_codex_event(Event { id: "sub-image".into(),