From c45ad3ef0e49a0999fb25c461fb6a7717a69757a Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Apr 2026 16:28:33 -0400 Subject: [PATCH 1/7] Bump Version --- cecli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cecli/__init__.py b/cecli/__init__.py index 5ff9ffdc8a9..1527c4f728f 100644 --- a/cecli/__init__.py +++ b/cecli/__init__.py @@ -1,6 +1,6 @@ from packaging import version -__version__ = "0.99.2.dev" +__version__ = "0.99.3.dev" safe_version = __version__ try: From b9e855bd52e210da5a73cb3ec683e58fb39f5ff9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Apr 2026 19:28:37 -0400 Subject: [PATCH 2/7] Simplify cost and token reporting in TUI --- cecli/coders/base_coder.py | 38 ++++++++++++++++++---------- cecli/tui/app.py | 8 ++++++ cecli/tui/styles.tcss | 1 + cecli/tui/widgets/input_container.py | 5 ++++ cecli/tui/widgets/output.py | 4 +-- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index 6e41c80a4e9..b0efe96fcf9 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -2233,10 +2233,11 @@ async def send_message(self, inp): self.multi_response_content = "" if self.show_pretty(): - spinner_text = ( - f"Waiting for {self.get_active_model().name} •" - f" ${self.format_cost(self.total_cost)} session" - ) + spinner_text = f"Waiting for {self.get_active_model().name}" + + if not self.tui: + spinner_text += f" • ${self.format_cost(self.total_cost)} session" + self.io.start_spinner(spinner_text) if self.stream: self.mdstream = True @@ -3541,13 +3542,18 @@ def calculate_and_show_tokens_and_cost(self, messages, completion=None): self.message_tokens_received += completion_tokens - tokens_report = f"Tokens: {format_tokens(self.message_tokens_sent)} sent" + # Build the new streamlined format + tokens_parts = [format_tokens(prompt_tokens)] - if cache_write_tokens: - tokens_report += f", {format_tokens(cache_write_tokens)} cache write" if cache_hit_tokens: - tokens_report += f", {format_tokens(cache_hit_tokens)} cache hit" - tokens_report += f", {format_tokens(self.message_tokens_received)} received." + tokens_parts.append(f"{format_tokens(cache_hit_tokens)}") + if cache_write_tokens: + tokens_parts.append(f"{format_tokens(cache_write_tokens)}") + + tokens_str = "/".join(tokens_parts) + + tokens_report = f"{tokens_str} ↑ {format_tokens(completion_tokens)} ↓" + tokens_report = self.token_profiler.add_to_usage_report( tokens_report, self.message_tokens_sent, self.message_tokens_received ) @@ -3570,9 +3576,12 @@ def calculate_and_show_tokens_and_cost(self, messages, completion=None): self.total_cost += cost self.message_cost += cost + total_combined_tokens = ( + self.total_tokens_sent + self.total_tokens_received + prompt_tokens + completion_tokens + ) cost_report = ( - f"Cost: ${self.format_cost(self.message_cost)} message," - f" ${self.format_cost(self.total_cost)} session." + f"${self.format_cost(self.message_cost)} • {format_tokens(total_combined_tokens)} ↑↓" + f" ${self.format_cost(self.total_cost)}" ) if cache_hit_tokens and cache_write_tokens: @@ -3630,8 +3639,11 @@ def show_usage_report(self): self.total_tokens_sent += self.message_tokens_sent self.total_tokens_received += self.message_tokens_received - self.io.tool_output(self.usage_report) - self.io.rule() + if self.tui and self.tui(): + self.tui().update_cost(self.usage_report.replace("\n", " ")) + else: + self.io.tool_output(self.usage_report) + self.io.rule() self.message_cost = 0.0 self.message_tokens_sent = 0 diff --git a/cecli/tui/app.py b/cecli/tui/app.py index e92feec1653..464726a61c1 100644 --- a/cecli/tui/app.py +++ b/cecli/tui/app.py @@ -393,6 +393,14 @@ def update_key_hints_left(self, text: str): except Exception: pass + def update_cost(self, cost_text: str): + """Update the cost display in the input container's border subtitle.""" + try: + container = self.query_one(InputContainer) + container.update_cost(cost_text) + except Exception: + pass + def _update_key_hints_for_commands(self, text: str, is_completion: bool = False): """ Update key hints left area with command description. diff --git a/cecli/tui/styles.tcss b/cecli/tui/styles.tcss index d49c83a27ad..4912577663b 100644 --- a/cecli/tui/styles.tcss +++ b/cecli/tui/styles.tcss @@ -42,6 +42,7 @@ Screen { margin: 0 1 0 1; border: round $accent 50%; border-title-align: right; + border-subtitle-align: right; background: $surface; } diff --git a/cecli/tui/widgets/input_container.py b/cecli/tui/widgets/input_container.py index 2e6b42ce2c4..574c9386d3d 100644 --- a/cecli/tui/widgets/input_container.py +++ b/cecli/tui/widgets/input_container.py @@ -17,3 +17,8 @@ def update_mode(self, mode: str): self.coder_mode = mode self.border_title = self.coder_mode self.refresh() + + def update_cost(self, cost_text: str): + """Update the cost display in the border subtitle.""" + self.border_subtitle = cost_text + self.refresh() diff --git a/cecli/tui/widgets/output.py b/cecli/tui/widgets/output.py index f0f286e8d3c..fbe5d50c3c9 100644 --- a/cecli/tui/widgets/output.py +++ b/cecli/tui/widgets/output.py @@ -86,9 +86,10 @@ def _wrap_text_with_prefix(self, text: str, prefix: str = "• ") -> str: async def stream_chunk(self, text: str): """Stream a chunk of markdown text.""" - if not text: + if not text or text == "(empty response)": return + self.set_last_write_type("assistant") # Check for cost updates in the text self._check_cost(text) # Add text to line buffer @@ -98,7 +99,6 @@ async def stream_chunk(self, text: str): while "\n" in self._line_buffer: line, self._line_buffer = self._line_buffer.split("\n", 1) if line.rstrip(): - self.set_last_write_type("assistant") # Format with prefix on first line, proper indentation on subsequent lines if self._first_line_of_response: wrapped_line = self._wrap_text_with_prefix(line.rstrip(), prefix="• ") From d40058bf7bce8602146dfafed6351d6fe81f939f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Apr 2026 19:32:36 -0400 Subject: [PATCH 3/7] Clear invocation cache on user input and let commands clear it as well --- cecli/coders/agent_coder.py | 2 ++ cecli/tools/command.py | 1 + 2 files changed, 3 insertions(+) diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 342a183eb1a..d0cbce1cdd1 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -25,6 +25,7 @@ from cecli.hooks import HookIntegration from cecli.llm import litellm from cecli.mcp import LocalServer, McpServerManager +from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.registry import ToolRegistry from cecli.utils import copy_tool_call, tool_call_to_dict @@ -1217,6 +1218,7 @@ async def preproc_user_input(self, inp): inp = await super().preproc_user_input(inp) inp = self.wrap_user_input(inp) + BaseTool.clear_invocation_cache() self.agent_finished = False self.turn_count = 0 return inp diff --git a/cecli/tools/command.py b/cecli/tools/command.py index e6cd80bb8e9..a8236779870 100644 --- a/cecli/tools/command.py +++ b/cecli/tools/command.py @@ -9,6 +9,7 @@ class Tool(BaseTool): NORM_NAME = "command" + TRACK_INVOCATIONS = False SCHEMA = { "type": "function", "function": { From b0fc3a9b4ffa2430e0a159b6a0fd7ff9d2048ccc Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Apr 2026 16:32:11 -0400 Subject: [PATCH 4/7] Multiple Changes: - Integrate cymbal library in the explore_symbols tool - Remove view_files_matching* tools - Make terminal output for tools more consistent with custom format_output() methods - Add automatic tool calls corrections for array-like tools --- cecli/args.py | 6 + cecli/coders/agent_coder.py | 11 +- cecli/coders/base_coder.py | 16 +- cecli/helpers/conversation/integration.py | 10 +- cecli/prompts/agent.yml | 14 +- cecli/prompts/hashline.yml | 3 +- cecli/tools/__init__.py | 8 +- cecli/tools/command.py | 54 +++- cecli/tools/context_manager.py | 33 +++ cecli/tools/explore_symbols.py | 287 ++++++++++++++++++++++ cecli/tools/grep.py | 51 ++++ cecli/tools/list_changes.py | 71 ------ cecli/tools/ls.py | 25 ++ cecli/tools/show_context.py | 38 ++- cecli/tools/utils/base_tool.py | 22 +- cecli/tools/view_files_matching.py | 138 ----------- cecli/tools/view_files_with_symbol.py | 117 --------- requirements.txt | 4 + requirements/common-constraints.txt | 2 + requirements/requirements.in | 1 + 20 files changed, 547 insertions(+), 364 deletions(-) create mode 100644 cecli/tools/explore_symbols.py delete mode 100644 cecli/tools/list_changes.py delete mode 100644 cecli/tools/view_files_matching.py delete mode 100644 cecli/tools/view_files_with_symbol.py diff --git a/cecli/args.py b/cecli/args.py index 395fbb17cd4..aa46b678ff9 100644 --- a/cecli/args.py +++ b/cecli/args.py @@ -593,6 +593,12 @@ def get_parser(default_config_files, git_root): help="Show token processing and generation speed in usage report (default: False)", default=False, ) + group.add_argument( + "--use-reminders", + action=argparse.BooleanOptionalAction, + default=True, + help="Enable/disable injecting reminder messages (default: True)", + ) group.add_argument( "--completion-menu-color", metavar="COLOR", diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index d0cbce1cdd1..37cb3ed920e 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -53,12 +53,9 @@ def __init__(self, *args, **kwargs): self.read_tools = { "command", "commandinteractive", - "viewfilesatglob", - "viewfilesmatching", + "exploresymbols", "ls", - "viewfileswithsymbol", "grep", - "listchanges", "showcontext", "thinking", "updatetodolist", @@ -329,7 +326,7 @@ async def _execute_local_tool_calls(self, tool_calls_list): {"role": "tool", "tool_call_id": tool_call.id, "content": result_message} ) - if self.auto_lint and used_write_tool and not self.edit_allowed: + if self.auto_lint and used_write_tool: edited = list(self.files_edited_by_tools) lint_errors = self.lint_edited(edited, show_output=False) self.lint_outcome = not lint_errors @@ -1161,7 +1158,7 @@ def _add_file_to_context(self, file_path, explicit=False): Parameters: - file_path: Path to the file to add - - explicit: Whether this was an explicit view command (vs. implicit through ViewFilesMatching) + - explicit: Whether this was an explicit view command (vs. implicit through other tools) """ abs_path = self.abs_root_path(file_path) rel_path = self.get_rel_fname(abs_path) @@ -1206,7 +1203,7 @@ async def check_for_file_mentions(self, content): Override parent's method to disable implicit file mention handling in agent mode. Files should only be added via explicit tool commands - (`View`, `ViewFilesAtGlob`, `ViewFilesMatching`, `ViewFilesWithSymbol`). + (`ContextManager`). """ pass diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index b0efe96fcf9..3d4f16b699d 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -921,10 +921,10 @@ def get_files_content(self, fnames=None): file_stub = RepoMap.get_file_stub(fname, self.io) # Add message about showing definitions instead of full content - self.io.tool_output( - f"⚠️ '{relative_fname}' is very large ({file_tokens} tokens). " - "Use /context-management to toggle truncation off if needed." - ) + # self.io.tool_output( + # f"⚠️ '{relative_fname}' is very large ({file_tokens} tokens). " + # "Use /context-management to toggle truncation off if needed." + # ) # Add a message in the content itself so the model knows it's truncated truncation_note = ( @@ -989,10 +989,10 @@ def get_read_only_files_content(self): file_stub = RepoMap.get_file_stub(fname, self.io) # Add message about showing definitions instead of full content - self.io.tool_output( - f"⚠️ '{relative_fname}' is very large ({file_tokens} tokens). " - "Use /context-management to toggle truncation off if needed." - ) + # self.io.tool_output( + # f"⚠️ '{relative_fname}' is very large ({file_tokens} tokens). " + # "Use /context-management to toggle truncation off if needed." + # ) # Add a message in the content itself so the model knows it's truncated truncation_note = ( diff --git a/cecli/helpers/conversation/integration.py b/cecli/helpers/conversation/integration.py index 7f602645d79..40b247a2a2e 100644 --- a/cecli/helpers/conversation/integration.py +++ b/cecli/helpers/conversation/integration.py @@ -85,7 +85,12 @@ def add_system_messages(self) -> None: ) # Add system reminder as a pre-prompt context block - if hasattr(coder.gpt_prompts, "system_reminder") and coder.gpt_prompts.system_reminder: + use_reminders = getattr(coder.args, "use_reminders", True) + if ( + use_reminders + and hasattr(coder.gpt_prompts, "system_reminder") + and coder.gpt_prompts.system_reminder + ): msg = dict( role="user", content=coder.fmt_system_prompt(coder.gpt_prompts.system_reminder), @@ -235,7 +240,8 @@ def add_file_list_reminder(self) -> None: for f in editable_rel_files: reminder_lines.append(f" - {f}") - if len(reminder_lines) > 1: # Only add reminder if there are files + use_reminders = getattr(coder.args, "use_reminders", True) + if use_reminders and len(reminder_lines) > 1: # Only add reminder if there are files reminder_lines.append("\n") reminder_content = "\n".join(reminder_lines) ConversationService.get_manager(coder).add_message( diff --git a/cecli/prompts/agent.yml b/cecli/prompts/agent.yml index fcadb95228d..dc15634561b 100644 --- a/cecli/prompts/agent.yml +++ b/cecli/prompts/agent.yml @@ -17,16 +17,16 @@ files_no_full_files_with_repo_map: | main_system: | ## Core Directives - **Act Proactively**: Autonomously use discovery and management tools (`ViewFilesAtGlob`, `ViewFilesMatching`, `Ls`, `ContextManager`) to fulfill the request. Chain tool calls across multiple turns for continuous exploration. + **Act Proactively**: Autonomously use discovery and management tools to fulfill the request. Chain tool calls across multiple turns for continuous exploration. **Be Decisive**: Trust your findings. Do not repeat identical searches or ask redundant questions. - **Be Efficient**: Batch tool calls when tools allow you to. Respect usage limits while maximizing the utility of each turn. + **Be Efficient**: You may use multiple tools each turn. Batch tool calls when tools allow you to. Respect usage limits while maximizing the utility of each turn. **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ### 1. FILE FORMAT - Files are provided in "hashline" format. Each line starts with a case-sensitive content hash followed by `::`. - Do not attempt to write these content hashes. They are automatically generated, maintained, and subject to change. + Files are provided in "hashline" format. Each line starts with a case-sensitive content hash followed by `::`. These are used as identifiers for editing tools. + Do not attempt to search for these content hashes. They are automatically generated, maintained, and subject to change. **Example File Format :** il9n::#!/usr/bin/env python3 @@ -39,8 +39,8 @@ main_system: | ## Core Workflow 1. **Plan**: Start by using `UpdateTodoList` to outline the task. - 2. **Explore**: Use `Grep` for broad searches. If results exceed 50 matches, refine your pattern immediately. Use discovery tools to add files as context. - 3. **Execute**: Use the appropriate editing tool. Mark files as editable with `ContextManager` when needed. Proactively use skills if they are available. + 2. **Explore**: Use discovery tools (`ExploreSymbols`, `Grep`, `Ls`) to research and gather understanding for you task. Use `ContextManager` to add files contents as context. + 3. **Execute**: Use the appropriate editing tool. Mark files as editable with `ContextManager` before attempting edits. Proactively use skills if they are available. 4. **Verify & Recover**: Review the diff output of every edit. If an edit fails or introduces linting errors, use `UndoChange` immediately. 5. **Finished**: Use the `Finished` tool only after verifying the solution. Briefly summarize the changes for the user. @@ -65,7 +65,7 @@ system_reminder: | ## Reminders **Strict Scope**: Stay on task. Do not alter functionality and syntax that is out of scope or pursue unrequested refactors. Do not attempt to modify large files in one shot. Work step by step. - **Context Hygiene**: Remove files and loaded skills from context using `ContextManager` or `RemoveSkill` once they are no longer needed to save tokens and prevent confusion. + **Context Hygiene**: Remove files and loaded skills from context using `ContextManager` and/or `RemoveSkill` once they are no longer needed to save tokens and prevent confusion. **Turn Management**: Tool calls trigger the next turn. Do not include tool calls in your final summary to the user. You must use `ShowContext` to view the relevant hashline range before each edit. **Sandbox**: Use `.cecli/temp` for all verification and temporary logic. **Novelty**: Do not repeat phrases in your responses to the user. You do not need to declare you understand the task. Simply proceed. Only give status updates when you have new information. diff --git a/cecli/prompts/hashline.yml b/cecli/prompts/hashline.yml index 0acc0eb3f52..fd300b1acf5 100644 --- a/cecli/prompts/hashline.yml +++ b/cecli/prompts/hashline.yml @@ -6,7 +6,8 @@ main_system: | Act as an expert software developer. Plan carefully, explain your logic briefly, and execute via LOCATE/CONTENTS blocks. ### 1. FILE FORMAT - Files are provided in "hashline" format. Each line starts with a case-sensitive content hash followed by `::`. + Files are provided in "hashline" format. Each line starts with a case-sensitive content hash followed by `::`. + These hashes are used as identifiers for lines when editing. **Example File Format :** il9n::#!/usr/bin/env python3 diff --git a/cecli/tools/__init__.py b/cecli/tools/__init__.py index e08adc59209..0327bf24a9a 100644 --- a/cecli/tools/__init__.py +++ b/cecli/tools/__init__.py @@ -7,6 +7,7 @@ command_interactive, context_manager, delete_text, + explore_symbols, finished, git_branch, git_diff, @@ -16,7 +17,6 @@ git_status, grep, insert_text, - list_changes, load_skill, ls, remove_skill, @@ -25,8 +25,6 @@ thinking, undo_change, update_todo_list, - view_files_matching, - view_files_with_symbol, ) # List of all available tool modules for dynamic discovery @@ -35,6 +33,7 @@ command_interactive, context_manager, delete_text, + explore_symbols, finished, git_branch, git_diff, @@ -44,7 +43,6 @@ git_status, grep, insert_text, - list_changes, load_skill, ls, remove_skill, @@ -53,6 +51,4 @@ thinking, undo_change, update_todo_list, - view_files_matching, - view_files_with_symbol, ] diff --git a/cecli/tools/command.py b/cecli/tools/command.py index a8236779870..3b541eb8c8b 100644 --- a/cecli/tools/command.py +++ b/cecli/tools/command.py @@ -1,10 +1,12 @@ # Import necessary functions +import json import os import platform from cecli.helpers.background_commands import BackgroundCommandManager from cecli.run_cmd import run_cmd_subprocess from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.output import color_markers, tool_footer, tool_header class Tool(BaseTool): @@ -184,7 +186,7 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Fal from cecli.helpers.background_commands import CircularBuffer - coder.io.tool_output(f"⚙️ Executing shell command with {timeout}s timeout: {command_string}") + coder.io.tool_output(f"⚙️ Executing shell command with {timeout}s timeout.") shell = os.environ.get("SHELL", "/bin/sh") @@ -285,7 +287,7 @@ async def _execute_foreground(cls, coder, command_string): tui = coder.tui() should_print = False - coder.io.tool_output(f"⚙️ Executing shell command: {command_string}") + coder.io.tool_output("⚙️ Executing shell command.") # Use run_cmd_subprocess for non-interactive execution exit_status, combined_output = run_cmd_subprocess( @@ -329,8 +331,52 @@ async def _stop_background_command(cls, coder, command_key): else: return output # Error message from manager - @classmethod async def _handle_errors(cls, coder, command_string, e): """Handle errors during command execution.""" - coder.io.tool_error(f"Error executing shell command '{command_string}': {str(e)}") + coder.io.tool_error(f"Error executing shell command: {str(e)}") return f"Error executing command: {str(e)}" + + @classmethod + def format_output(cls, coder, mcp_server, tool_response): + """Format output for Command tool.""" + color_start, color_end = color_markers(coder) + + try: + params = json.loads(tool_response.function.arguments) + except json.JSONDecodeError: + coder.io.tool_error("Invalid Tool JSON") + return + + # Output header + tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) + + command = params.get("command", "") + background = params.get("background", False) + stop = params.get("stop", False) + stdin = params.get("stdin") + pty = params.get("pty", False) + + coder.io.tool_output("") + + # Show additional parameters if they are not default + extras = [] + if background: + extras.append("background=True") + if stop: + extras.append("stop=True") + if pty: + extras.append("pty=True") + + if extras: + coder.io.tool_output(f"{color_start}Options:{color_end} {', '.join(extras)}") + + if stdin: + coder.io.tool_output(f"{color_start}Stdin:{color_end}") + coder.io.tool_output(stdin) + + coder.io.tool_output(f"{color_start}Command:{color_end}") + coder.io.tool_output(command) + coder.io.tool_output("") + + # Output footer + tool_footer(coder=coder, tool_response=tool_response) diff --git a/cecli/tools/context_manager.py b/cecli/tools/context_manager.py index b5cf402781c..6dfa12c3a69 100644 --- a/cecli/tools/context_manager.py +++ b/cecli/tools/context_manager.py @@ -1,8 +1,10 @@ +import json import os import time from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError, parse_arg_as_list +from cecli.tools.utils.output import color_markers, tool_footer, tool_header class Tool(BaseTool): @@ -75,6 +77,7 @@ def execute(cls, coder, remove=None, editable=None, view=None, create=None, **kw if not remove_files and not editable_files and not view_files and not create_files: raise ToolError("You must specify at least one of: remove, editable, view, or create") + coder.io.tool_output("⚙️ Modifying Context.") messages = [] for f in create_files: @@ -94,6 +97,36 @@ def execute(cls, coder, remove=None, editable=None, view=None, create=None, **kw return "\n".join(messages) + @classmethod + def format_output(cls, coder, mcp_server, tool_response): + """Format output for ContextManager tool.""" + color_start, color_end = color_markers(coder) + + try: + params = json.loads(tool_response.function.arguments) + except json.JSONDecodeError: + coder.io.tool_error("Invalid Tool JSON") + return + + tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) + + # Define action display names + action_names = { + "create": "create", + "remove": "remove", + "view": "view", + "editable": "editable", + } + + # Output each action with comma-separated file list + for action_key, display_name in action_names.items(): + files = parse_arg_as_list(params.get(action_key)) + if files: + file_list = ", ".join(files) + coder.io.tool_output(f"{color_start}{display_name}:{color_end} {file_list}") + + tool_footer(coder=coder, tool_response=tool_response) + @staticmethod def _remove(coder, file_path): """Remove a file from the coder's context.""" diff --git a/cecli/tools/explore_symbols.py b/cecli/tools/explore_symbols.py new file mode 100644 index 00000000000..8fa061486db --- /dev/null +++ b/cecli/tools/explore_symbols.py @@ -0,0 +1,287 @@ +import json +import os + +from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.helpers import ToolError +from cecli.tools.utils.output import color_markers, tool_footer, tool_header + +try: + import cymbal + + CYMBAL_AVAILABLE = True +except ImportError: + CYMBAL_AVAILABLE = False + + +class Tool(BaseTool): + NORM_NAME = "exploresymbols" + SCHEMA = { + "type": "function", + "function": { + "name": "ExploreSymbols", + "description": ( + "Search, investigate, and find references to symbols using the Cymbal code indexing" + " library. This is the preferred tool for analyzing code structure." + ), + "parameters": { + "type": "object", + "properties": { + "queries": { + "type": "array", + "items": { + "type": "object", + "properties": { + "symbol": { + "type": "string", + "description": ( + "The symbol name to search for, investigate, or find" + " references to. This should be a single symbol" + " (e.g. method, function, or class name)." + ), + }, + "action": { + "type": "string", + "enum": ["search", "investigate", "find_references"], + "description": ( + "Action to perform: 'search', 'investigate', or" + " 'find_references'.\n\nNote: For the 'investigate' action," + " you can use filename-based disambiguation:\n - '{file" + " name}:{symbol}' to specify a symbol in a particular file" + " (e.g., 'config.go:Config')\n" + ), + }, + "limit": { + "type": "integer", + "description": ( + "Maximum number of results to return. Defaults to 15." + ), + "default": 15, + }, + }, + "required": ["symbol", "action"], + }, + "description": ( + "Array of exploration queries. Maximum of 5 queries at a time." + ), + } + }, + "required": ["queries"], + }, + }, + } + + @classmethod + def execute(cls, coder, queries, **kwargs): + """ + Search, investigate, or find references to symbols using the Cymbal code indexing library. + + Args: + coder: The Coder instance. + queries (list): Array of exploration queries {symbol, action, limit}. + + Returns: + str: Formatted results from the Cymbal operations. + """ + try: + # Check if cymbal is available + if not CYMBAL_AVAILABLE: + coder.io.tool_error( + "Cymbal library is not available. Please install it with: pip install py-cymbal" + ) + return "Error: Cymbal library is not available" + + # Initialize Cymbal and index if necessary + c = cymbal.Cymbal() + repo_path = getattr(coder, "root", ".") + + try: + # If we can't get a db_path or it doesn't exist, index it. + if not os.path.exists(c.db_path): + c.index(repo_path) + except Exception as e: + error_msg = f"Failed to index repository: {str(e)}" + coder.io.tool_error(error_msg) + return f"Error: {error_msg}" + + all_results = [] + all_failed_queries = [] + total_successful_queries = 0 + + for query in queries: + symbol = query.get("symbol") + action = query.get("action") + limit = query.get("limit", 15) + + try: + if action == "search": + results = c.search(symbol, limit=limit) + all_results.append(cls._format_search_results(results, symbol)) + elif action == "investigate": + # Parse symbol for file hint format: {file}:{symbol} or {package}.{symbol} + symbol_name = symbol + file_hint = "" + + # Check for file:symbol format (e.g., "config.go:Config") + if ":" in symbol: + parts = symbol.split(":", 1) + if len(parts) == 2: + file_hint = parts[0] + symbol_name = parts[1] + + try: + investigation = c.investigate(symbol_name, file_hint) + all_results.append( + cls._format_investigation_results(investigation, symbol) + ) + except Exception as e: + if "multiple matches" in str(e).lower(): + # Fallback to search to show locations + results = c.search(symbol_name, limit=10) + locations = "\n".join( + [f"- {r['file']}:{r['start_line']}" for r in results] + ) + msg = ( + f"Error: Multiple matches found for '{symbol}'.\nPlease use a" + " more specific name or check the locations" + f" below:\n{locations}" + ) + all_results.append(msg) + else: + raise e + elif action == "find_references": + references = c.find_references(symbol, limit=limit) + all_results.append(cls._format_reference_results(references, symbol)) + else: + all_failed_queries.append( + f"Error for symbol '{symbol}': Unknown action '{action}'" + ) + continue + + total_successful_queries += 1 + except Exception as e: + all_failed_queries.append(f"Error for symbol '{symbol}': {str(e)}") + + if total_successful_queries == 0: + error_msg = "No queries were successfully executed:\n" + "\n".join( + all_failed_queries + ) + raise ToolError(error_msg) + + if all_failed_queries: + for failed_msg in all_failed_queries: + coder.io.tool_error(failed_msg) + else: + coder.io.tool_output("✅ All queries successful.") + + return "\n\n" + "=" * 40 + "\n\n".join(all_results) + + except Exception as e: + coder.io.tool_error(f"Error in ExploreSymbols: {str(e)}") + return f"Error: {str(e)}" + + @classmethod + def _format_search_results(cls, results, symbol): + """Format search results for display.""" + if not results: + return f"No symbols found matching '{symbol}'" + + formatted = [f"Found {len(results)} symbols matching '{symbol}':"] + for i, result in enumerate(results[:15], 1): + # Extract symbol attributes (adjust based on actual cymbal result structure) + # Extract symbol attributes from dictionary + name = result.get("name", "Unknown") + kind = result.get("kind", "unknown") + file = result.get("file", "Unknown") + start_line = result.get("start_line", 0) + + formatted.append(f"{i}. {name} ({kind}) at {file}:{start_line}") + + if len(results) > 15: + formatted.append(f"... and {len(results) - 15} more results") + + return "\n".join(formatted) + + @classmethod + def _format_investigation_results(cls, investigation, symbol): + """Format investigation results for display.""" + if not investigation: + return f"No information found for symbol '{symbol}'" + + formatted = [f"Investigation of symbol '{symbol}':"] + + # Extract definition information + definition = investigation.get("symbol") + if definition: + def_name = definition.get("name", symbol) + def_file = definition.get("file", "Unknown") + def_line = definition.get("start_line", 0) + def_kind = definition.get("kind", "unknown") + formatted.append(f"Definition: {def_name} ({def_kind}) at {def_file}:{def_line}") + + references = investigation.get("refs", []) + ref_count = len(references) if references else 0 + formatted.append(f"\nReferences found: {ref_count}") + + if references and ref_count > 0: + formatted.append("Top references:") + for i, ref in enumerate(references[:10], 1): + ref_file = ref.get("file", "Unknown") + ref_line = ref.get("line", 0) + formatted.append(f"{i}. {ref_file}:{ref_line}") + + if ref_count > 10: + formatted.append(f"... and {ref_count - 10} more references") + + return "\n".join(formatted) + + @classmethod + def _format_reference_results(cls, references, symbol): + """Format reference finding results for display.""" + if not references: + return f"No references found for symbol '{symbol}'" + + formatted = [f"Found {len(references)} references to '{symbol}':"] + for i, ref in enumerate(references[:15], 1): + # Extract reference attributes from dictionary + file = ref.get("file", "Unknown") + line = ref.get("line", 0) + + formatted.append(f"{i}. {file}:{line}") + + if len(references) > 15: + formatted.append(f"... and {len(references) - 15} more references") + + return "\n".join(formatted) + + @classmethod + def format_output(cls, coder, mcp_server, tool_response): + """Format output for ExploreSymbols tool.""" + color_start, color_end = color_markers(coder) + + try: + params = json.loads(tool_response.function.arguments) + except json.JSONDecodeError: + coder.io.tool_error("Invalid Tool JSON") + return + + # Output header + tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) + + # Output each query with the requested format + queries = params.get("queries", []) + if queries: + coder.io.tool_output("") + for i, query in enumerate(queries): + symbol = query.get("symbol", "") + action = query.get("action", "") + limit = query.get("limit", 15) + + # Format as "{action}: • {symbol} • {limit}" with action wrapped in color markers + # Capitalize action and replace underscores with spaces + formatted_action = action + formatted_query = f"{color_start}{formatted_action}:{color_end} {symbol} • {limit}" + coder.io.tool_output(formatted_query) + coder.io.tool_output("") + + # Output footer + tool_footer(coder=coder, tool_response=tool_response) diff --git a/cecli/tools/grep.py b/cecli/tools/grep.py index b04f1b8090e..68ca5a103b2 100644 --- a/cecli/tools/grep.py +++ b/cecli/tools/grep.py @@ -1,3 +1,4 @@ +import json import shutil from pathlib import Path @@ -5,6 +6,7 @@ from cecli.run_cmd import run_cmd_subprocess from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.output import color_markers, tool_footer, tool_header class Tool(BaseTool): @@ -226,3 +228,52 @@ def execute( coder.io.tool_output(ui_message) return final_message + + @classmethod + def format_output(cls, coder, mcp_server, tool_response): + """Format output for Grep tool.""" + color_start, color_end = color_markers(coder) + + try: + params = json.loads(tool_response.function.arguments) + except json.JSONDecodeError: + coder.io.tool_error("Invalid Tool JSON") + return + + tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) + + # Output each search operation with the requested format + searches = params.get("searches", []) + if searches: + coder.io.tool_output("") + for i, search_op in enumerate(searches): + pattern = search_op.get("pattern", "") + file_pattern = search_op.get("file_pattern", "*") + directory = search_op.get("directory", ".") + use_regex = search_op.get("use_regex", False) + case_insensitive = search_op.get("case_insensitive", False) + context_before = search_op.get("context_before", 5) + context_after = search_op.get("context_after", 5) + + # Format as "search: • pattern • file_pattern • directory • options" + formatted_query = ( + f"{color_start}search_{i + 1}:{color_end} {pattern} • {file_pattern} •" + f" {directory}" + ) + + # Add options if they differ from defaults + options = [] + if use_regex: + options.append("regex") + if case_insensitive: + options.append("case-insensitive") + if context_before != 5 or context_after != 5: + options.append(f"context:{context_before}/{context_after}") + + if options: + formatted_query += f" • {' '.join(options)}" + + coder.io.tool_output(formatted_query) + coder.io.tool_output("") + + tool_footer(coder=coder, tool_response=tool_response) diff --git a/cecli/tools/list_changes.py b/cecli/tools/list_changes.py deleted file mode 100644 index 3e8e4331684..00000000000 --- a/cecli/tools/list_changes.py +++ /dev/null @@ -1,71 +0,0 @@ -import traceback -from datetime import datetime - -from cecli.tools.utils.base_tool import BaseTool - - -class Tool(BaseTool): - NORM_NAME = "listchanges" - SCHEMA = { - "type": "function", - "function": { - "name": "ListChanges", - "description": "List recent changes made.", - "parameters": { - "type": "object", - "properties": { - "file_path": {"type": "string"}, - "limit": {"type": "integer", "default": 10}, - }, - }, - }, - } - - @classmethod - def execute(cls, coder, file_path=None, limit=10, **kwargs): - """ - List recent changes made to files. - - Parameters: - - coder: The Coder instance - - file_path: Optional path to filter changes by file - - limit: Maximum number of changes to list - - Returns a formatted list of changes. - """ - try: - # If file_path is specified, get the absolute path - rel_file_path = None - if file_path: - abs_path = coder.abs_root_path(file_path) - rel_file_path = coder.get_rel_fname(abs_path) - - # Get the list of changes - changes = coder.change_tracker.list_changes(rel_file_path, limit) - - if not changes: - if file_path: - return f"No changes found for file '{file_path}'" - else: - return "No changes have been made yet" - - # Format the changes into a readable list - result = "Recent changes:\n" - for i, change in enumerate(changes): - change_time = datetime.fromtimestamp(change["timestamp"]).strftime("%H:%M:%S") - change_type = change["type"] - file_path = change["file_path"] - change_id = change["id"] - - result += ( - f"{i + 1}. [{change_id}] {change_time} - {change_type.upper()} on {file_path}\n" - ) - - coder.io.tool_output(result) # Also print to console for user - return result - - except Exception as e: - coder.io.tool_error( - f"Error in ListChanges: {str(e)}\n{traceback.format_exc()}" - ) # Add traceback - return f"Error: {str(e)}" diff --git a/cecli/tools/ls.py b/cecli/tools/ls.py index 200816db435..c6d19761fe3 100644 --- a/cecli/tools/ls.py +++ b/cecli/tools/ls.py @@ -1,6 +1,8 @@ +import json import os from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.output import color_markers, tool_footer, tool_header class Tool(BaseTool): @@ -75,3 +77,26 @@ def execute(cls, coder, dir_path=None, directory=None, **kwargs): except Exception as e: coder.io.tool_error(f"Error in ls: {str(e)}") return f"Error: {str(e)}" + + @classmethod + def format_output(cls, coder, mcp_server, tool_response): + """Format output for Ls tool.""" + color_start, color_end = color_markers(coder) + + try: + params = json.loads(tool_response.function.arguments) + except json.JSONDecodeError: + coder.io.tool_error("Invalid Tool JSON") + return + + tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) + + # Output the directory parameter with the requested format + directory = params.get("directory", "") + if directory: + # Format as "ls: • directory" + formatted_query = f"{color_start}directory:{color_end} {directory}" + coder.io.tool_output(formatted_query) + coder.io.tool_output("") + + tool_footer(coder=coder, tool_response=tool_response) diff --git a/cecli/tools/show_context.py b/cecli/tools/show_context.py index f1f55e1af54..82516936c58 100644 --- a/cecli/tools/show_context.py +++ b/cecli/tools/show_context.py @@ -1,3 +1,4 @@ +import json import os from cecli.helpers.hashline import hashline, strip_hashline @@ -8,6 +9,7 @@ is_provided, resolve_paths, ) +from cecli.tools.utils.output import color_markers, tool_footer, tool_header class Tool(BaseTool): @@ -88,7 +90,7 @@ def execute(cls, coder, show, **kwargs): try: # 1. Validate show parameter if not isinstance(show, list): - raise ToolError("show parameter must be an array") + show = [show] if isinstance(show, dict) else show if len(show) == 0: raise ToolError("show array cannot be empty") @@ -279,7 +281,7 @@ def execute(cls, coder, show, **kwargs): "Do not call ShowContext again until you edit the file." ) else: - coder.io.tool_output(f"Successfully retrieved context for {len(show)} file(s)") + coder.io.tool_output(f"✅ Successfully retrieved context for {len(show)} file(s)") return f"Successfully retrieved most recent contents for {len(show)} file(s)" except ToolError as e: @@ -289,6 +291,38 @@ def execute(cls, coder, show, **kwargs): # Handle unexpected errors during processing return handle_tool_error(coder, tool_name, e) + @classmethod + def format_output(cls, coder, mcp_server, tool_response): + """Format output for ShowContext tool.""" + color_start, color_end = color_markers(coder) + + try: + params = json.loads(tool_response.function.arguments) + except json.JSONDecodeError: + coder.io.tool_error("Invalid Tool JSON") + return + + tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) + + show_ops = params.get("show", []) + if show_ops: + coder.io.tool_output("") + for i, show_op in enumerate(show_ops): + file_path = show_op.get("file_path", "") + start_text = strip_hashline(show_op.get("start_text", "")).strip() + end_text = strip_hashline(show_op.get("end_text", "")).strip() + padding = show_op.get("padding", 5) + + # Format as "show: • file_path • start_text • end_text • padding" + formatted_query = ( + f"{color_start}range_{i + 1}:{color_end} {file_path} • {start_text} •" + f" {end_text} • {padding}" + ) + coder.io.tool_output(formatted_query) + coder.io.tool_output("") + + tool_footer(coder=coder, tool_response=tool_response) + @classmethod def on_duplicate_request(cls, coder, **kwargs): coder.edit_allowed = True diff --git a/cecli/tools/utils/base_tool.py b/cecli/tools/utils/base_tool.py index ddb2a8bcebc..ce981bbfd25 100644 --- a/cecli/tools/utils/base_tool.py +++ b/cecli/tools/utils/base_tool.py @@ -50,6 +50,26 @@ def process_response(cls, coder, params): if "parameters" in function_schema and "required" in function_schema["parameters"]: required_params = function_schema["parameters"]["required"] + properties = function_schema["parameters"].get("properties", {}) + + # Auto-correction: If a required parameter is missing but it's an array, + # and the current params look like a single item of that array, wrap it. + if len(required_params) == 1: + missing_param = required_params[0] + if missing_param not in params and params: + param_schema = properties.get(missing_param, {}) + if param_schema.get("type") == "array": + params = {missing_param: [params]} + + # Auto-correction: If a required parameter is present but is a dict instead of an array + for param_name in required_params: + if param_name in params: + param_schema = properties.get(param_name, {}) + if param_schema.get("type") == "array" and isinstance( + params[param_name], dict + ): + params[param_name] = [params[param_name]] + missing_params = [param for param in required_params if param not in params] if missing_params: tool_name = function_schema.get("name", "Unknown Tool") @@ -79,7 +99,7 @@ def process_response(cls, coder, params): if prev_params_tuple == current_params_tuple: error_msg = ( f"Tool '{tool_name}' has been called with identical parameters recently. " - "This request is denied to prevent repeated operations." + "This request is denied." ) cls.on_duplicate_request(coder, **params) return handle_tool_error( diff --git a/cecli/tools/view_files_matching.py b/cecli/tools/view_files_matching.py deleted file mode 100644 index b148ab584ec..00000000000 --- a/cecli/tools/view_files_matching.py +++ /dev/null @@ -1,138 +0,0 @@ -import fnmatch -import re - -from cecli.tools.utils.base_tool import BaseTool - - -class Tool(BaseTool): - NORM_NAME = "viewfilesmatching" - SCHEMA = { - "type": "function", - "function": { - "name": "ViewFilesMatching", - "description": "View files containing a specific pattern.", - "parameters": { - "type": "object", - "properties": { - "pattern": { - "type": "string", - "description": "The pattern to search for in file contents.", - }, - "file_pattern": { - "type": "string", - "description": ( - "An optional glob pattern to filter which files are searched." - ), - }, - "regex": { - "type": "boolean", - "description": ( - "Whether the pattern is a regular expression. Defaults to False." - ), - }, - }, - "required": ["pattern"], - }, - }, - } - - @classmethod - def execute(cls, coder, pattern, file_pattern=None, regex=False, **kwargs): - """ - Search for pattern (literal string or regex) in files and return matching files as text. - - Args: - coder: The Coder instance. - pattern (str): The pattern to search for. - Treated as a literal string by default. - file_pattern (str, optional): Glob pattern to filter which files are searched. - Defaults to None (search all files). - regex (bool, optional): If True, treat pattern as a regular expression. - Defaults to False. - - This tool lets the LLM search for content within files, mimicking - how a developer would use grep or regex search to find relevant code. - """ - try: - # Get list of files to search - if file_pattern: - # Use glob pattern to filter files - all_files = coder.get_all_relative_files() - files_to_search = [] - for file in all_files: - if fnmatch.fnmatch(file, file_pattern): - files_to_search.append(file) - - if not files_to_search: - return f"No files matching '{file_pattern}' to search for pattern '{pattern}'" - else: - # Search all files if no pattern provided - files_to_search = coder.get_all_relative_files() - - # Search for pattern in files - matches = {} - num_matches = 0 - inspecific_search_flag = False - - for file in files_to_search: - abs_path = coder.abs_root_path(file) - - if num_matches >= 25: - inspecific_search_flag = True - - try: - if coder.repo.ignored_file(abs_path): - continue - - with open(abs_path, "r", encoding="utf-8") as f: - content = f.read() - match_count = 0 - if regex: - try: - matches_found = re.findall(pattern, content) - match_count = len(matches_found) - except re.error as e: - # Handle invalid regex patterns gracefully - coder.io.tool_error(f"Invalid regex pattern '{pattern}': {e}") - # Skip this file for this search if regex is invalid - continue - else: - # Exact string matching - match_count = content.count(pattern) - - if match_count > 0: - matches[file] = match_count - num_matches += 1 - except Exception: - # Skip files that can't be read (binary, etc.) - pass - - # Return formatted text instead of adding to context - if matches: - # Sort by number of matches (most matches first) - sorted_matches = sorted(matches.items(), key=lambda x: x[1], reverse=True) - match_list = [f"{file} ({count} matches)" for file, count in sorted_matches] - - if len(matches) > 10: - result = ( - f"Found '{pattern}' in {len(matches)} files:" - f" {', '.join(match_list[:10])} and {len(matches) - 10} more" - "\nTry more specific search terms going forward" - if inspecific_search_flag - else "" - ) - coder.io.tool_output(f"🔍 Found '{pattern}' in {len(matches)} files") - else: - result = f"Found '{pattern}' in {len(matches)} files: {', '.join(match_list)}" - coder.io.tool_output( - f"🔍 Found '{pattern}' in:" - f" {', '.join(match_list[:5])}{' and more' if len(matches) > 5 else ''}" - ) - - return result - else: - coder.io.tool_output(f"⚠️ Pattern '{pattern}' not found in any files") - return "Pattern not found in any files" - except Exception as e: - coder.io.tool_error(f"Error in ViewFilesMatching: {str(e)}") - return f"Error: {str(e)}" diff --git a/cecli/tools/view_files_with_symbol.py b/cecli/tools/view_files_with_symbol.py deleted file mode 100644 index f32175bbc3f..00000000000 --- a/cecli/tools/view_files_with_symbol.py +++ /dev/null @@ -1,117 +0,0 @@ -from cecli.tools.utils.base_tool import BaseTool - - -class Tool(BaseTool): - NORM_NAME = "viewfileswithsymbol" - SCHEMA = { - "type": "function", - "function": { - "name": "ViewFilesWithSymbol", - "description": "View files that contain a specific symbol (e.g., class, function).", - "parameters": { - "type": "object", - "properties": { - "symbol": { - "type": "string", - "description": "The symbol to search for.", - }, - }, - "required": ["symbol"], - }, - }, - } - - @classmethod - def execute(cls, coder, symbol, **kwargs): - """ - Find files containing a symbol using RepoMap and return them as text. - Checks files already in context first. - """ - if not coder.repo_map: - coder.io.tool_output("⚠️ Repo map not available, cannot use ViewFilesWithSymbol tool.") - return "Repo map not available" - - if not symbol: - return "Error: Missing 'symbol' parameter for ViewFilesWithSymbol" - - # 1. Check files already in context - files_in_context = list(coder.abs_fnames) + list(coder.abs_read_only_fnames) - found_in_context = [] - for abs_fname in files_in_context: - rel_fname = coder.get_rel_fname(abs_fname) - try: - # Use get_tags for consistency with RepoMap usage elsewhere for now. - tags = coder.repo_map.get_tags(abs_fname, rel_fname) - for tag in tags: - if tag.name == symbol: - found_in_context.append(rel_fname) - break # Found in this file, move to next - except Exception as e: - coder.io.tool_warning( - f"Could not get symbols for {rel_fname} while checking context: {e}" - ) - - if found_in_context: - # Symbol found in already loaded files. Report this and stop. - file_list = ", ".join(sorted(list(set(found_in_context)))) - coder.io.tool_output(f"Symbol '{symbol}' found in already loaded file(s): {file_list}") - return f"Symbol '{symbol}' found in already loaded file(s): {file_list}" - - # 2. If not found in context, search the repository using RepoMap - coder.io.tool_output(f"🔎 Searching for symbol '{symbol}' in repository...") - try: - found_files = set() - current_context_files = coder.abs_fnames | coder.abs_read_only_fnames - files_to_search = set(coder.get_all_abs_files()) - current_context_files - - rel_fname_to_abs = {} - all_tags = [] - - for fname in files_to_search: - rel_fname = coder.get_rel_fname(fname) - rel_fname_to_abs[rel_fname] = fname - try: - tags = coder.repo_map.get_tags(fname, rel_fname) - all_tags.extend(tags) - except Exception as e: - coder.io.tool_warning(f"Could not get tags for {rel_fname}: {e}") - - # Find matching symbols - for tag in all_tags: - if tag.name == symbol: - # Use absolute path directly if available, otherwise resolve from relative path - abs_fname = rel_fname_to_abs.get(tag.rel_fname) or coder.abs_root_path( - tag.fname - ) - if ( - abs_fname in files_to_search - ): # Ensure we only add files we intended to search - found_files.add(coder.get_rel_fname(abs_fname)) - - # Return formatted text instead of adding to context - if found_files: - found_files_list = sorted(list(found_files)) - if len(found_files) > 10: - result = ( - f"Found symbol '{symbol}' in {len(found_files)} files:" - f" {', '.join(found_files_list[:10])} and {len(found_files) - 10} more" - ) - coder.io.tool_output(f"🔎 Found '{symbol}' in {len(found_files)} files") - else: - result = ( - f"Found symbol '{symbol}' in {len(found_files)} files:" - f" {', '.join(found_files_list)}" - ) - coder.io.tool_output( - f"🔎 Found '{symbol}' in files:" - f" {', '.join(found_files_list[:5])}{' and more' if len(found_files) > 5 else ''}" - ) - - return result - else: - coder.io.tool_output(f"⚠️ Symbol '{symbol}' not found in searchable files") - return f"Symbol '{symbol}' not found in searchable files" - - except Exception as e: - coder.io.tool_error(f"Error in ViewFilesWithSymbol: {str(e)}") - return f"Error: {str(e)}" diff --git a/requirements.txt b/requirements.txt index 4b23fbf7c98..954d9ec1a6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -274,6 +274,10 @@ ptyprocess==0.7.0 # via # -c requirements/common-constraints.txt # pexpect +py-cymbal==0.1.5 + # via + # -c requirements/common-constraints.txt + # -r requirements/requirements.in pycodestyle==2.14.0 # via # -c requirements/common-constraints.txt diff --git a/requirements/common-constraints.txt b/requirements/common-constraints.txt index b86916c6337..6e526a7c092 100644 --- a/requirements/common-constraints.txt +++ b/requirements/common-constraints.txt @@ -345,6 +345,8 @@ psutil==7.1.3 # via -r requirements/requirements.in ptyprocess==0.7.0 # via pexpect +py-cymbal==0.1.5 + # via -r requirements/requirements.in pycodestyle==2.14.0 # via flake8 pycparser==2.23 diff --git a/requirements/requirements.in b/requirements/requirements.in index 8c577e88be5..edd319f89b5 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -32,6 +32,7 @@ textual>=6.0.0 tomlkit>=0.14.0 truststore xxhash>=3.6.0 +py-cymbal>=0.1.5 # Replaced networkx with rustworkx for better performance in repomap rustworkx>=0.15.0 From b500640ef9884fbe3748931239c3892b9e9ea967 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Apr 2026 17:57:40 -0400 Subject: [PATCH 5/7] Strip hashlines when the model adds them for text operations --- cecli/tools/insert_text.py | 4 ++-- cecli/tools/replace_text.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cecli/tools/insert_text.py b/cecli/tools/insert_text.py index 615eb156ea8..e12e0c15f76 100644 --- a/cecli/tools/insert_text.py +++ b/cecli/tools/insert_text.py @@ -1,4 +1,4 @@ -from cecli.helpers.hashline import apply_hashline_operation +from cecli.helpers.hashline import apply_hashline_operation, strip_hashline from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ( ToolError, @@ -84,7 +84,7 @@ def execute( start_line_hash=start_line, end_line_hash=start_line, # For insert, end_line is same as start_line operation="insert", - text=content, + text=strip_hashline(content), ) except Exception as e: coder.edit_allowed = True diff --git a/cecli/tools/replace_text.py b/cecli/tools/replace_text.py index 30566281b21..5d3f5ed14de 100644 --- a/cecli/tools/replace_text.py +++ b/cecli/tools/replace_text.py @@ -129,7 +129,7 @@ def execute( for edit_index, edit in file_edits: try: - edit_replace_text = edit.get("replace_text") + edit_replace_text = strip_hashline(edit.get("replace_text")) edit_start_line = edit.get("start_line") edit_end_line = edit.get("end_line") @@ -310,7 +310,7 @@ def format_output(cls, coder, mcp_server, tool_response): for edit_index, edit in file_edits: # Show diff for this edit using hashline diff - replace_text = edit.get("replace_text", "") + replace_text = strip_hashline(edit.get("replace_text", "")) start_line = edit.get("start_line") end_line = edit.get("end_line") From 2991a018419714bf37ca906717628d953dd7329f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Apr 2026 22:17:11 -0400 Subject: [PATCH 6/7] Add observational memory steps to compaction strategy --- cecli/coders/base_coder.py | 189 ++++++++---------- cecli/commands/clear.py | 1 + cecli/commands/reset.py | 1 + cecli/helpers/observations/__init__.py | 0 cecli/helpers/observations/manager.py | 98 +++++++++ cecli/prompts/base.yml | 44 ++-- pytest.ini | 1 + .../observations/test_observation_manager.py | 185 +++++++++++++++++ 8 files changed, 396 insertions(+), 123 deletions(-) create mode 100644 cecli/helpers/observations/__init__.py create mode 100644 cecli/helpers/observations/manager.py create mode 100644 tests/helpers/observations/test_observation_manager.py diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index 3d4f16b699d..1c221547ee9 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -42,6 +42,7 @@ from cecli.exceptions import LiteLLMExceptions from cecli.helpers import command_parser, coroutines, nested, responses from cecli.helpers.conversation import ConversationService, MessageTag +from cecli.helpers.observations.manager import ObservationManager from cecli.helpers.profiler import TokenProfiler from cecli.history import ChatSummary from cecli.hooks import HookIntegration @@ -441,6 +442,7 @@ def __init__( # Initialize conversation system if enabled ConversationService.get_chunks(self).initialize_conversation_system() + self.observation_manager = ObservationManager.get_instance(self) self.commands = commands or Commands(self.io, self, args=args) self.commands.coder = self @@ -1746,11 +1748,14 @@ async def compact_context_if_needed(self, force=False, message=""): if not self.enable_context_compaction: return - # Check if combined messages exceed the token limit, - # Get messages from ConversationManager - done_messages = ConversationService.get_manager(self).get_messages_dict(MessageTag.DONE) - cur_messages = ConversationService.get_manager(self).get_messages_dict(MessageTag.CUR) - diff_messages = ConversationService.get_manager(self).get_messages_dict(MessageTag.DIFFS) + # Trigger background observation/reflection check + await self.observation_manager.check_and_trigger() + + manager = ConversationService.get_manager(self) + done_messages = manager.get_messages_dict(MessageTag.DONE) + cur_messages = manager.get_messages_dict(MessageTag.CUR) + diff_messages = manager.get_messages_dict(MessageTag.DIFFS) + # Exclude first cur_message since that's the user's initial input done_tokens = self.summarizer.count_tokens(done_messages) cur_tokens = self.summarizer.count_tokens(cur_messages[1:] if len(cur_messages) > 1 else []) @@ -1768,129 +1773,97 @@ async def compact_context_if_needed(self, force=False, message=""): self.io.update_spinner("Compacting...") try: - # Check if done_messages alone exceed the limit - if done_tokens > self.context_compaction_max_tokens or done_tokens > cur_tokens: - # Create a summary of the done_messages - # Append custom message to compaction prompt if provided - compaction_prompt = self.gpt_prompts.compaction_prompt - if message: - compaction_prompt = f"{compaction_prompt}\n\n{message}" - - summary_text = await self.summarizer.summarize_all_as_text( - done_messages, + compaction_prompt = self.gpt_prompts.compaction_prompt + if message: + compaction_prompt = f"{compaction_prompt}\n\n{message}" + + async def summarize_and_update(messages, tag): + text = await self.summarizer.summarize_all_as_text( + messages, compaction_prompt, self.context_compaction_summary_tokens, ) + if not text: + raise ValueError(f"Summarization of {tag} messages returned empty.") - if not summary_text: - raise ValueError("Summarization returned an empty result.") + if self.observation_manager.observations: + obs_text = "\n".join(self.observation_manager.observations) + text = f"HISTORICAL OBSERVATIONS:\n{obs_text}\n\n{text}" - # Replace old DONE messages with the summary in ConversationManager - ConversationService.get_manager(self).clear_tag(MessageTag.DONE) - ConversationService.get_manager(self).add_message( - message_dict={ - "role": "user", - "content": summary_text, - }, - tag=MessageTag.DONE, - ) - ConversationService.get_manager(self).add_message( - message_dict={ - "role": "assistant", - "content": ( - "Ok, I will use this summary as the context for our conversation going" - " forward." - ), - }, - tag=MessageTag.DONE, - ) + manager.clear_tag(tag) - # Check if cur_messages alone exceed the limit (after potentially compacting done_messages) - if cur_tokens > self.context_compaction_max_tokens or cur_tokens > done_tokens: - # Create a summary of the cur_messages - # Append custom message to compaction prompt if provided - compaction_prompt = self.gpt_prompts.compaction_prompt - if message: - compaction_prompt = f"{compaction_prompt}\n\n{message}" - - cur_summary_text = await self.summarizer.summarize_all_as_text( - cur_messages, - compaction_prompt, - self.context_compaction_summary_tokens, - ) - - if not cur_summary_text: - raise ValueError("Summarization of current messages returned an empty result.") + if tag == MessageTag.DONE: + manager.add_message({"role": "user", "content": text}, tag=tag) + manager.add_message( + { + "role": "assistant", + "content": ( + "Ok, I will use this summary and the observations as context for" + " our conversation going forward." + ), + }, + tag=tag, + ) + else: + if self.last_user_message: + manager.add_message( + {"role": "user", "content": self.last_user_message}, tag=tag + ) - # Replace current CUR messages with the summary in ConversationManager - ConversationService.get_manager(self).clear_tag(MessageTag.CUR) + manager.add_message( + { + "role": "assistant", + "content": "Ok. I am awaiting your summary of our goals to proceed.", + }, + tag=tag, + force=True, + ) - if self.last_user_message: - ConversationService.get_manager(self).add_message( - message_dict={ + manager.add_message( + { "role": "user", - "content": self.last_user_message, + "content": ( + "Here is a summary of our current goals and historical" + f" context:\n{text}" + ), }, - tag=MessageTag.CUR, + tag=tag, ) - # Add the summary conversation - ConversationService.get_manager(self).add_message( - message_dict={ - "role": "assistant", - "content": "Ok. I am awaiting your summary of our goals to proceed.", - }, - tag=MessageTag.CUR, - force=True, - ) - ConversationService.get_manager(self).add_message( - message_dict={ - "role": "user", - "content": f"Here is a summary of our current goals:\n{cur_summary_text}", - }, - tag=MessageTag.CUR, - ) - ConversationService.get_manager(self).add_message( - message_dict={ - "role": "assistant", - "content": ( - "Ok, I will use this summary and proceed with our task." - " I will first apply any changes in the summary and then" - " continue exploration as necessary." - ), - }, - tag=MessageTag.CUR, - force=True, - ) - - # Find the last assistant messages in the current conversation - latest_messages = [] + manager.add_message( + { + "role": "assistant", + "content": ( + "Ok, I will use this summary and proceed with our task. I will" + " first apply any changes in the summary and then continue" + " exploration as necessary." + ), + }, + tag=tag, + force=True, + ) - # Search from the end to find the most recent assistant messages - for msg in reversed(cur_messages): - latest_messages.append(msg) + latest_messages = [] + for msg in reversed(messages): + latest_messages.append(msg) + if msg["role"] == "assistant": + break + for msg in reversed(latest_messages): + manager.add_message(msg, tag=tag) - if msg["role"] == "assistant": - break + if done_tokens > self.context_compaction_max_tokens or done_tokens > cur_tokens: + await summarize_and_update(done_messages, MessageTag.DONE) - for msg in reversed(latest_messages): - ConversationService.get_manager(self).add_message( - message_dict={ - "role": msg["role"], - "content": msg["content"], - }, - tag=MessageTag.CUR, - ) + if cur_tokens > self.context_compaction_max_tokens or cur_tokens > done_tokens: + await summarize_and_update(cur_messages, MessageTag.CUR) self.io.tool_output("...chat history compacted.") self.io.update_spinner(self.io.last_spinner_text) - # Clear all diff and file context messages - ConversationService.get_manager(self).clear_tag(MessageTag.DIFFS) - ConversationService.get_manager(self).clear_tag(MessageTag.FILE_CONTEXTS) - - # Reset ConversationFiles cache entirely + manager.clear_tag(MessageTag.DIFFS) + manager.clear_tag(MessageTag.FILE_CONTEXTS) ConversationService.get_files(self).clear_file_cache() + except Exception as e: self.io.tool_warning(f"Context compaction failed: {e}") self.io.tool_warning("Proceeding with full history for now.") diff --git a/cecli/commands/clear.py b/cecli/commands/clear.py index 7fb98001ca5..c12449d1d98 100644 --- a/cecli/commands/clear.py +++ b/cecli/commands/clear.py @@ -19,6 +19,7 @@ async def execute(cls, io, coder, args, **kwargs): ConversationService.get_manager(coder).clear_tag(MessageTag.FILE_CONTEXTS) ConversationService.get_files(coder).reset() + coder.observation_manager.reset() # Clear TUI output if available if coder.tui and coder.tui(): diff --git a/cecli/commands/reset.py b/cecli/commands/reset.py index 53d002f29f3..6d47b9c734f 100644 --- a/cecli/commands/reset.py +++ b/cecli/commands/reset.py @@ -22,6 +22,7 @@ async def execute(cls, io, coder, args, **kwargs): # Re-initialize Conversation components with current coder ConversationService.get_manager(coder).initialize(reformat=True) ConversationService.get_files(coder) # Ensure instance exists/initialized + coder.observation_manager.reset() # Clear TUI output if available if coder.tui and coder.tui(): diff --git a/cecli/helpers/observations/__init__.py b/cecli/helpers/observations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cecli/helpers/observations/manager.py b/cecli/helpers/observations/manager.py new file mode 100644 index 00000000000..3d0ebf5d864 --- /dev/null +++ b/cecli/helpers/observations/manager.py @@ -0,0 +1,98 @@ +import asyncio +from datetime import datetime + +from cecli.helpers.conversation.service import ConversationService +from cecli.helpers.conversation.tags import MessageTag + + +class ObservationManager: + _instances = {} + + @classmethod + def get_instance(cls, coder): + if coder.uuid not in cls._instances: + cls._instances[coder.uuid] = cls(coder) + return cls._instances[coder.uuid] + + def __init__(self, coder): + self.coder = coder + self.observation_threshold = max((coder.context_compaction_max_tokens or 0) / 3, 20000) + self.reflection_threshold = self.observation_threshold * 2 + self.is_processing = False + self._last_observed_index = 0 + self.observations = [] # Internal storage + + async def check_and_trigger(self): + if self.is_processing: + return + + manager = ConversationService.get_manager(self.coder) + cur_messages = manager.get_messages_dict(MessageTag.CUR) + + # Calculate unobserved tokens + unobserved = cur_messages[self._last_observed_index :] + if not unobserved: + return + + tokens = self.coder.summarizer.count_tokens(unobserved) + + if tokens >= self.observation_threshold: + asyncio.create_task(self.run_observation(unobserved)) + self._last_observed_index = len(cur_messages) + + obs_tokens = self.coder.summarizer.count_tokens( + [{"role": "user", "content": o} for o in self.observations] + ) + + if obs_tokens >= self.reflection_threshold: + asyncio.create_task(self.run_reflection()) + + async def run_observation(self, messages): + self.is_processing = True + try: + manager = ConversationService.get_manager(self.coder) + all_messages = manager.get_messages_dict() + prompt = self.coder.gpt_prompts.observation_prompt + observation = await self.coder.summarizer.summarize_all_as_text( + all_messages, prompt, max_tokens=8192 + ) + self.observations.append(self.format_observation(observation)) + except asyncio.CancelledError: + raise + except Exception as e: + self.coder.io.tool_error(f"Error during observation: {e}") + finally: + self.is_processing = False + + async def run_reflection(self): + self.is_processing = True + try: + # Prepare observations for the reflector + obs_text = "\n".join([f"- {o}" for o in self.observations]) + + # Use the Reflector to condense and get next steps + reflection_prompt = self.coder.gpt_prompts.reflection_prompt + reflection = await self.coder.summarizer.summarize_all_as_text( + [{"role": "user", "content": obs_text}], + reflection_prompt, + max_tokens=8192, + ) + + # 1. Internal State Update: Store the condensed log internally + self.observations = [reflection] + + self._last_observed_index = 0 + except asyncio.CancelledError: + raise + except Exception as e: + self.coder.io.tool_error(f"Error during reflection: {e}") + finally: + self.is_processing = False + + def reset(self): + self.observations = [] + self._last_observed_index = 0 + + def format_observation(self, text): + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M") + return f"[{timestamp}] {text}" diff --git a/cecli/prompts/base.yml b/cecli/prompts/base.yml index ce4ad217705..f9ada82d002 100644 --- a/cecli/prompts/base.yml +++ b/cecli/prompts/base.yml @@ -85,29 +85,43 @@ go_ahead_tip: "" compaction_prompt: | # Instruction: Context Compaction & State Preservation - The following conversation is exceeding the context limit. Transform this history into a "Mission Briefing" that allows a new LLM instance to resume with zero loss of technical momentum. + The following conversation is exceeding the context limit. Transform this history into a "Mission Intent" summary that allows a new LLM instance to resume with zero loss of technical momentum. ## Required Output Format: ### 1. Core Objective A concise statement of the final goal and the specific success criteria. - ### 2. Narrative Event Log (Up to 50 Outcomes) - Provide a bulleted list documenting the sequence of **outcomes and milestones** reached. Do not describe tool syntax; describe what was learned or changed in one sentence per bullet: - - (e.g., "Mapped the project structure and identified `core/logic.py` as the primary target.") - - (e.g., "Discovered that the connection timeout error is triggered by the `RetryPolicy` class.") - - (e.g., "Successfully refactored the `validate_input` function to handle null bytes.") - - (e.g., "Reverted changes to `db.py` after determining the issue was in the environment config instead.") - - (e.g., "Verified that the fix works in isolation using a temporary script in `.cecli/temp`.") + ### 2. Current Technical Context + - **Files In-Scope**: List paths currently being edited or actively referenced. + - **Verified Facts**: List specific findings about the code logic that are now "known truths." + - **Discarded Hypotheses**: List paths or theories that were tested and proven incorrect to avoid repetition. - ### 3. Current Technical Context - - **Files In-Scope**: List paths currently being edited or actively referenced. - - **Verified Facts**: List specific findings about the code logic that are now "known truths." - - **Discarded Hypotheses**: List paths or theories that were tested and proven incorrect to avoid repetition. - - ### 4. Strategic Pivot & Next Steps + ### 3. Strategic Pivot & Next Steps - **Current Intent**: What is the model currently trying to prove or implement? - **Immediate Next Steps**: The prioritized next tool calls and logic steps. - --- ## Conversation History to Compact: + +observation_prompt: | + Review the following conversation history and extract an + event-based log of key decisions, tool outputs, and user requests. + Focus on "what happened" rather than a narrative summary. + Use a bulleted list of concise, factual statements. + Document the sequence of **outcomes and milestones** reached. + Do not describe tool syntax; describe what was learned or changed in one sentence per bullet. + +reflection_prompt: | + Condense the following observation log into a consolidated set of + key historical facts. Additionally, explicitly state the + CURRENT GOALS and NEXT STEPS based on the history. + + Format as: + OBSERVATIONS: + - [fact 1] + - [fact 2] + - [fact 3] + + STATUS: + - Goal: [current goal] + - Next Step: [immediate next step] diff --git a/pytest.ini b/pytest.ini index 1c14c4bc82d..3916c33d4ab 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,6 +8,7 @@ testpaths = tests/coders tests/conversations tests/helpers/monorepo + tests/helpers/observations tests/hooks tests/mcp tests/help diff --git a/tests/helpers/observations/test_observation_manager.py b/tests/helpers/observations/test_observation_manager.py new file mode 100644 index 00000000000..e19dc7996a6 --- /dev/null +++ b/tests/helpers/observations/test_observation_manager.py @@ -0,0 +1,185 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from cecli.helpers.observations.manager import ObservationManager + + +@pytest.mark.asyncio +async def test_observation_manager_initialization(): + coder = MagicMock() + coder.uuid = "test-uuid" + coder.context_compaction_max_tokens = 60000 + + manager = ObservationManager.get_instance(coder) + assert manager.observation_threshold == 20000 + assert manager.reflection_threshold == 40000 + assert manager.observations == [] + + +@pytest.mark.asyncio +async def test_observation_manager_reset(): + coder = MagicMock() + coder.uuid = "test-uuid-reset" + coder.context_compaction_max_tokens = 60000 + manager = ObservationManager.get_instance(coder) + + manager.observations = ["obs1"] + manager._last_observed_index = 5 + + manager.reset() + assert manager.observations == [] + assert manager._last_observed_index == 0 + + +@pytest.mark.asyncio +async def test_check_and_trigger_observation(monkeypatch): + coder = MagicMock() + coder.uuid = "test-uuid-trigger" + coder.context_compaction_max_tokens = 30000 + # threshold = 20000 + + mock_manager = MagicMock() + mock_manager.get_tag_messages.return_value = [{"role": "user", "content": "hello"}] * 100 + + with patch( + "cecli.helpers.observations.manager.ConversationService.get_manager", + return_value=mock_manager, + ): + coder.summarizer.count_tokens.return_value = 25000 + + manager = ObservationManager.get_instance(coder) + + with patch.object(manager, "run_observation", new_callable=AsyncMock) as mock_run: + await manager.check_and_trigger() + # Should trigger observation because 25000 > 20000 + assert mock_run.called + + +@pytest.mark.asyncio +async def test_compact_context_with_observations(): + from cecli.coders.base_coder import Coder + + coder = MagicMock(spec=Coder) + coder.uuid = "test-coder-compaction" + coder.enable_context_compaction = True + coder.context_compaction_max_tokens = 1000 + coder.context_compaction_summary_tokens = 100 + coder.last_user_message = "Last user msg" + coder.io = MagicMock() + + # Mock observation manager with some observations + obs_manager = ObservationManager.get_instance(coder) + obs_manager.observations = ["Observation 1"] + coder.observation_manager = obs_manager + + # Mock prompts + coder.gpt_prompts = MagicMock() + coder.gpt_prompts.compaction_prompt = "Compaction Prompt" + + # Mock summarizer + coder.summarizer = MagicMock() + # Calls to count_tokens: + # 1. check_and_trigger: count_tokens(unobserved) + # 2. check_and_trigger: count_tokens(observations) + # 3. compact_context_if_needed: done_tokens + # 4. compact_context_if_needed: cur_tokens + # 5. compact_context_if_needed: diff_tokens + # 6. summarize_and_update: count_tokens inside + coder.summarizer.count_tokens.side_effect = [100, 100, 100, 1000, 0, 50] + coder.summarizer.summarize_all_as_text = AsyncMock(return_value="Summary Text") + + # Mock manager + mock_conv_manager = MagicMock() + cur_messages = [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi"}] + # 1. check_and_trigger (CUR) + # 2. compact (DONE) + # 3. compact (CUR) + # 4. compact (DIFFS) + mock_conv_manager.get_messages_dict.side_effect = [cur_messages, [], cur_messages, []] + + with patch( + "cecli.coders.base_coder.ConversationService.get_manager", return_value=mock_conv_manager + ): + # Call the method + await Coder.compact_context_if_needed(coder, force=True) + + # Verify summarize_all_as_text was called + assert coder.summarizer.summarize_all_as_text.called + + # Verify observations were prepended to the summary + expected_content = "HISTORICAL OBSERVATIONS:\nObservation 1\n\nSummary Text" + + # Check that add_message was called with the expected prepended content + all_calls = mock_conv_manager.add_message.call_args_list + found = False + for c in all_calls: + msg_dict = c[0][0] if c[0] else c[1].get("message_dict") + if msg_dict and expected_content in msg_dict.get("content", ""): + found = True + break + assert found, "Expected summary with observations not found in add_message calls" + + +@pytest.mark.asyncio +async def test_compact_context_with_observations_integration(): + from cecli.coders.base_coder import Coder + + coder = MagicMock(spec=Coder) + coder.uuid = "test-coder-compaction-int" + coder.enable_context_compaction = True + coder.context_compaction_max_tokens = 1000 + coder.context_compaction_summary_tokens = 100 + coder.last_user_message = "Last user msg" + coder.io = MagicMock() + + # Mock observation manager with some observations + obs_manager = ObservationManager.get_instance(coder) + obs_manager.observations = ["Observation 1"] + coder.observation_manager = obs_manager + + # Mock prompts + coder.gpt_prompts = MagicMock() + coder.gpt_prompts.compaction_prompt = "Compaction Prompt" + + # Mock summarizer + coder.summarizer = MagicMock() + # 1. check_and_trigger: unobserved + # 2. check_and_trigger: obs + # 3. compact: done + # 4. compact: cur + # 5. compact: diff + # 6. summarize_and_update: inner + coder.summarizer.count_tokens.side_effect = [100, 100, 100, 1000, 0, 50] + coder.summarizer.summarize_all_as_text = AsyncMock(return_value="Summary Text") + + # Mock manager + mock_conv_manager = MagicMock() + cur_messages = [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi"}] + # 1. check_and_trigger (CUR) + # 2. compact (DONE) + # 3. compact (CUR) + # 4. compact (DIFFS) + mock_conv_manager.get_messages_dict.side_effect = [cur_messages, [], cur_messages, []] + + with patch( + "cecli.coders.base_coder.ConversationService.get_manager", return_value=mock_conv_manager + ): + # Call the method + await Coder.compact_context_if_needed(coder, force=True) + + # Verify summarize_all_as_text was called + assert coder.summarizer.summarize_all_as_text.called + + # Verify observations were prepended to the summary + expected_content = "HISTORICAL OBSERVATIONS:\nObservation 1\n\nSummary Text" + + # Check that add_message was called with the expected prepended content + all_calls = mock_conv_manager.add_message.call_args_list + found = False + for c in all_calls: + msg_dict = c[0][0] if c[0] else c[1].get("message_dict") + if msg_dict and expected_content in msg_dict.get("content", ""): + found = True + break + assert found, "Expected summary with observations not found in add_message calls" From 5c485ae001f4ed99f0f464cf56210fd2607fadf0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Apr 2026 23:28:04 -0400 Subject: [PATCH 7/7] Fix test paths on windows --- tests/basic/test_find_or_blocks.py | 12 ++++-- tests/basic/test_repo.py | 60 ++++++++++++++++++------------ 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/tests/basic/test_find_or_blocks.py b/tests/basic/test_find_or_blocks.py index fb42fb1371f..6ef4048ad90 100755 --- a/tests/basic/test_find_or_blocks.py +++ b/tests/basic/test_find_or_blocks.py @@ -4,6 +4,7 @@ import io import re import sys +from pathlib import Path import pytest @@ -73,17 +74,20 @@ def process_markdown(filename, fh): class TestFindOrBlocks: def test_process_markdown(self): + # Get the fixtures directory path + fixtures_dir = Path(__file__).parent.parent / "fixtures" + # Path to the input markdown file - input_file = "tests/fixtures/chat-history.md" + input_file = fixtures_dir / "chat-history.md" # Path to the expected output file - expected_output_file = "tests/fixtures/chat-history-search-replace-gold.txt" + expected_output_file = fixtures_dir / "chat-history-search-replace-gold.txt" # Create a StringIO object to capture the output output = io.StringIO() # Run process_markdown - process_markdown(input_file, output) + process_markdown(str(input_file), output) # Get the actual output actual_output = output.getvalue() @@ -98,7 +102,7 @@ def test_process_markdown(self): diff = difflib.unified_diff( expected_output.splitlines(keepends=True), actual_output.splitlines(keepends=True), - fromfile=expected_output_file, + fromfile=str(expected_output_file), tofile="actual output", ) diff --git a/tests/basic/test_repo.py b/tests/basic/test_repo.py index 42c5062e82e..c6b3c341c09 100644 --- a/tests/basic/test_repo.py +++ b/tests/basic/test_repo.py @@ -137,53 +137,65 @@ async def test_get_commit_message(self, mock_send): model2 = Model("gpt-4") dump(model1) dump(model2) - repo = GitRepo(InputOutput(), None, None, models=[model1, model2]) - # Call the get_commit_message method with dummy diff and context - result = await repo.get_commit_message("dummy diff", "dummy context") + with GitTemporaryDirectory(): + repo = GitRepo(InputOutput(), None, None, models=[model1, model2]) + + # Call the get_commit_message method with dummy diff and context + result = await repo.get_commit_message("dummy diff", "dummy context") - # Assert that the returned message is the expected one from the second model - assert result == "a good commit message" + # Assert that the returned message is the expected one from the second model + assert result == "a good commit message" - # Check that simple_send_with_retries was called twice - assert mock_send.call_count == 2 + # Check that simple_send_with_retries was called twice + assert mock_send.call_count == 2 - # Check that both calls were made with the same messages - first_call_messages = mock_send.call_args_list[0][0][0] # Get messages from first call - second_call_messages = mock_send.call_args_list[1][0][0] # Get messages from second call - assert first_call_messages == second_call_messages + # Check that both calls were made with the same messages + first_call_messages = mock_send.call_args_list[0][0][0] # Get messages from first call + second_call_messages = mock_send.call_args_list[1][0][ + 0 + ] # Get messages from second call + assert first_call_messages == second_call_messages @patch("cecli.models.Model.simple_send_with_retries", new_callable=AsyncMock) async def test_get_commit_message_strip_quotes(self, mock_send): mock_send.return_value = '"a good commit message"' - repo = GitRepo(InputOutput(), None, None, models=[self.GPT35]) - # Call the get_commit_message method with dummy diff and context - result = await repo.get_commit_message("dummy diff", "dummy context") + with GitTemporaryDirectory(): + repo = GitRepo(InputOutput(), None, None, models=[self.GPT35]) + # Call the get_commit_message method with dummy diff and context + result = await repo.get_commit_message("dummy diff", "dummy context") - # Assert that the returned message is the expected one - assert result == "a good commit message" + # Assert that the returned message is the expected one + assert result == "a good commit message" @patch("cecli.models.Model.simple_send_with_retries", new_callable=AsyncMock) async def test_get_commit_message_no_strip_unmatched_quotes(self, mock_send): mock_send.return_value = 'a good "commit message"' - repo = GitRepo(InputOutput(), None, None, models=[self.GPT35]) - # Call the get_commit_message method with dummy diff and context - result = await repo.get_commit_message("dummy diff", "dummy context") + with GitTemporaryDirectory(): + repo = GitRepo(InputOutput(), None, None, models=[self.GPT35]) + # Call the get_commit_message method with dummy diff and context + result = await repo.get_commit_message("dummy diff", "dummy context") - # Assert that the returned message is the expected one - assert result == 'a good "commit message"' + # Assert that the returned message is the expected one + assert result == 'a good "commit message"' @patch("cecli.models.Model.simple_send_with_retries", new_callable=AsyncMock) async def test_get_commit_message_with_custom_prompt(self, mock_send): mock_send.return_value = "Custom commit message" custom_prompt = "Generate a commit message in the style of Shakespeare" - repo = GitRepo(InputOutput(), None, None, models=[self.GPT35], commit_prompt=custom_prompt) - result = await repo.get_commit_message("dummy diff", "dummy context") + with GitTemporaryDirectory(): + repo = GitRepo( + InputOutput(), None, None, models=[self.GPT35], commit_prompt=custom_prompt + ) + result = await repo.get_commit_message("dummy diff", "dummy context") - assert result == "Custom commit message" + assert result == "Custom commit message" + mock_send.assert_called_once() + args = mock_send.call_args[0] # Get positional args + assert args[0][0]["content"] == custom_prompt # Check first message content mock_send.assert_called_once() args = mock_send.call_args[0] # Get positional args assert args[0][0]["content"] == custom_prompt # Check first message content