From 74f4e728654d5a42c37203c14eb3480335ae7730 Mon Sep 17 00:00:00 2001 From: wwzeng1 Date: Fri, 19 Apr 2024 20:04:15 +0000 Subject: [PATCH 1/9] modify improvements --- sweepai/agents/modify.py | 6 ++++-- sweepai/core/prompts.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index b8cd89ccdc..7d138bbced 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -450,9 +450,9 @@ def create_user_message( continue relevant_file_paths_string += f"\n\n\n{cloned_repo.get_file_contents(file_path=relevant_file_path)}\n" relevant_file_paths_string = f"\n{relevant_file_paths_string}\n" - combined_request_message.replace("{relevant_files}", f'\nHere are some relevant modules, such as useful helper functions for resolving this issue. You likely will not need to edit these modules but may need to import them or understand their usage interface: {relevant_file_paths_string}\n') + combined_request_message = combined_request_message.replace("{relevant_files}", f'\nHere are some relevant modules, such as useful helper functions for resolving this issue. You likely will not need to edit these modules but may need to import them or understand their usage interface: {relevant_file_paths_string}\n') else: - combined_request_message.replace("{relevant_files}", "") + combined_request_message = combined_request_message.replace("{relevant_files}", "") user_message = f"\n{request}\n\n{combined_request_message}" return user_message @@ -703,6 +703,7 @@ def handle_function_call( else: llm_response = NO_TOOL_CALL_PROMPT elif tool_name == "make_change": + breakpoint() error_message = "" for key in ["file_name", "original_code", "new_code"]: if key not in tool_call: @@ -744,6 +745,7 @@ def handle_function_call( # if the original_code couldn't be found in the chunk we need to let the llm know if original_code not in file_contents and correct_indent == -1: # TODO: add weighted ratio to the choices, penalize whitespace less + breakpoint() best_match, best_score = find_best_match(original_code, file_contents) if best_score > 80: diff --git a/sweepai/core/prompts.py b/sweepai/core/prompts.py index f452b87a21..0757931870 100644 --- a/sweepai/core/prompts.py +++ b/sweepai/core/prompts.py @@ -234,12 +234,13 @@ [additional creates] -Instructions for modifying one section of the file. Reference change locations using surrounding code or functions. +Instructions for modifying one section of the file. Reference change locations using surrounding code or functions. Referenced code must be copied verbatim from the original file. Include relevant type definitions, interfaces, schemas. -Instructions for modifying a different section of the same file. Use multiple blocks for the same file to separate distinct changes. +Instructions for modifying a different section of the same file. Referenced code must be copied verbatim from the original file. +Use multiple blocks for the same file to separate distinct changes. [additional modifies as needed, for the same file or different files] From 196c3e862ca808ef2bf32a29950d83cd53d55dfd Mon Sep 17 00:00:00 2001 From: wwzeng1 Date: Fri, 19 Apr 2024 20:08:59 +0000 Subject: [PATCH 2/9] fix --- sweepai/core/prompts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sweepai/core/prompts.py b/sweepai/core/prompts.py index 0757931870..9c61a9a144 100644 --- a/sweepai/core/prompts.py +++ b/sweepai/core/prompts.py @@ -234,12 +234,12 @@ [additional creates] -Instructions for modifying one section of the file. Reference change locations using surrounding code or functions. Referenced code must be copied verbatim from the original file. +Instructions for modifying one section of the file. If you reference specific lines of code, code must be copied VERBATIM from the file. Never paraphrase code or comments. Include relevant type definitions, interfaces, schemas. -Instructions for modifying a different section of the same file. Referenced code must be copied verbatim from the original file. +Instructions for modifying a different section of the same file. If you reference specific lines of code, code must be copied VERBATIM from the file. Never paraphrase code or comments. Use multiple blocks for the same file to separate distinct changes. From 4e79a013dc58d01b7f1e743f2bfd6cf9640e9b56 Mon Sep 17 00:00:00 2001 From: wwzeng1 Date: Fri, 19 Apr 2024 22:07:03 +0000 Subject: [PATCH 3/9] tune issue cleanup --- sweepai/agents/issue_cleanup_agent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sweepai/agents/issue_cleanup_agent.py b/sweepai/agents/issue_cleanup_agent.py index cab91f3370..5071604086 100644 --- a/sweepai/agents/issue_cleanup_agent.py +++ b/sweepai/agents/issue_cleanup_agent.py @@ -3,14 +3,14 @@ from sweepai.core.chat import ChatGPT from sweepai.core.entities import Message -system_prompt = """Remove irrelevant text from the issue description. Keep stacktraces and informative debugging information.""" +system_prompt = """Remove any completely irrelevant text from the issue description. Keep stacktraces and informative debugging information""" prompt = """\ {issue_description} -Delete irrelevant text from the issue. Keep stacktraces and informative debugging information. +Remove any completely irrelevant text from the issue description. Keep stacktraces and informative debugging information. Copy as much text verbatim as possible. Format your response in tags:""" @@ -29,7 +29,7 @@ def cleanup_issue( ] issue_desc_response = self.chat( # gpt4 04-09 had a better one in minimal (1 example) testing, seems smart content=prompt.format( - issue_description=issue_description, + issue_description=issue_description.strip("\n"), ), temperature=0.2, ) From ee025e9dbb76b6449833e2b2a72052499d5072a6 Mon Sep 17 00:00:00 2001 From: wwzeng1 Date: Fri, 19 Apr 2024 22:25:22 +0000 Subject: [PATCH 4/9] add 1shot ex to planning; modify bugfixing --- sweepai/agents/issue_cleanup_agent.py | 3 +- sweepai/agents/modify.py | 43 +++--- sweepai/core/prompts.py | 180 +++++++++++++++++++++++++- sweepai/core/sweep_bot.py | 4 +- 4 files changed, 204 insertions(+), 26 deletions(-) diff --git a/sweepai/agents/issue_cleanup_agent.py b/sweepai/agents/issue_cleanup_agent.py index 5071604086..22b4fdf517 100644 --- a/sweepai/agents/issue_cleanup_agent.py +++ b/sweepai/agents/issue_cleanup_agent.py @@ -12,7 +12,8 @@ Remove any completely irrelevant text from the issue description. Keep stacktraces and informative debugging information. Copy as much text verbatim as possible. -Format your response in tags:""" +Explain what needs to be removed. +Then format your response in tags:""" class IssueCleanupBot(ChatGPT): diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index 7d138bbced..c6d4eecb06 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -255,9 +255,9 @@ self_review_prompt = """First, review and critique the change(s) you have made. Consider the following points: -1. Analyze code patch and indicate: - - Purpose and impact of each change - - Check for potential errors: +1. Analyze the code patch and indicate: + a. Purpose and impact of each change + b. Check for potential errors: - Logic errors - Unhandled edge cases - Missing imports @@ -265,17 +265,13 @@ - Undefined variables/functions - Usage of nullable attributes - Non-functional code - - Alignment with plan and requirements + c. Alignment with plan and requirements 2. Perform critical contextual analysis: - Break down changes - Explain reasoning - - Identify logic issues, edge cases, plan deviations - - Consider all scenarios and pitfalls - - Consider backwards compatibility and future-proofing - - Suggest fixes for problems -3. Be extremely critical. Do not overlook ANY issues. + - Identify plan deviations -Limit the scope of the critique to the current task, which is: +Be extremely critical but limit the scope of the critique to the current task, which is: {current_task} @@ -439,11 +435,12 @@ def create_user_message( .replace("{files_to_modify}", files_to_modify_string.lstrip('\n')) \ .replace("{files_to_modify_list}", english_join(deduped_file_names)) \ .replace("{completed_prompt}", completed_prompt) + precomputed_file_list = cloned_repo.get_file_list() if relevant_filepaths: relevant_file_paths_string = "" for relevant_file_path in relevant_filepaths: - if relevant_file_path not in cloned_repo.get_file_list(): - logger.warning(f"Relevant file path {relevant_file_path} not found in cloned repo.") + if relevant_file_path not in precomputed_file_list: + logger.warning(f"Relevant file path {relevant_file_path} not found in cloned repo.") # the relevant file paths aren't well formatted, so we get some issues here continue if relevant_file_path in [fcr.filename for fcr in fcrs]: logger.warning(f"Relevant file path {relevant_file_path} is already in the list of files to modify.") @@ -679,6 +676,7 @@ def handle_function_call( tool_name = function_call.function_name tool_call = function_call.function_parameters if tool_name == "submit_task": + breakpoint() changes_made = generate_diffs(modify_files_dict) if changes_made: llm_response = "DONE" @@ -746,21 +744,22 @@ def handle_function_call( if original_code not in file_contents and correct_indent == -1: # TODO: add weighted ratio to the choices, penalize whitespace less breakpoint() - best_match, best_score = find_best_match(original_code, file_contents) + best_match, best_score = find_best_match(original_code, file_contents) # TODO: this should check other files for exact to 90% match if best_score > 80: error_message = f"The original_code provided does not appear to be present in file {file_name}. The original_code contains:\n```\n{tool_call['original_code']}\n```\nDid you mean the following?\n```\n{best_match}\n```\nHere is the diff:\n```\n{generate_diff(tool_call['original_code'], best_match)}\n```" else: error_message = f"The original_code provided does not appear to be present in file {file_name}. The original_code contains:\n```\n{tool_call['original_code']}\n```\nBut this section of code was not found anywhere inside the current file. DOUBLE CHECK that the change you are trying to make is not already implemented in the code!" - - # first check the lines in original_code, if it is too long, ask for smaller changes - original_code_lines_length = len(original_code.split("\n")) - if original_code_lines_length > 10: - error_message += f"\n\nThe original_code seems to be quite long with {original_code_lines_length} lines of code. Break this large change up into a series of SMALLER changes to avoid errors like these! Try to make sure the original_code is under 10 lines. DOUBLE CHECK to make sure that this make_change tool call is only attempting a singular change, if it is not, make sure to split this make_change tool call into multiple smaller make_change tool calls!" - else: - # generate the diff between the original code and the current chunk to help the llm identify what it messed up - # chunk_original_code_diff = generate_diff(original_code, current_chunk) - not necessary - error_message += "\n\nDOUBLE CHECK that the original_code you have provided is correct, if it is not, correct it then make another replacement with the corrected original_code. The original_code MUST be in section A in order for you to make a change. DOUBLE CHECK to make sure that this make_change tool call is only attempting a singular change, if it is not, make sure to split this make_change tool call into multiple smaller make_change tool calls!" + + # first check the lines in original_code, if it is too long, ask for smaller changes + original_code_lines_length = len(original_code.split("\n")) + if original_code_lines_length > 10: # I moved this into the else statement because if you had a good match you don't need the extra warnings. + error_message += f"\n\nThe original_code seems to be quite long with {original_code_lines_length} lines of code. Break this large change up into a series of SMALLER changes to avoid errors like these! Try to make sure the original_code is under 10 lines. DOUBLE CHECK to make sure that this make_change tool call is only attempting a singular change, if it is not, make sure to split this make_change tool call into multiple smaller make_change tool calls!" + else: + # generate the diff between the original code and the current chunk to help the llm identify what it messed up + # chunk_original_code_diff = generate_diff(original_code, current_chunk) - not necessary + # WARNING: sometimes this occurs because the LLM selected the wrong file, so we need to provide the llm with the correct file + error_message += "\n\nDOUBLE CHECK that the original_code you have provided is correct, if it is not, correct it then make another replacement with the corrected original_code. The original_code MUST be in the selected file in order for you to make a change. DOUBLE CHECK to make sure that this make_change tool call is only attempting a singular change, if it is not, make sure to split this make_change tool call into multiple smaller make_change tool calls!" break # ensure original_code and new_code has the correct indents new_code_lines = new_code.split("\n") diff --git a/sweepai/core/prompts.py b/sweepai/core/prompts.py index 9c61a9a144..c601c4d6b5 100644 --- a/sweepai/core/prompts.py +++ b/sweepai/core/prompts.py @@ -179,6 +179,183 @@ files_to_change_abstract_prompt = """Write an abstract minimum plan to address this issue in the least amount of change possible. Try to originate the root causes of this issue. Be clear and concise. 1 paragraph.""" +# add two newlines +files_to_change_example = """ + +Here is an example issue and output to illustrate the desired format: + + +# Issue + +The product list page is displaying all products in one long list. Update it to paginate the results, showing 10 products per page with next/previous navigation links. The pagination should be done on the client-side by updating the necessary React components and Redux store. + + + + +The current implementation of the product list page has the following issues: + +1. All products are displayed on a single page. + - This can lead to slow load times, especially for large product catalogs. + - Users have to scroll through a long list to browse products. + - There is no way to navigate to different subsets of products. + +2. The ProductList component in src/components/ProductList.js renders the entire list of products passed to it. + - It needs to be updated to only render the products for the current page. + - Pagination navigation links need to be added to allow users to switch between pages. + +3. The ProductsContainer in src/containers/ProductsContainer.js fetches all products in the componentDidMount lifecycle method. + - It should be updated to keep track of the current page number in the component's state. + - When the page changes (by clicking on a pagination link), it should update the current page in the state. + +4. The products reducer in src/reducers/productReducer.js stores the fetched products in the state. + - It should be updated to store the current page number. + - When the products are fetched successfully, it should update the products and current page in the state. + +To implement client-side pagination, the following changes are needed: + +- src/components/ProductList.js: + - Modify to render pagination links and display only the products for the current page. +- src/containers/ProductsContainer.js: + - Update to keep track of the current page in its state. + - Pass the current page to the ProductList component. + - Update the current page in the state when the page changes. +- src/reducers/productReducer.js: + - Update to store the current page number in the state. + - Handle the fetched products and update the current page in the state. + +By making these changes, the product list page will be paginated on the client-side, showing only a limited number of products per page. Users can navigate between pages using the pagination links. This improves the user experience for browsing large product catalogs. + +The pagination will be implemented purely on the client-side by updating the necessary React components and Redux store. The server-side API endpoint for fetching products will remain unchanged. + + + + +Import the Link component from react-router-dom: +```js +import { Link } from 'react-router-dom'; +``` + +Add `currentPage` and `totalPages` props to the component: +```js +const ProductList = ({ products, onAddToCart, currentPage, totalPages }) => ( +``` + +Update the PropTypes definition to include `currentPage` and `totalPages`: +```js +ProductList.propTypes = { + products: PropTypes.arrayOf( + PropTypes.shape({ + id: PropTypes.number.isRequired, + title: PropTypes.string.isRequired, + price: PropTypes.number.isRequired, + }) + ).isRequired, + onAddToCart: PropTypes.func.isRequired, + currentPage: PropTypes.number.isRequired, + totalPages: PropTypes.number.isRequired, +}; +``` + +Add pagination navigation links before the closing ``: +```js +
+ {currentPage > 1 && ( + Previous + )} + {currentPage} + {currentPage < totalPages && ( + Next + )} +
+``` +This renders "Previous" and "Next" links that navigate to the corresponding page. The links are conditionally rendered based on the current page number. + + + +Add `currentPage` to the component state in the constructor: +```js +constructor(props) { + super(props); + this.state = { + currentPage: 1, + }; +} +``` + +Add a `componentDidUpdate` method to update the current page when the URL changes: +```js +componentDidUpdate(prevProps) { + const { location } = this.props; + const { search } = location; + const query = new URLSearchParams(search); + const page = parseInt(query.get('page'), 10) || 1; + + if (prevProps.location.search !== search) { + this.setState({ currentPage: page }); + } +} +``` +This method compares the current URL search params with the previous ones. If the "page" query param has changed, it updates the `currentPage` state. + +Calculate the `totalPages` based on the number of products and pass it along with the `currentPage` state to the `ProductList` component: +```js +render() { + const { products } = this.props; + const { currentPage } = this.state; + const totalPages = Math.ceil(products.length / 10); + + const startIndex = (currentPage - 1) * 10; + const endIndex = startIndex + 10; + const currentProducts = products.slice(startIndex, endIndex); + + return ( +
+

Products

+ +
+ ); +} +``` +This calculates the `totalPages` based on the number of products and passes it to the `ProductList` component. It also slices the `products` array to get the products for the current page. + + + +Add a `currentPage` field to the initial state: +```js +const initialState = { + items: [], + currentPage: 1, + loading: false, + error: null, +}; +``` + +Update the `FETCH_PRODUCTS_SUCCESS` case in the reducer to set the `currentPage` to 1: +```js +case FETCH_PRODUCTS_SUCCESS: + return { + ...state, + loading: false, + items: action.payload, + currentPage: 1, + }; +``` +This ensures that when the products are fetched successfully, the current page is reset to 1. + + + + +src/actions/types.js +src/store.js + + +""" + files_to_change_system_prompt = """You are an AI assistant helping an intern write code to resolve a GitHub issue. The user will provide code snippets, a description of the issue, and relevant parts of the codebase. Your role is to analyze the issue and codebase, then provide a clear, step-by-step plan the intern can follow to make the necessary code changes to resolve the issue. Reference specific files, functions, variables and code snippets in your plan. Give detailed natural language instructions and explanations the intern can follow to write the code themselves. Organize the steps logically and break them into small, manageable tasks. @@ -248,7 +425,8 @@ [List of all relevant files to reference while making changes, one per line] -""" +""" + files_to_change_example + plan_selection_prompt = """Critique the pros and cons of each plan based on the following guidelines, prioritizing thoroughness and correctness over potential performance overhead: - Correctness: The code change should fully address the original issue or requirement without introducing new bugs, security vulnerabilities, or performance problems. Follow defensive programming practices, such as avoiding implicit assumptions, validating inputs, and handling edge cases. Consider the potential impact on all relevant data structures and ensure the solution maintains data integrity and consistency. Thoroughness is a top priority. diff --git a/sweepai/core/sweep_bot.py b/sweepai/core/sweep_bot.py index 7a34b16bf2..63fb1257d7 100644 --- a/sweepai/core/sweep_bot.py +++ b/sweepai/core/sweep_bot.py @@ -218,7 +218,7 @@ def get_files_to_change( relevant_snippets = [snippet for snippet in max_snippets if any(snippet.file_path == relevant_snippet.file_path for relevant_snippet in relevant_snippets)] read_only_snippets = [snippet for snippet in max_snippets if not any(snippet.file_path == relevant_snippet.file_path for relevant_snippet in relevant_snippets)] - relevant_snippet_template = '# Relevant codebase snippets:\nHere are the relevant snippets from the codebase. These will be your primary reference to solve the problem:\n\n\n{file_path}\n\n\n{content}\n\n' + relevant_snippet_template = '\n\n{file_path}\n\n\n{content}\n\n' read_only_snippet_template = '\n\n{file_path}\n\n\n{content}\n\n' # attach all relevant snippets joined_relevant_snippets = "\n".join( @@ -228,7 +228,7 @@ def get_files_to_change( content=snippet.expand(300).get_snippet(add_lines=False), ) for i, snippet in enumerate(relevant_snippets) ) - relevant_snippets_message = f"\n{joined_relevant_snippets}\n" + relevant_snippets_message = f"# Relevant codebase snippets:\nHere are the relevant snippets from the codebase. These will be your primary reference to solve the problem:\n\n\n{joined_relevant_snippets}\n" messages.append( Message( role="user", From 205b57200cdf0955a8d570bbb537198c3b060e38 Mon Sep 17 00:00:00 2001 From: martin ye Date: Fri, 19 Apr 2024 22:38:15 +0000 Subject: [PATCH 5/9] state clean up fix --- sweepai/agents/modify.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index c6d4eecb06..70c7f14743 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -498,6 +498,14 @@ def render_current_task(fcrs: list[FileChangeRequest]) -> str: fcr = fcrs[current_fcr_index] return f"The CURRENT TASK is to {fcr.change_type} {fcr.filename}. The specific instructions to do so are listed below:\n\n\n{fcr.instructions}\n" +# return the number of tasks completed +def tasks_completed(fcrs: list[FileChangeRequest]): + completed_tasks = 0 + for fcr in fcrs: + if fcr.is_completed: + completed_tasks += 1 + return completed_tasks + def modify( fcrs: list[FileChangeRequest], request: str, @@ -536,12 +544,13 @@ def modify( modify_files_dict = {} llm_state = { "initial_check_results": {}, - "done_counter": 0, + "done_counter": 0, # keep track of how many times the submit_task tool has been called "request": request, "plan": render_plan(fcrs), "current_task": render_current_task(fcrs), - "user_message_index": 1, - "user_message_index_chat_logger": 1, + "user_message_index": 1, # used for detailed chat logger messages + "user_message_index_chat_logger": 1, # used for detailed chat logger messages + "fcrs": fcrs, "previous_attempt": "", } @@ -555,6 +564,7 @@ def modify( else: function_call = validate_and_parse_function_call(function_calls_string, chat_gpt) if function_call: + num_of_tasks_done = tasks_completed(fcrs) # note that detailed_chat_logger_messages is meant to be modified in place by handle_function_call function_output, modify_files_dict, llm_state = handle_function_call(cloned_repo, function_call, modify_files_dict, llm_state, chat_logger_messages=detailed_chat_logger_messages, use_openai=use_openai) if function_output == "DONE": @@ -582,9 +592,9 @@ def modify( modify_files_dict=modify_files_dict ) user_message = f"Here is the UPDATED user request, plan, and state of the code changes. REVIEW THIS CAREFULLY!\n{user_message}" - - # update context if a change was made - if changes_made(modify_files_dict, previous_modify_files_dict): + # state cleanup should only occur after a task has been finished and if a change was made and if a change was made + current_num_of_tasks_done = tasks_completed(fcrs) + if changes_made(modify_files_dict, previous_modify_files_dict) and current_num_of_tasks_done > num_of_tasks_done: # remove the previous user message and add it to the end, do not remove if it is the inital user message if llm_state["user_message_index"] != 1: chat_gpt.messages.pop(llm_state["user_message_index"]) From 14f2ec3861178caf9dba95b7f534d863b4a7a1fa Mon Sep 17 00:00:00 2001 From: wwzeng1 Date: Fri, 19 Apr 2024 22:43:55 +0000 Subject: [PATCH 6/9] fix current fcr --- sweepai/agents/modify.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index c6d4eecb06..70c8acb56c 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -384,10 +384,8 @@ def create_user_message( relevant_filepaths: list[str] = None, modify_files_dict: dict[str, dict[str, str]] = None ) -> str: - current_fcr_index = 0 - for current_fcr_index, fcr in enumerate(fcrs): - if not fcr.is_completed: - break + current_fcr_index = [i for i, fcr in enumerate(fcrs) if not fcr.is_completed][0] if any([not fcr.is_completed for fcr in fcrs]) else 0 + breakpoint() combined_request_unformatted = "{relevant_files}# Plan of Code Changes\n\nIn order to solve the user's request you will need to modify or create {files_to_modify_list}.{completed_prompt} Here are the instructions for the edits you need to make:\n\n\n{files_to_modify}\n" completed_prompt = "" if current_fcr_index == 0 else f" You have already completed {current_fcr_index} of the {len(fcrs)} required changes." if modify_files_dict: @@ -476,10 +474,7 @@ def ordinal(n: int): return "%d%s" % (n,"tsnrhtdd"[(n//10%10!=1)*(n%10<4)*n%10::4]) # noqa def render_plan(fcrs: list[FileChangeRequest]) -> str: - current_fcr_index = 0 - for current_fcr_index, fcr in enumerate(fcrs): - if not fcr.is_completed: - break + current_fcr_index = [i for i, fcr in enumerate(fcrs) if not fcr.is_completed][0] if any([not fcr.is_completed for fcr in fcrs]) else 0 plan = f"You have {len(fcrs)} changes to make and you are currently working on the {ordinal(current_fcr_index + 1)} task." for i, fcr in enumerate(fcrs): if i < current_fcr_index: @@ -545,6 +540,7 @@ def modify( "fcrs": fcrs, "previous_attempt": "", } + breakpoint() # rerender plan and also show the current task # this message list is for the chat logger to have a detailed insight into why failures occur detailed_chat_logger_messages = [{"role": message.role, "content": message.content} for message in chat_gpt.messages] # used to determine if changes were made @@ -557,6 +553,7 @@ def modify( if function_call: # note that detailed_chat_logger_messages is meant to be modified in place by handle_function_call function_output, modify_files_dict, llm_state = handle_function_call(cloned_repo, function_call, modify_files_dict, llm_state, chat_logger_messages=detailed_chat_logger_messages, use_openai=use_openai) + fcrs = llm_state["fcrs"] if function_output == "DONE": # add the diff of all changes to chat_logger if chat_logger: @@ -692,7 +689,7 @@ def handle_function_call( fcr.is_completed = True llm_response = f"SUCCESS\n\nThe current task is complete. Please move on to the next task. {llm_state['current_task']}" break - + if all([fcr.is_completed for fcr in llm_state["fcrs"]]): llm_response = "DONE" elif tool_name == "no_tool_call": From 89cb271ee6f28ef1b42bb6cbf593f5b06d8cd01d Mon Sep 17 00:00:00 2001 From: martin ye Date: Fri, 19 Apr 2024 23:00:23 +0000 Subject: [PATCH 7/9] another bug fix --- sweepai/agents/modify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index c0df621113..b55783f48c 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -697,7 +697,7 @@ def handle_function_call( for fcr in llm_state["fcrs"]: if not fcr.is_completed: fcr.is_completed = True - llm_response = f"SUCCESS\n\nThe current task is complete. Please move on to the next task. {llm_state['current_task']}" + llm_response = f"SUCCESS\n\nThe current task is complete. Please move on to the next task. {render_next_task(llm_state['fcrs'])}" break if all([fcr.is_completed for fcr in llm_state["fcrs"]]): From 63d940298db5778aca100a58f693988e73e0937d Mon Sep 17 00:00:00 2001 From: wwzeng1 Date: Fri, 19 Apr 2024 23:11:35 +0000 Subject: [PATCH 8/9] fixing bugs with generating diff --- sweepai/agents/modify.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index c0df621113..6041a58eb6 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -549,7 +549,6 @@ def modify( "fcrs": fcrs, "previous_attempt": "", } - breakpoint() # rerender plan and also show the current task # this message list is for the chat logger to have a detailed insight into why failures occur detailed_chat_logger_messages = [{"role": message.role, "content": message.content} for message in chat_gpt.messages] # used to determine if changes were made @@ -693,13 +692,13 @@ def handle_function_call( llm_response = "DONE" else: llm_response = "ERROR\n\nNo changes were made. Please continue working on your task." - + breakpoint() for fcr in llm_state["fcrs"]: if not fcr.is_completed: fcr.is_completed = True - llm_response = f"SUCCESS\n\nThe current task is complete. Please move on to the next task. {llm_state['current_task']}" break - + llm_state['current_task'] = render_current_task(llm_state["fcrs"]) # rerender the current task + llm_response = f"SUCCESS\n\nThe previous task is now complete. Please move on to the next task. {llm_state['current_task']}" if all([fcr.is_completed for fcr in llm_state["fcrs"]]): llm_response = "DONE" elif tool_name == "no_tool_call": @@ -753,10 +752,13 @@ def handle_function_call( breakpoint() best_match, best_score = find_best_match(original_code, file_contents) # TODO: this should check other files for exact to 90% match + # expand best_match to include surrounding lines + + if best_score > 80: - error_message = f"The original_code provided does not appear to be present in file {file_name}. The original_code contains:\n```\n{tool_call['original_code']}\n```\nDid you mean the following?\n```\n{best_match}\n```\nHere is the diff:\n```\n{generate_diff(tool_call['original_code'], best_match)}\n```" + error_message = f"The original_code provided does not appear to be present in file {file_name}. Your provided original_code contains:\n```\n{tool_call['original_code']}\n```\nDid you mean the following?\n```\n{best_match}\n```\nHere is the diff:\n```\n{generate_diff(tool_call['original_code'], best_match, n=10)}\n```" else: - error_message = f"The original_code provided does not appear to be present in file {file_name}. The original_code contains:\n```\n{tool_call['original_code']}\n```\nBut this section of code was not found anywhere inside the current file. DOUBLE CHECK that the change you are trying to make is not already implemented in the code!" + error_message = f"The original_code provided does not appear to be present in file {file_name}. Your provided original_code contains:\n```\n{tool_call['original_code']}\n```\nBut this section of code was not found anywhere inside the current file. DOUBLE CHECK that the change you are trying to make is not already implemented in the code!" # first check the lines in original_code, if it is too long, ask for smaller changes original_code_lines_length = len(original_code.split("\n")) @@ -849,7 +851,7 @@ def handle_function_call( if not error_message: success_message = ( f"SUCCESS\n\nThe following changes have been applied to {file_name}:\n\n" - + generate_diff(file_contents, new_file_contents) + + generate_diff(file_contents, new_file_contents, n=10) ) + f"{warning_message}\n\nYou can continue to make changes to the file {file_name} and call the make_change tool again, or handle the rest of the plan. REMEMBER to add all necessary imports at the top of the file, if the import is not already there!" # set contents if file_name not in modify_files_dict: From cd3b2e4a3e7786ac7435f33516e10ee04d23ce97 Mon Sep 17 00:00:00 2001 From: martin ye Date: Mon, 22 Apr 2024 18:06:18 +0000 Subject: [PATCH 9/9] v1 --- sweepai/agents/modify.py | 2 - sweepai/core/prompts.py | 136 ++++++++++++++++++++++++++ sweepai/core/sweep_bot.py | 200 +++++++++++++++++++++++++++++++++++++- 3 files changed, 332 insertions(+), 6 deletions(-) diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index e1196a95c8..f09533aa97 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -691,7 +691,6 @@ def handle_function_call( tool_name = function_call.function_name tool_call = function_call.function_parameters if tool_name == "submit_task": - breakpoint() changes_made = generate_diffs(modify_files_dict) if changes_made: llm_response = "DONE" @@ -715,7 +714,6 @@ def handle_function_call( else: llm_response = NO_TOOL_CALL_PROMPT elif tool_name == "make_change": - breakpoint() error_message = "" for key in ["file_name", "original_code", "new_code"]: if key not in tool_call: diff --git a/sweepai/core/prompts.py b/sweepai/core/prompts.py index 0c213ff76e..2002ed8280 100644 --- a/sweepai/core/prompts.py +++ b/sweepai/core/prompts.py @@ -297,6 +297,142 @@ [Your explanation of why this plan was chosen and how it aligns with the guidelines and any modications made to this plan] """ +plan_review_prompt = """Now you are to double check the plan you have generated using the following guidelines: +- Name the types for each variable you are using/creating. You can do this by adding comments above each line of code. +- Double check that the attributes you are accessing/modifying for classes and variables actually exist. Comment where in the code each attribute is defined. + +FOCUS ONLY ON THE GUIDELINES AND YOUR EVALUATION + +List out all mistakes/errors you spot, and list out how to fix them. If the mistake/error is currently not fixable due to a lack of context, explain what you would need in order to be able to fi the issue. If there are no errors simply continue to the next check. +After you have made your evaluation you will be given a chance to correct these mistakes at a later date. +Respond in the following xml format: + + + +[Describe the mistake and how to fix it. If the mistake is not fixable due to a lack of context, explain what you would need to fix it.] + +... + +[Describe the mistake and how to fix it. If the mistake is not fixable due to a lack of context, explain what you would need to fix it.] + + + + +[A comma separated list of file paths to fetch the contents of. Make sure that the file paths are correct and spelled correct. Incorrectly spelled file paths will not be fetched, if you don't include this tag, it is assumed you do not need any extra code files to be fetched] + + + +['yes' if you did not find any mistakes and the original plan is good to go as is. 'no' otherwise. If you don't include this tag, it will be assumed that the original plan was good to go] +""" + +planning_tool_for_eval_view = """ + +view_file + +View a full code file in the code base. + + + +justification +str + +Explain why you need to view this file specifically. + + + +file_name +str + +Name of the file that you want to view. Ensure correct spelling as this is case-sensitive. + + + + +""" + +planning_tool_for_eval_submit = """ + +submit_final_plan + +Once you are certain all issues have been resolved, submit the final revised plan. + + + +explanation +Explain how this final plan differs from the original (if any) and why it is ready to submit. +str + + + + +final_plan +str + +Your final plan revised should go here. Make sure fix ALL the mistakes you identified while still solving the user request. Be sure that the final plan is still in the correct xml format. +Here is an example: + +[Example instructions here] + +... + +[More example instructions here] + +[Your explanation of why this plan was chosen and how it aligns with the guidelines and any modications made to this plan] + + + +""" + +planning_tools_for_eval = {"submit_final_plan": planning_tool_for_eval_submit, + "view_file": planning_tool_for_eval_view} + +planning_tools_prompt = """Now that you have evaluated the plan, you will take the necessary steps in order to improve the plan. +In this environment, you have access to the following tools to assist in fulfilling the user request: + +You MUST call them like this: + + +$TOOL_NAME + +<$PARAMETER_NAME>$PARAMETER_VALUE +... + + + + +Here are the tools available: +""" + +planning_redo_prompt = """ +Here are the code files that you have requested as further context to improve the plan: + +{code_files} + + +With this new context, you are to improve the inital plan created by fixing all the issues that were discovered in the evaluation of the plan. +Remember to respond in the following xml format: + + + +Instructions for creating the new file. Reference imports and entity names. Include relevant type definitions, interfaces, schemas. + +[additional creates] + + +Instructions for modifying one section of the file. If you reference specific lines of code, code must be copied VERBATIM from the file. Never paraphrase code or comments. +Include relevant type definitions, interfaces, schemas. + + + +Instructions for modifying a different section of the same file. If you reference specific lines of code, code must be copied VERBATIM from the file. Never paraphrase code or comments. +Use multiple blocks for the same file to separate distinct changes. + + +[additional modifies as needed, for the same file or different files] + + +MAKE SURE TO ADDRESS ALL ISSUES/ERRORS FOUND IN THE EVALUATION OF THE PLAN""" + context_files_to_change_prompt = """Your job is to write a high quality, detailed, step-by-step plan for an intern to help resolve a user's GitHub issue. You will analyze the provided code snippets, repository, and GitHub issue to understand the requested change. Create a step-by-step plan for an intern to fully resolve the user's GitHub issue. The plan should utilize the relevant code snippets and utility modules provided. Give detailed instructions for updating the code logic, as the intern is unfamiliar with the codebase. diff --git a/sweepai/core/sweep_bot.py b/sweepai/core/sweep_bot.py index 9eb331607a..5e05707741 100644 --- a/sweepai/core/sweep_bot.py +++ b/sweepai/core/sweep_bot.py @@ -11,6 +11,7 @@ from networkx import Graph from pydantic import BaseModel +from sweepai.agents.modify import validate_and_parse_function_call from sweepai.agents.modify_file import modify_file from sweepai.config.client import SweepConfig, get_blocked_dirs, get_branch_name_config from sweepai.config.server import DEFAULT_GPT4_32K_MODEL, DEFAULT_GPT35_MODEL @@ -33,9 +34,14 @@ context_files_to_change_prompt, pull_request_prompt, subissues_prompt, - files_to_change_system_prompt + files_to_change_system_prompt, + plan_review_prompt, + planning_tools_for_eval, + planning_tools_prompt, + planning_redo_prompt ) from sweepai.utils.chat_logger import ChatLogger, discord_log_error +from sweepai.utils.convert_openai_anthropic import AnthropicFunctionCall from sweepai.utils.progress import ( AssistantAPIMessage, AssistantConversation, @@ -46,6 +52,33 @@ from sweepai.utils.github_utils import ClonedRepo, commit_multi_file_changes BOT_ANALYSIS_SUMMARY = "bot_analysis_summary" +MODEL = "claude-3-opus-20240229" + +NO_FUNCTION_CALL = """ERROR!\n\nNo function call was made. If you attempted to make a function call but failed retry again but with the correct xml format. +If you are finished with fixing the issues with the plan you can submit the final plan by using the `submit_final_plan` tool. +An example is given below: + + + + + +[Explanation of why this plan was chosen, what issues were fixed (if any) and how it solves the original problem] + + + +[Example instructions here] + +... + +[More example instructions here] + +[Your explanation of why this plan was chosen and how it aligns with the guidelines and any modications made to this plan] + + + + + +""" def to_raw_string(s): @@ -174,7 +207,7 @@ def organize_snippets(snippets: list[Snippet], fuse_distance: int=600) -> list[S def get_max_snippets( snippets: list[Snippet], budget: int = 150_000 * 3.5, # 140k tokens - expand: int = 300, + expand: int = 3000, # testing expand ): """ Start with max number of snippets and then remove then until the budget is met. @@ -187,6 +220,159 @@ def get_max_snippets( return proposed_snippets raise Exception("Budget number of chars too low!") +def parse_xml_tag_from_string(tag: str, string: str) -> str: + match = re.search(f"<{tag}>(.*?)", string, re.DOTALL) + return match.group(1) if match else None + +# handles function calls made by planning +def handle_planning_function_call( + function_call: AnthropicFunctionCall, + llm_state: dict[str, str | bool | list[str]], + cloned_repo: ClonedRepo): + tool_name = function_call.function_name + tool_call = function_call.function_parameters + if tool_name == "submit_final_plan": + llm_state["done"] = True + final_plan = tool_call["final_plan"].strip("\n") + return final_plan, llm_state + elif tool_name == "view_file": + file_path = tool_call["file_name"].strip() # strip ALL whitespace + try: + file_contents = cloned_repo.get_file_contents(file_path) + success_message = f'SUCCESS!\n\nFile {file_path} found in the codebase. Here are the contents:\n\n\n{file_contents}\n' + return success_message, llm_state + except FileNotFoundError: + import pdb; pdb.set_trace() + error_message = f"ERROR!\n\nFile {file_path} not found in the codebase." + return error_message, llm_state + else: + available_tools = ", ".join(llm_state["available_tools"]) + error_message = f"ERROR!\n\nUnknown tool {tool_name}:\n\nYou have access to the following tools only:\n{available_tools}\n\nMake sure you respond with the correct xml format." + return error_message, llm_state + +# iterate on the initial plan to improve it using an agent +def iterate_on_plan(chat_gpt: ChatGPT, cloned_repo: ClonedRepo, chat_logger: ChatLogger = None): + # keep track of state + llm_state = { + "done": False, + "available_tools": ["view_file", "submit_final_plan"] + } + # make initial function call + planning_tools_prompt_string = planning_tools_prompt + # give agent what tools it has available + for tool in llm_state["available_tools"]: + planning_tools_prompt_string += f'\n\n{planning_tools_for_eval[tool]}' + function_calls_string = chat_gpt.chat_anthropic( + content=planning_tools_prompt_string, + model=MODEL, + stop_sequences=[""], + temperature=0.1 + ) + final_plan = "" + # max 10 iterations anymore probably means something has gone wrong. + max_iterations = 10 + for i in range(max_iterations): + function_call = validate_and_parse_function_call(function_calls_string, chat_gpt) + if function_call: + function_output, llm_state = handle_planning_function_call(function_call, llm_state, cloned_repo) + # check if we are done + if llm_state["done"]: + # update chat logger + if chat_logger: + chat_logger.add_chat( + { + "model": MODEL, + "messages": [{"role": message.role, "content": message.content} for message in chat_gpt.messages], + "output": f"We are done! Here is the final output:\n\n{function_output}", + } + ) + final_plan = function_output + break + # get the next function call + function_calls_string = chat_gpt.chat_anthropic( + content=function_output, + model=MODEL, + stop_sequences=[""], + temperature=0.1 + ) + else: + # get the next function call + function_calls_string = chat_gpt.chat_anthropic( + content=NO_FUNCTION_CALL, + model=MODEL, + stop_sequences=[""], + temperature=0.1 + ) + + if chat_logger: + output_message = function_call + if i == max_iterations - 1: + output_message += f"\n\nMAX ITERATIONS REACHED!" + + chat_logger.add_chat( + { + "model": MODEL, + "messages": [{"role": message.role, "content": message.content} for message in chat_gpt.messages], + "output": output_message, + } + ) + return final_plan + +# parse the output of the evaluation of the planning +def parse_planning_evaluation(evaluation: str) -> tuple[list[str], bool]: + initial_plan_is_good = parse_xml_tag_from_string("initial_plan_is_good", evaluation) + code_files_to_fetch = parse_xml_tag_from_string("code_files_to_fetch", evaluation) + initial_plan_is_good = False if initial_plan_is_good and 'no' in initial_plan_is_good else True # default to True if no tag is found + code_files_to_fetch = [file.strip() for file in code_files_to_fetch.split(",")] if code_files_to_fetch and code_files_to_fetch.strip() else [] # default to empty list if no tag is found + return code_files_to_fetch, initial_plan_is_good + +def planning_qc_pipeline(chat_gpt: ChatGPT, cloned_repo: ClonedRepo, chat_logger: ChatLogger = None): + # get initial evaluation + initial_evaluation = chat_gpt.chat_anthropic( + content=plan_review_prompt, + model=MODEL, + temperature=0.1 + ) + + if chat_logger: + chat_logger.add_chat( + { + "model": MODEL, + "messages": [{"role": message.role, "content": message.content} for message in chat_gpt.messages], + "output": initial_evaluation, + }) + # based on results of evaluation, iterate on the plan + # first parse the initial evaluation to see if there are any code files we need to fetch + code_files_to_fetch, initial_plan_is_good = parse_planning_evaluation(initial_evaluation) + if initial_plan_is_good: + return "" # return if no fixes are needed + fetched_code_files = {} + for code_file in code_files_to_fetch: + try: + fetched_code_file = cloned_repo.get_file_contents(code_file) + fetched_code_files[code_file] = fetched_code_file + except FileNotFoundError: + pass + formatted_code_files = "" + for code_file, fetched_code_file in fetched_code_files.items(): + formatted_code_files += f'\n\n{fetched_code_file}\n\n' + # now we get a new plan + formatted_planning_redo_prompt = planning_redo_prompt.format(code_files=formatted_code_files) + final_plan = chat_gpt.chat_anthropic( + content=formatted_planning_redo_prompt, + model=MODEL, + temperature=0.1 + ) + if chat_logger: + chat_logger.add_chat( + { + "model": MODEL, + "messages": [{"role": message.role, "content": message.content} for message in chat_gpt.messages], + "output": final_plan, + }) + return final_plan + +# get the plan and fcrs for the change def get_files_to_change( relevant_snippets: list[Snippet], read_only_snippets: list[Snippet], @@ -257,7 +443,7 @@ def get_files_to_change( Message( role="user", content=relevant_snippets_message, - key="relevant_snippets", + key="relevant_code_files", ) ) joined_relevant_read_only_snippets = "\n".join( @@ -328,12 +514,18 @@ def get_files_to_change( ), ], ) - MODEL = "claude-3-opus-20240229" + # get initial plan files_to_change_response = chat_gpt.chat_anthropic( content=joint_message + "\n\n" + (files_to_change_prompt if not context else context_files_to_change_prompt), model=MODEL, temperature=0.1 ) + + final_plan = planning_qc_pipeline(chat_gpt, cloned_repo, chat_logger=chat_logger) + if final_plan: + files_to_change_response = final_plan # update our final plan + + # files_to_change_response = iterate_on_plan(chat_gpt, cloned_repo, chat_logger=chat_logger) if chat_logger: chat_logger.add_chat( {