From 990891d6a1ba8094fac4ee487bf6260d736f50f9 Mon Sep 17 00:00:00 2001 From: xingshui Date: Sat, 25 May 2024 00:40:34 +0800 Subject: [PATCH 1/2] feat:Load exist project in local file system that not in database. --- pilot/load_exist.py | 266 ++++++++++++++++++ .../documentation/summary_codebase.prompt | 46 +++ 2 files changed, 312 insertions(+) create mode 100644 pilot/load_exist.py create mode 100644 pilot/prompts/documentation/summary_codebase.prompt diff --git a/pilot/load_exist.py b/pilot/load_exist.py new file mode 100644 index 000000000..a9bfcf692 --- /dev/null +++ b/pilot/load_exist.py @@ -0,0 +1,266 @@ +# ------------------------------------------------------------------------------ +# usage: +# cd to gpt-pilot VS Code Extension xxx/gpt-pilot +# copy /my_project to xxx/gpt-pilot/workspace +# cd xxx/gpt-pilot/pilot +# python ./load_exist.py name=my_project +# open VS Code Extension +# Click 'Load App' button, and select 'my_project', then click 'Load' button,choice 'Yes' twice +# It will directly goto the develpment step finsh status,and ask you to input the next feature, +# then you can input your new feature requirements to continue the development. +# ------------------------------------------------------------------------------ +import builtins +import json +import os +import platform +import sys +import traceback + + +try: + from dotenv import load_dotenv +except ImportError: + gpt_pilot_root = os.path.dirname(os.path.dirname(__file__)) + venv_path = os.path.join(gpt_pilot_root, 'pilot-env') + requirements_path = os.path.join(gpt_pilot_root, 'requirements.txt') + if sys.prefix == sys.base_prefix: + venv_python_path = os.path.join(venv_path, 'scripts' if sys.platform == 'win32' else 'bin', 'python') + print('Python environment for GPT Pilot is not set up.') + print(f'Please create Python virtual environment: {sys.executable} -m venv {venv_path}') + print(f'Then install the required dependencies with: {venv_python_path} -m pip install -r {requirements_path}') + else: + print('Python environment for GPT Pilot is not completely set up.') + print(f'Please run `{sys.executable} -m pip install -r {requirements_path}` to finish Python setup, and rerun GPT Pilot.') + sys.exit(-1) + +load_dotenv(override=True) + +from const.function_calls import GET_DOCUMENTATION_FILE, ARCHITECTURE, DEVELOPMENT_PLAN +from helpers.Project import Project +from helpers.AgentConvo import AgentConvo +from helpers.agents.TechnicalWriter import TechnicalWriter +from helpers.exceptions import ApiError, TokenLimitError, GracefulExit +from helpers.files import get_directory_contents +from helpers.agents.ProductOwner import PROJECT_DESCRIPTION_STEP +from helpers.agents import Architect, ARCHITECTURE_STEP, ENVIRONMENT_SETUP_STEP, TechLead +from helpers.agents.TechLead import DEVELOPMENT_PLANNING_STEP + +from templates import PROJECT_TEMPLATES +from utils.utils import generate_app_data +from utils.style import color_red +from utils.custom_print import get_custom_print +from utils.arguments import get_arguments +from utils.exit import exit_gpt_pilot +from utils.settings import settings, loader, get_version +from logger.logger import logger +from database.models.app import App +from peewee import DoesNotExist +from database.database import ( + database_exists, + create_database, + tables_exist, + create_tables, + save_app, + get_created_apps, + save_development_step, save_progress, update_app_status, +) + + +def init(): + # Check if the database exists, if not, create it + if not database_exists(): + create_database() + + # Check if the tables exist, if not, create them + if not tables_exist(): + create_tables() + + arguments = get_arguments() + + logger.info('Starting with args: %s', arguments) + + return arguments + + +def exit_if_app_not_in_workspace(): + if not os.path.exists(app_root_path): + print(f'App {app_name} does not exist in the workspace directory.') + sys.exit(-1) + + +def planning_architecture(): + global convo, llm_response + print("Planning project architecture...\n") + project.architect = Architect(project) + convo = AgentConvo(project.architect) + llm_response = convo.send_message('architecture/technologies.prompt', + {'name': project.args['name'], + 'app_summary': project.project_description, + 'user_stories': project.user_stories, + 'user_tasks': project.user_tasks, + "os": platform.system(), + 'app_type': project.args['app_type'], + "templates": PROJECT_TEMPLATES, + }, + ARCHITECTURE + ) + project.architecture = llm_response["architecture"] + project.system_dependencies = llm_response["system_dependencies"] + project.package_dependencies = llm_response["package_dependencies"] + project.project_template = llm_response["template"] + + save_progress(app.id, ARCHITECTURE_STEP, { + "messages": convo.messages, + "architecture": llm_response, + "app_data": app_data + }) + + +def load_codebase(): + global messages, llm_response + prompt_path = "development/task/breakdown.prompt" + prompt_data = {"app_summary": "", "tasks": [{"description": "Implement Nothing", "finished": False}], "current_task": "Nothing", "files": [], "file_summaries": {}, "all_feedbacks": [], "modified_files": [], "files_at_start_of_task": [], "previous_features": None, "current_feature": None} + messages = '''[{"role": "system", "content": ""},{"role": "user", "content": ""}]''' + llm_response = {'text': 'DONE'} + save_development_step(project, prompt_path, prompt_data, messages, llm_response, exception=None) + + +def summary_codebase(): + global project + print('Creating project description') + project.technical_writer = TechnicalWriter(project) + convo = AgentConvo(project.technical_writer) + llm_response = convo.send_message('documentation/summary_codebase.prompt', { + "name": project.args['name'], + "app_type": project.app_type, + "app_summary": project.project_description, + "user_stories": project.user_stories, + "user_tasks": project.user_tasks, + "directory_tree": project.get_directory_tree(True), + "files": project.get_all_coded_files(), + "previous_features": project.previous_features, + "current_feature": project.current_feature, + }, GET_DOCUMENTATION_FILE) + app_summary = llm_response.get('content') + project.project_description = app_summary + save_progress(app.id, PROJECT_DESCRIPTION_STEP, { + "prompt": app_summary, + "messages": [], + "summary": app_summary, + "app_data": app_data + }) + + +def setup_environment(): + print("Setting up the environment...\n") + save_progress(app.id, ENVIRONMENT_SETUP_STEP, { + "os_specific_technologies": project.system_dependencies, + "newly_installed_technologies": [], + "app_data": app_data + }) + + +def planning_development(): + global llm_response, project + print('Plan for development is created.\n') + project.development_plan = [{"description": "Load exist codebase"}] + save_progress(app.id, DEVELOPMENT_PLANNING_STEP, { + "development_plan": project.development_plan, "app_data": app_data + }) + + +if __name__ == "__main__": + ''' + This script is used to load the existing app from the workspace directory. + It will check if the app exists in the database, if not, it will create it by summaries codes. + ''' + ask_feedback = True + project = None + run_exit_fn = True + gpt_pilot_root = os.path.dirname(os.path.dirname(__file__)) + workspace_path = os.path.join(gpt_pilot_root, 'workspace') + args = init() + try: + builtins.print, ipc_client_instance = get_custom_print(args) + run_exit_fn = False + app_name = args.get('name') + app_root_path = os.path.join(workspace_path, app_name) + print('----------------------------------------------------------------------------------------') + print(f'Loading App { app_name }，{ app_root_path }') + print('----------------------------------------------------------------------------------------') + + exit_if_app_not_in_workspace() + + if 'app_type' not in args: + args['app_type'] = 'App' + + + project = Project(args) + project.set_root_path(app_root_path) + try: + app = App.get(App.name == app_name) + if app.status: + print(f'App {app_name} already exist in database.') + sys.exit(-1) + except DoesNotExist: + project.app = None + if 'new_app_id' in args: + project.args['app_id'] = args['new_app_id'] + + app = save_app(project) + print(f'App {app_name} saved in database.') + + project.app = app + project.args['app_id'] = str(app.id) + project.app_type = args['app_type'] + project.current_step = 'loading' + project.skip_steps = True + + load_codebase() + + project.files = project.get_all_coded_files() + app_data = generate_app_data(project.args) + summary_codebase() + planning_architecture() + setup_environment() + planning_development() + project.technical_writer.create_readme() + + print('Load exist App done.') + + except (ApiError, TokenLimitError) as err: + + run_exit_fn = False + if isinstance(err, TokenLimitError): + print( + "We sent too large request to the LLM, resulting in an error. " + "This is usually caused by including framework files in an LLM request. " + "Here's how you can get GPT Pilot to ignore those extra files: " + "https://bit.ly/faq-token-limit-error" + ) + print('Exit') + + except KeyboardInterrupt: + if project is not None and project.check_ipc(): + run_exit_fn = False + + except GracefulExit: + # can't call project.finish_loading() here because project can be None + run_exit_fn = False + print('Exit') + + except Exception as err: + print(color_red('---------- GPT PILOT EXITING WITH ERROR ----------')) + print(err) + traceback.print_exc() + print(color_red('--------------------------------------------------')) + ask_feedback = False + + finally: + if project is not None: + if project.check_ipc(): + ask_feedback = False + project.current_task.exit() + project.finish_loading(do_cleanup=False) + if run_exit_fn: + exit_gpt_pilot(project, ask_feedback) diff --git a/pilot/prompts/documentation/summary_codebase.prompt b/pilot/prompts/documentation/summary_codebase.prompt new file mode 100644 index 000000000..4c994a624 --- /dev/null +++ b/pilot/prompts/documentation/summary_codebase.prompt @@ -0,0 +1,46 @@ +You are working on a {{ app_type }} called "{{ name }}" and you need to create a detailed description for current state of project by analyzing the codebase. Your first task is to create a description text. + +{{ files_list }} + +Now, based on the project details provided, think step by step and create description text for this project. The file should have the following format: +''' +Short description (a few sentences) of the project based on the project details. + +Features: +Description of what the app can do and how it can be used. + +Functional Specification: +Description of what operations the app will perform and how it is expected to behave. It should include the app's behavior, the app's interactions with the user, and the app's responses to inputs. Also include any other necessary elements to ensure it comprehensively describes the functionality of the application. + +Technical Specification: +Description of the architecture and technologies used in the project, and the project structure. +''' + + +Here is a example: +''' +The application is a simple TaskManage app built using React. Its primary function is to allow users to manage a list of tasks (todos). Each task has a description and a state (open or completed, with the default state being open). The application is frontend-only, with no user sign-up or authentication process. The goal is to provide a straightforward and user-friendly interface for task management. + +Features: +1. Display of Todos: A list that displays all task items. Each item shows its description and a checkbox to indicate its state (open or completed). +2. Add New task: A button to add a new task item. Clicking this button will prompt the user to enter a description for the new task. +3. Toggle State: Each task item includes a checkbox. Checking/unchecking this box toggles the task's state between open and completed. +4. Local Storage: The application will use the browser's local storage to persist todos between sessions, ensuring that users do not lose their data upon reloading the application. + +Functional Specification: +- Upon loading the application, it fetches existing todos from the local storage and displays them in a list. +- Each task item in the list displays a checkbox and a description. The checkbox reflects the task's current state (checked for completed, unchecked for open). +- When the user checks or unchecks a checkbox, the application updates the state of the corresponding task item and saves the updated list to local storage. +- Clicking the "Add New task" button prompts the user to enter a description for the new task. Upon confirmation, the application adds the new task (with the default state of open) to the list and updates local storage. +- The application does not support deleting or editing task items to keep the interface and interactions simple. +- Todos persist between sessions using the browser's local storage. The application saves any changes to the task list (additions or state changes) in local storage and retrieves this data when the application is reloaded. + +Technical Specification: +- Platform/Technologies: The application is a web application developed using React. No backend technologies are required. +- Styling: Use Bootstrap 5 for a simple and functional interface. Load Boostrap from the CDN (don't install it locally): + - https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css + - https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js +- State Management: Directly in the React component + - make sure to initialize the state from the local storage as default (... = useState(JSON.parse(localStorage.getItem('todos')) || []) to avoid race conditions +- Data Persistence: The application uses the browser's local storage to persist todos between sessions. It stores the array of todos as a JSON string and parses this data on application load. +''' From b5372ad35ac287c216e527cdb6e3b16ac48ca059 Mon Sep 17 00:00:00 2001 From: xingshui Date: Wed, 29 May 2024 17:54:47 +0800 Subject: [PATCH 2/2] fix:Sometimes when summarizing codebase, there may be errors in outputting examples instead of summarizing the code correctly. --- .../documentation/summary_codebase.prompt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pilot/prompts/documentation/summary_codebase.prompt b/pilot/prompts/documentation/summary_codebase.prompt index 4c994a624..b970201db 100644 --- a/pilot/prompts/documentation/summary_codebase.prompt +++ b/pilot/prompts/documentation/summary_codebase.prompt @@ -3,7 +3,8 @@ You are working on a {{ app_type }} called "{{ name }}" and you need to create a {{ files_list }} Now, based on the project details provided, think step by step and create description text for this project. The file should have the following format: -''' +-----------------------START_OF_FORMAT------------------- +``` Short description (a few sentences) of the project based on the project details. Features: @@ -14,11 +15,12 @@ Description of what operations the app will perform and how it is expected to be Technical Specification: Description of the architecture and technologies used in the project, and the project structure. -''' +``` +------------------------END_OF_FORMAT--------------------- - -Here is a example: -''' +Here is an example: +-----------------------START_OF_EXAMPLES------------------ +``` The application is a simple TaskManage app built using React. Its primary function is to allow users to manage a list of tasks (todos). Each task has a description and a state (open or completed, with the default state being open). The application is frontend-only, with no user sign-up or authentication process. The goal is to provide a straightforward and user-friendly interface for task management. Features: @@ -43,4 +45,6 @@ Technical Specification: - State Management: Directly in the React component - make sure to initialize the state from the local storage as default (... = useState(JSON.parse(localStorage.getItem('todos')) || []) to avoid race conditions - Data Persistence: The application uses the browser's local storage to persist todos between sessions. It stores the array of todos as a JSON string and parses this data on application load. -''' +``` +-----------------------END_OF_EXAMPLES------------------------ +