From d0fd844e7964d1ac18313237a617c6189521461b Mon Sep 17 00:00:00 2001
From: Code7G <111247923+Code7G@users.noreply.github.com>
Date: Thu, 28 Dec 2023 02:45:01 +0100
Subject: [PATCH] Fixed some major bugs and updated the code
(see further details in the commit)
---
DAIA.py | 3 ++
Pipfile | 12 -----
.../DVAI/GPT_4_with_Vision.py | 0
.../Interpreter/interpreter.py | 31 ++++++++---
.../OS_control/os_controller.py | 0
Versions/DAIA_GPT4V_FreeThink/run.py | 5 +-
.../DVAI/GPT_4_with_Vision.py | 0
.../{ => Components}/Memory/memory.py | 0
.../OS_control/os_controller.py | 0
.../Optimizer/optimization.py | 0
.../{ => Components}/Thinker/thinking.py | 53 +++++++++++++++----
Versions/DAIA_GPT4V_PreProgrammed/run.py | 24 ++++++---
utils/setup.py | 2 +-
13 files changed, 89 insertions(+), 41 deletions(-)
delete mode 100644 Pipfile
rename Versions/DAIA_GPT4V_FreeThink/{ => Components}/DVAI/GPT_4_with_Vision.py (100%)
rename Versions/DAIA_GPT4V_FreeThink/{ => Components}/Interpreter/interpreter.py (87%)
rename Versions/DAIA_GPT4V_FreeThink/{ => Components}/OS_control/os_controller.py (100%)
rename Versions/DAIA_GPT4V_PreProgrammed/{ => Components}/DVAI/GPT_4_with_Vision.py (100%)
rename Versions/DAIA_GPT4V_PreProgrammed/{ => Components}/Memory/memory.py (100%)
rename Versions/DAIA_GPT4V_PreProgrammed/{ => Components}/OS_control/os_controller.py (100%)
rename Versions/DAIA_GPT4V_PreProgrammed/{ => Components}/Optimizer/optimization.py (100%)
rename Versions/DAIA_GPT4V_PreProgrammed/{ => Components}/Thinker/thinking.py (91%)
diff --git a/DAIA.py b/DAIA.py
index 2f538a8..16599e6 100644
--- a/DAIA.py
+++ b/DAIA.py
@@ -17,6 +17,7 @@
from Versions.DAIA_GPT4V_PreProgrammed.run import run
from Versions.DAIA_GPT4V_FreeThink.run import run_f
from config import openai_api_key
+from utils.setup import setup
def main(key: str = None) -> None:
@@ -34,6 +35,8 @@ def main(key: str = None) -> None:
key = input_api_key
+ setup()
+
# User Interaction
try:
option = int(
diff --git a/Pipfile b/Pipfile
deleted file mode 100644
index ecb9dbb..0000000
--- a/Pipfile
+++ /dev/null
@@ -1,12 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-openai = "*"
-
-[dev-packages]
-
-[requires]
-python_version = "3.11"
diff --git a/Versions/DAIA_GPT4V_FreeThink/DVAI/GPT_4_with_Vision.py b/Versions/DAIA_GPT4V_FreeThink/Components/DVAI/GPT_4_with_Vision.py
similarity index 100%
rename from Versions/DAIA_GPT4V_FreeThink/DVAI/GPT_4_with_Vision.py
rename to Versions/DAIA_GPT4V_FreeThink/Components/DVAI/GPT_4_with_Vision.py
diff --git a/Versions/DAIA_GPT4V_FreeThink/Interpreter/interpreter.py b/Versions/DAIA_GPT4V_FreeThink/Components/Interpreter/interpreter.py
similarity index 87%
rename from Versions/DAIA_GPT4V_FreeThink/Interpreter/interpreter.py
rename to Versions/DAIA_GPT4V_FreeThink/Components/Interpreter/interpreter.py
index 71ebf7b..f427a2a 100644
--- a/Versions/DAIA_GPT4V_FreeThink/Interpreter/interpreter.py
+++ b/Versions/DAIA_GPT4V_FreeThink/Components/Interpreter/interpreter.py
@@ -15,8 +15,10 @@
# along with this program. If not, see .
from openai import OpenAI
-from Versions.DAIA_GPT4V_FreeThink.OS_control.os_controller import OSController
-from Versions.DAIA_GPT4V_FreeThink.DVAI.GPT_4_with_Vision import DVAI
+from Versions.DAIA_GPT4V_FreeThink.Components.OS_control.os_controller import (
+ OSController,
+)
+from Versions.DAIA_GPT4V_FreeThink.Components.DVAI.GPT_4_with_Vision import DVAI
from pathlib import Path
from random import randint
from selenium import webdriver
@@ -93,13 +95,20 @@ def interpret_commands(self, input_content: str):
)
continue
- # If there is only one parameter to the command, then convert the parameter to a string
+ # If there is only one parameter to the command, then convert the parameter to a string and don't include the *
if len(params) == 1:
params = params[0]
+ if command_name in self.commands:
+ command_function = self.commands[command_name]
+ command_return = command_function(params)
+ command_returns.append(
+ f"{command_name} command return: {command_return}"
+ )
+ continue
if command_name in self.commands:
command_function = self.commands[command_name]
- command_return = command_function(params)
+ command_return = command_function(*params)
command_returns.append(
f"{command_name} command return: {command_return}"
)
@@ -130,7 +139,7 @@ def write_down(self, string: str, identifier_title: str):
"""
writedown_file = Path(
- f"DAIA/Versions/DAIA_GPT4V_Free/Memory/Writedown_files/{identifier_title}.txt"
+ f"Versions/DAIA_GPT4V_Free/Memory/Writedown_files/{identifier_title}.txt"
)
writedown_file.write_text(string)
@@ -144,7 +153,7 @@ def retrieve(self, identifier: str):
"""
writedown_file = Path(
- f"DAIA/Versions/DAIA_GPT4V_Free/Memory/Writedown_files/{identifier}.txt"
+ f"Versions/DAIA_GPT4V_Free/Memory/Writedown_files/{identifier}.txt"
)
# writedown_files = writedown_dir.glob('*.txt')
@@ -168,6 +177,10 @@ def web(self, website_url: str):
browser = webdriver.Chrome(options=options)
browser.get(website_url)
+ # Detach the browser to prevent automatic closing after function return
+ browser.execute_cdp_cmd(
+ "Browser.detachFromTarget", {"sessionId": browser.session_id}
+ )
return f"Successfully visited {website_url} with Google Chrome"
@@ -203,6 +216,8 @@ def click_on_item(self, item: str):
f"Where on the provided screenshot is {item}? Provide the location in x and y coordinates. For example: 500x 200y"
)
x_value, y_value = self.extract_coordinates(location)
+ if x_value == None or y_value == None:
+ return "Sorry, but we couldn't find the coordinates to your specified item"
self.click(x_value, y_value)
@@ -225,6 +240,8 @@ def move_cursor_to_item(self, item: str):
f"Where on the provided screenshot is {item}? Provide the location in x and y coordinates. For example: 500x 200y"
)
x_value, y_value = self.extract_coordinates(location)
+ if x_value == None or y_value == None:
+ return "Sorry, but we couldn't find the coordinates to your specified item"
self.move_cursor_to(x_value, y_value)
@@ -241,7 +258,7 @@ def screenshot(self, need: str):
"""
screenshot_savepath = Path(
- f'DAIA/Screenshots/screenshot{"".join([str(e + randint(1, 9)) for e in range(10)])}.png'
+ f'Screenshots/screenshot{"".join([str(e + randint(1, 9)) for e in range(10)])}.png'
)
self.os_controller.screenshot(screenshot_savepath)
diff --git a/Versions/DAIA_GPT4V_FreeThink/OS_control/os_controller.py b/Versions/DAIA_GPT4V_FreeThink/Components/OS_control/os_controller.py
similarity index 100%
rename from Versions/DAIA_GPT4V_FreeThink/OS_control/os_controller.py
rename to Versions/DAIA_GPT4V_FreeThink/Components/OS_control/os_controller.py
diff --git a/Versions/DAIA_GPT4V_FreeThink/run.py b/Versions/DAIA_GPT4V_FreeThink/run.py
index a395fb3..e467d85 100644
--- a/Versions/DAIA_GPT4V_FreeThink/run.py
+++ b/Versions/DAIA_GPT4V_FreeThink/run.py
@@ -15,13 +15,10 @@
# along with this program. If not, see .
from openai import OpenAI
-from Versions.DAIA_GPT4V_FreeThink.Interpreter.interpreter import Interpreter
-from utils.setup import setup
+from Versions.DAIA_GPT4V_FreeThink.Components.Interpreter.interpreter import Interpreter
def run_f(api_key):
- setup()
-
interpreter = Interpreter(api_key)
client = OpenAI(
api_key=api_key,
diff --git a/Versions/DAIA_GPT4V_PreProgrammed/DVAI/GPT_4_with_Vision.py b/Versions/DAIA_GPT4V_PreProgrammed/Components/DVAI/GPT_4_with_Vision.py
similarity index 100%
rename from Versions/DAIA_GPT4V_PreProgrammed/DVAI/GPT_4_with_Vision.py
rename to Versions/DAIA_GPT4V_PreProgrammed/Components/DVAI/GPT_4_with_Vision.py
diff --git a/Versions/DAIA_GPT4V_PreProgrammed/Memory/memory.py b/Versions/DAIA_GPT4V_PreProgrammed/Components/Memory/memory.py
similarity index 100%
rename from Versions/DAIA_GPT4V_PreProgrammed/Memory/memory.py
rename to Versions/DAIA_GPT4V_PreProgrammed/Components/Memory/memory.py
diff --git a/Versions/DAIA_GPT4V_PreProgrammed/OS_control/os_controller.py b/Versions/DAIA_GPT4V_PreProgrammed/Components/OS_control/os_controller.py
similarity index 100%
rename from Versions/DAIA_GPT4V_PreProgrammed/OS_control/os_controller.py
rename to Versions/DAIA_GPT4V_PreProgrammed/Components/OS_control/os_controller.py
diff --git a/Versions/DAIA_GPT4V_PreProgrammed/Optimizer/optimization.py b/Versions/DAIA_GPT4V_PreProgrammed/Components/Optimizer/optimization.py
similarity index 100%
rename from Versions/DAIA_GPT4V_PreProgrammed/Optimizer/optimization.py
rename to Versions/DAIA_GPT4V_PreProgrammed/Components/Optimizer/optimization.py
diff --git a/Versions/DAIA_GPT4V_PreProgrammed/Thinker/thinking.py b/Versions/DAIA_GPT4V_PreProgrammed/Components/Thinker/thinking.py
similarity index 91%
rename from Versions/DAIA_GPT4V_PreProgrammed/Thinker/thinking.py
rename to Versions/DAIA_GPT4V_PreProgrammed/Components/Thinker/thinking.py
index c1580b6..c4fca30 100644
--- a/Versions/DAIA_GPT4V_PreProgrammed/Thinker/thinking.py
+++ b/Versions/DAIA_GPT4V_PreProgrammed/Components/Thinker/thinking.py
@@ -14,15 +14,17 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-from Versions.DAIA_GPT4V_PreProgrammed.Memory.memory import Memory
-from Versions.DAIA_GPT4V_PreProgrammed.OS_control.os_controller import OSController
-from Versions.DAIA_GPT4V_PreProgrammed.DVAI.GPT_4_with_Vision import DVAI
+from Versions.DAIA_GPT4V_PreProgrammed.Components.Memory.memory import Memory
+from Versions.DAIA_GPT4V_PreProgrammed.Components.OS_control.os_controller import (
+ OSController,
+)
+from Versions.DAIA_GPT4V_PreProgrammed.Components.DVAI.GPT_4_with_Vision import DVAI
from utils.setup import setup
from openai import OpenAI
from pathlib import Path
from random import randint
-
+print
class Think:
"""
The main class for operations involving the GPT for the DAIA
@@ -42,9 +44,7 @@ def goal_completer(self, suggestions: str):
Compleate goals
"""
- setup()
-
- # Setup system info and commands
+ # Setup system information and commands
os_controller = OSController()
system_info = os_controller.get_system_info()
commands = [
@@ -59,11 +59,11 @@ def goal_completer(self, suggestions: str):
for suggestion in first_suggestions:
# Take a screenshot and save it
screenshot_savepath = Path(
- f'DAIA/Screenshots/screenshot{"".join([str(e + randint(1, 9)) for e in range(10)])}.png'
+ f'Screenshots/screenshot{"".join([str(e + randint(1, 9)) for e in range(10)])}.png'
)
os_controller.screenshot(screenshot_savepath)
- # Get the current screen information with the screenshot (the prompt needs improvements)
+ # Get the current screen information with the screenshot (dev note: the prompt needs improvements)
prompt = f"""
Please state what is in the provided screenshot of the {str(system_info.get('OS'))} OS that relates to {suggestion} of the goal {self.goal}.
"""
@@ -427,6 +427,41 @@ def explanation_to_suggestions(self, explanation: str, prev_data: bool):
return suggestions
+ def explanation_is_suggestions(self, explanation):
+ '''
+ Is the explanation already made out of suggestions?
+
+ For example:
+ If explanation = "1. Go to the website, 2. Download the app, 3. Install it ....", then explanation_is_suggestion = true
+ If explanation = "You need to go to the website than download and install the app....", then explanation_is_suggestion = false
+ '''
+
+ prompt = f'''
+Given the explanation string, determine if it is composed of suggestions or not.
+Consider an explanation to be a suggestion if it follows the format of numbered steps starting with digits followed by a period, e.g., "1. Do something, 2. Do another thing, 3. Complete the process."
+
+explanatin string = {explanation}
+
+If the explanation string is a suggestion (or made out of suggestions) then type "is_suggestion"
+If the explanation string is NOT a suggestion (or made out of suggestions) then type "not_suggestion"
+'''
+ is_suggestions = self.client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[
+ {
+ "role": "user",
+ "content": prompt,
+ }
+ ],
+ )
+ is_suggestions = is_suggestions.choices[0].message.content
+
+ if is_suggestions.lower() == "is_suggestion":
+ return True
+
+ if is_suggestions.lower() == "not_suggestion":
+ return False
+
def short_remember(self, need: str):
"""
Remember a short period of history in detail from the DAIA MemoryDB,
diff --git a/Versions/DAIA_GPT4V_PreProgrammed/run.py b/Versions/DAIA_GPT4V_PreProgrammed/run.py
index 53a401b..03a5997 100644
--- a/Versions/DAIA_GPT4V_PreProgrammed/run.py
+++ b/Versions/DAIA_GPT4V_PreProgrammed/run.py
@@ -14,7 +14,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-from Versions.DAIA_GPT4V_PreProgrammed.Thinker.thinking import Think
+from Versions.DAIA_GPT4V_PreProgrammed.Components.Thinker.thinking import Think
from openai import OpenAI
@@ -77,7 +77,11 @@ def run(api_key):
goal_help = goal_help.choices[0].message.content
think.save_action(action1=question, action2=goal_help, category=0)
- suggestions = think.explanation_to_suggestions(goal_help, prev_data=False)
+ is_suggestions = think.explanation_is_suggestions(goal_help)
+ if not is_suggestions:
+ suggestions = think.explanation_to_suggestions(goal_help, prev_data=False)
+ else:
+ suggestions = goal_help
print(
f"""
General goal steps:
@@ -126,9 +130,11 @@ def run(api_key):
corrected_suggestions_response.choices[0].message.content
)
- suggestions = think.explanation_to_suggestions(
- corrected_suggestions_response, prev_data=False
- )
+ is_suggestions = think.explanation_is_suggestions(corrected_suggestions_response)
+ if not is_suggestions:
+ suggestions = think.explanation_to_suggestions(corrected_suggestions_response, prev_data=False)
+ else:
+ suggestions = corrected_suggestions_response
print(
f"""
General goal steps:
@@ -169,9 +175,11 @@ def run(api_key):
corrected_suggestions_response.choices[0].message.content
)
- suggestions = think.explanation_to_suggestions(
- corrected_suggestions_response, prev_data=False
- )
+ is_suggestions = think.explanation_is_suggestions(corrected_suggestions_response)
+ if not is_suggestions:
+ suggestions = think.explanation_to_suggestions(corrected_suggestions_response, prev_data=False)
+ else:
+ suggestions = corrected_suggestions_response
print(
f"""
General goal steps:
diff --git a/utils/setup.py b/utils/setup.py
index d912db8..0a53ee1 100644
--- a/utils/setup.py
+++ b/utils/setup.py
@@ -19,7 +19,7 @@
def setup():
- paths = [Path("DAIA/Screenshots")]
+ paths = [Path("Screenshots")]
for path in paths:
if not path.exists():