Vector memory revamp (part 1: refactoring)

.env.template

@@ -90,30 +90,18 @@ OPENAI_API_KEY=your-openai-api-key
 
 ### EMBEDDINGS
 ## EMBEDDING_MODEL       - Model to use for creating embeddings
-## EMBEDDING_TOKENIZER   - Tokenizer to use for chunking large inputs
-## EMBEDDING_TOKEN_LIMIT - Chunk size limit for large inputs
 # EMBEDDING_MODEL=text-embedding-ada-002
-# EMBEDDING_TOKENIZER=cl100k_base
-# EMBEDDING_TOKEN_LIMIT=8191
 
 ################################################################################
 ### MEMORY
 ################################################################################
 
 ### MEMORY_BACKEND - Memory backend type
-## local - Default
-## pinecone - Pinecone (if configured)
+## json_file - Default
 ## redis - Redis (if configured)
-## milvus - Milvus (if configured - also works with Zilliz)
 ## MEMORY_INDEX - Name of index created in Memory backend (Default: auto-gpt)
-# MEMORY_BACKEND=local
-# MEMORY_INDEX=auto-gpt
-
-### PINECONE
-## PINECONE_API_KEY - Pinecone API Key (Example: my-pinecone-api-key)
-## PINECONE_ENV - Pinecone environment (region) (Example: us-west-2)
-# PINECONE_API_KEY=your-pinecone-api-key
-# PINECONE_ENV=your-pinecone-region
+# MEMORY_BACKEND=json_file
+# MEMORY_INDEX=auto-gpt-memory
 
 ### REDIS
 ## REDIS_HOST - Redis host (Default: localhost, use "redis" for docker-compose)
@@ -125,38 +113,6 @@ OPENAI_API_KEY=your-openai-api-key
 # REDIS_PASSWORD=
 # WIPE_REDIS_ON_START=True
 
-### WEAVIATE
-## MEMORY_BACKEND - Use 'weaviate' to use Weaviate vector storage
-## WEAVIATE_HOST - Weaviate host IP
-## WEAVIATE_PORT - Weaviate host port
-## WEAVIATE_PROTOCOL - Weaviate host protocol (e.g. 'http')
-## USE_WEAVIATE_EMBEDDED - Whether to use Embedded Weaviate
-## WEAVIATE_EMBEDDED_PATH - File system path were to persist data when running Embedded Weaviate
-## WEAVIATE_USERNAME - Weaviate username
-## WEAVIATE_PASSWORD - Weaviate password
-## WEAVIATE_API_KEY - Weaviate API key if using API-key-based authentication
-# WEAVIATE_HOST="127.0.0.1"
-# WEAVIATE_PORT=8080
-# WEAVIATE_PROTOCOL="http"
-# USE_WEAVIATE_EMBEDDED=False
-# WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate"
-# WEAVIATE_USERNAME=
-# WEAVIATE_PASSWORD=
-# WEAVIATE_API_KEY=
-
-### MILVUS
-## MILVUS_ADDR - Milvus remote address (e.g. localhost:19530, https://xxx-xxxx.xxxx.xxxx.zillizcloud.com:443)
-## MILVUS_USERNAME - username for your Milvus database
-## MILVUS_PASSWORD - password for your Milvus database
-## MILVUS_SECURE - True to enable TLS. (Default: False)
-##   Setting MILVUS_ADDR to a `https://` URL will override this setting.
-## MILVUS_COLLECTION - Milvus collection, change it if you want to start a new memory and retain the old memory.
-# MILVUS_ADDR=localhost:19530
-# MILVUS_USERNAME=
-# MILVUS_PASSWORD=
-# MILVUS_SECURE=
-# MILVUS_COLLECTION=autogpt
-
 ################################################################################
 ### IMAGE GENERATION PROVIDER
 ################################################################################

BULLETIN.md

@@ -43,3 +43,11 @@ Auto-GPT now has support for plugins! With plugins, you can extend Auto-GPT's ab
 adding support for third-party services and more.
 See https://github.com/Significant-Gravitas/Auto-GPT-Plugins for instructions and
 available plugins. Specific plugins can be allowlisted/denylisted in .env.
+
+## Memory backend deprecation ⚠️
+The Milvus, Pinecone and Weaviate memory backends were rendered incompatible
+by work on the memory system, and have been removed in `master`. The Redis
+memory store was also temporarily removed but we aim to merge a new implementation
+before the next release.
+Whether built-in support for the others will be added back in the future is subject to
+discussion, feel free to pitch in: https://github.com/Significant-Gravitas/Auto-GPT/discussions/4280

autogpt/agent/agent.py

@@ -5,11 +5,14 @@
 from colorama import Fore, Style
 
 from autogpt.app import execute_command, get_command
+from autogpt.commands.command import CommandRegistry
 from autogpt.config import Config
+from autogpt.config.ai_config import AIConfig
 from autogpt.json_utils.json_fix_llm import fix_json_using_multiple_techniques
 from autogpt.json_utils.utilities import LLM_DEFAULT_RESPONSE_FORMAT, validate_json
-from autogpt.llm import chat_with_ai, create_chat_completion, create_chat_message
-from autogpt.llm.token_counter import count_string_tokens
+from autogpt.llm.base import ChatSequence
+from autogpt.llm.chat import chat_with_ai, create_chat_completion
+from autogpt.llm.utils import count_string_tokens
 from autogpt.log_cycle.log_cycle import (
     FULL_MESSAGE_HISTORY_FILE_NAME,
     NEXT_ACTION_FILE_NAME,
@@ -19,6 +22,8 @@
     LogCycleHandler,
 )
 from autogpt.logs import logger, print_assistant_thoughts
+from autogpt.memory.message_history import MessageHistory
+from autogpt.memory.vector import VectorMemory
 from autogpt.speech import say_text
 from autogpt.spinner import Spinner
 from autogpt.utils import clean_input
@@ -31,7 +36,6 @@
     Attributes:
         ai_name: The name of the agent.
         memory: The memory object to use.
-        full_message_history: The full message history.
         next_action_count: The number of actions to execute.
         system_prompt: The system prompt is the initial prompt that defines everything
           the AI needs to know to achieve its task successfully.
@@ -56,24 +60,19 @@
 
     def __init__(
         self,
-        ai_name,
-        memory,
-        full_message_history,
-        next_action_count,
-        command_registry,
-        config,
-        system_prompt,
-        triggering_prompt,
-        workspace_directory,
+        ai_name: str,
+        memory: VectorMemory,
+        next_action_count: int,
+        command_registry: CommandRegistry,
+        config: AIConfig,
+        system_prompt: str,
+        triggering_prompt: str,
+        workspace_directory: str,
     ):
         cfg = Config()
         self.ai_name = ai_name
         self.memory = memory
-        self.summary_memory = (
-            "I was created."  # Initial memory necessary to avoid hallucination
-        )
-        self.last_memory_index = 0
-        self.full_message_history = full_message_history
+        self.history = MessageHistory(self)
         self.next_action_count = next_action_count
         self.command_registry = command_registry
         self.config = config
@@ -114,7 +113,7 @@
                 self.config.ai_name,
                 self.created_at,
                 self.cycle_count,
-                self.full_message_history,
+                [m.raw() for m in self.history],
                 FULL_MESSAGE_HISTORY_FILE_NAME,
             )
             if (
@@ -132,8 +131,6 @@
                     self,
                     self.system_prompt,
                     self.triggering_prompt,
-                    self.full_message_history,
-                    self.memory,
                     cfg.fast_token_limit,
                 )  # TODO: This hardcodes the model to use GPT3.5. Make this an argument
 
@@ -260,9 +257,7 @@
 
             # Execute command
             if command_name is not None and command_name.lower().startswith("error"):
-                result = (
-                    f"Command {command_name} threw the following error: {arguments}"
-                )
+                result = f"Could not execute command: {arguments}"
             elif command_name == "human_feedback":
                 result = f"Human feedback: {user_input}"
             elif command_name == "self_feedback":
@@ -286,7 +281,7 @@
                     str(command_result), cfg.fast_llm_model
                 )
                 memory_tlength = count_string_tokens(
-                    str(self.summary_memory), cfg.fast_llm_model
+                    str(self.history.summary_message()), cfg.fast_llm_model
                 )
                 if result_tlength + memory_tlength + 600 > cfg.fast_token_limit:
                     result = f"Failure: command {command_name} returned too much output. \
@@ -302,12 +297,10 @@
             # Check if there's a result from the command append it to the message
             # history
             if result is not None:
-                self.full_message_history.append(create_chat_message("system", result))
+                self.history.add("system", result, "action_result")
                 logger.typewriter_log("SYSTEM: ", Fore.YELLOW, result)
             else:
-                self.full_message_history.append(
-                    create_chat_message("system", "Unable to execute command")
-                )
+                self.history.add("system", "Unable to execute command", "action_result")
                 logger.typewriter_log(
                     "SYSTEM: ", Fore.YELLOW, "Unable to execute command"
                 )
@@ -343,17 +336,18 @@
         thought = thoughts.get("thoughts", "")
         feedback_thoughts = thought + reasoning + plan
 
-        messages = [{"role": "user", "content": feedback_prompt + feedback_thoughts}]
+        prompt = ChatSequence.for_model(llm_model)
+        prompt.add("user", feedback_prompt + feedback_thoughts)
 
         self.log_cycle_handler.log_cycle(
             self.config.ai_name,
             self.created_at,
             self.cycle_count,
-            messages,
+            prompt.raw(),
             PROMPT_SUPERVISOR_FEEDBACK_FILE_NAME,
         )
 
-        feedback = create_chat_completion(messages, model=llm_model)
+        feedback = create_chat_completion(prompt)
 
         self.log_cycle_handler.log_cycle(
             self.config.ai_name,

autogpt/agent/agent_manager.py

@@ -1,10 +1,9 @@
 """Agent manager for managing GPT agents"""
 from __future__ import annotations
 
-from typing import List
-
-from autogpt.config.config import Config
-from autogpt.llm import Message, create_chat_completion
+from autogpt.config import Config
+from autogpt.llm.base import ChatSequence
+from autogpt.llm.chat import Message, create_chat_completion
 from autogpt.singleton import Singleton
 
 
@@ -13,55 +12,55 @@
 
     def __init__(self):
         self.next_key = 0
-        self.agents = {}  # key, (task, full_message_history, model)
+        self.agents: dict[
+            int, tuple[str, list[Message], str]
+        ] = {}  # key, (task, full_message_history, model)
         self.cfg = Config()
 
     # Create new GPT agent
     # TODO: Centralise use of create_chat_completion() to globally enforce token limit
 
-    def create_agent(self, task: str, prompt: str, model: str) -> tuple[int, str]:
+    def create_agent(
+        self, task: str, creation_prompt: str, model: str
+    ) -> tuple[int, str]:
         """Create a new agent and return its key
 
         Args:
             task: The task to perform
-            prompt: The prompt to use
-            model: The model to use
+            creation_prompt: Prompt passed to the LLM at creation
+            model: The model to use to run this agent
 
         Returns:
             The key of the new agent
         """
-        messages: List[Message] = [
-            {"role": "user", "content": prompt},
-        ]
+        messages = ChatSequence.for_model(model, [Message("user", creation_prompt)])
+
         for plugin in self.cfg.plugins:
             if not plugin.can_handle_pre_instruction():
                 continue
-            if plugin_messages := plugin.pre_instruction(messages):
-                messages.extend(iter(plugin_messages))
+            if plugin_messages := plugin.pre_instruction(messages.raw()):
+                messages.extend([Message(**raw_msg) for raw_msg in plugin_messages])
         # Start GPT instance
-        agent_reply = create_chat_completion(
-            model=model,
-            messages=messages,
-        )
+        agent_reply = create_chat_completion(prompt=messages)
 
-        messages.append({"role": "assistant", "content": agent_reply})
+        messages.add("assistant", agent_reply)
 
         plugins_reply = ""
         for i, plugin in enumerate(self.cfg.plugins):
             if not plugin.can_handle_on_instruction():
                 continue
-            if plugin_result := plugin.on_instruction(messages):
+            if plugin_result := plugin.on_instruction([m.raw() for m in messages]):
                 sep = "\n" if i else ""
                 plugins_reply = f"{plugins_reply}{sep}{plugin_result}"
 
         if plugins_reply and plugins_reply != "":
-            messages.append({"role": "assistant", "content": plugins_reply})
+            messages.add("assistant", plugins_reply)
         key = self.next_key
         # This is done instead of len(agents) to make keys unique even if agents
         # are deleted
         self.next_key += 1
 
-        self.agents[key] = (task, messages, model)
+        self.agents[key] = (task, list(messages), model)
 
         for plugin in self.cfg.plugins:
             if not plugin.can_handle_post_instruction():
@@ -83,33 +82,30 @@
         task, messages, model = self.agents[int(key)]
 
         # Add user message to message history before sending to agent
-        messages.append({"role": "user", "content": message})
+        messages = ChatSequence.for_model(model, messages)
+        messages.add("user", message)
 
         for plugin in self.cfg.plugins:
             if not plugin.can_handle_pre_instruction():
                 continue
-            if plugin_messages := plugin.pre_instruction(messages):
-                for plugin_message in plugin_messages:
-                    messages.append(plugin_message)
+            if plugin_messages := plugin.pre_instruction([m.raw() for m in messages]):
+                messages.extend([Message(**raw_msg) for raw_msg in plugin_messages])
 
         # Start GPT instance
-        agent_reply = create_chat_completion(
-            model=model,
-            messages=messages,
-        )
+        agent_reply = create_chat_completion(prompt=messages)
 
-        messages.append({"role": "assistant", "content": agent_reply})
+        messages.add("assistant", agent_reply)
 
         plugins_reply = agent_reply
         for i, plugin in enumerate(self.cfg.plugins):
             if not plugin.can_handle_on_instruction():
                 continue
-            if plugin_result := plugin.on_instruction(messages):
+            if plugin_result := plugin.on_instruction([m.raw() for m in messages]):
                 sep = "\n" if i else ""
                 plugins_reply = f"{plugins_reply}{sep}{plugin_result}"
         # Update full message history
         if plugins_reply and plugins_reply != "":
-            messages.append({"role": "assistant", "content": plugins_reply})
+            messages.add("assistant", plugins_reply)
 
         for plugin in self.cfg.plugins:
             if not plugin.can_handle_post_instruction():