feat(agent/core): Add Anthropic Claude 3 support (#7085)

- feat(agent/core): Add `AnthropicProvider`
  - Add `ANTHROPIC_API_KEY` to .env.template and docs

  Notable differences in logic compared to `OpenAIProvider`:
  - Merges subsequent user messages in `AnthropicProvider._get_chat_completion_args`
  - Merges and extracts all system messages into `system` parameter in `AnthropicProvider._get_chat_completion_args`
  - Supports prefill; merges prefill content (if any) into generated response

- Prompt changes to improve compatibility with `AnthropicProvider`
  Anthropic has a slightly different API compared to OpenAI, and has much stricter input validation. E.g. Anthropic only supports a single `system` prompt, where OpenAI allows multiple `system` messages. Anthropic also forbids sequences of multiple `user` or `assistant` messages and requires that messages alternate between roles.
  - Move response format instruction from separate message into main system prompt
  - Fix clock message format
  - Add pre-fill to `OneShot` generated prompt

- refactor(agent/core): Tweak `model_providers.schema`
  - Simplify `ModelProviderUsage`
     - Remove attribute `total_tokens` as it is always equal to `prompt_tokens + completion_tokens`
     - Modify signature of `update_usage(..)`; no longer requires a full `ModelResponse` object as input
  - Improve `ModelProviderBudget`
     - Change type of attribute `usage` to `defaultdict[str, ModelProviderUsage]` -> allow per-model usage tracking
     - Modify signature of `update_usage_and_cost(..)`; no longer requires a full `ModelResponse` object as input
     - Allow `ModelProviderBudget` zero-argument instantiation
  - Fix type of `AssistantChatMessage.role` to match `ChatMessage.role` (str -> `ChatMessage.Role`)
  - Add shared attributes and constructor to `ModelProvider` base class
  - Add `max_output_tokens` parameter to `create_chat_completion` interface
  - Add pre-filling as a global feature
    - Add `prefill_response` field to `ChatPrompt` model
    - Add `prefill_response` parameter to `create_chat_completion` interface
  - Add `ChatModelProvider.get_available_models()` and remove `ApiManager`
  - Remove unused `OpenAIChatParser` typedef in openai.py
  - Remove redundant `budget` attribute definition on `OpenAISettings`
  - Remove unnecessary `usage` in `OpenAIProvider` > `default_settings` > `budget`

- feat(agent): Allow use of any available LLM provider through `MultiProvider`
  - Add `MultiProvider` (`model_providers.multi`)
  - Replace all references to / uses of `OpenAIProvider` with `MultiProvider`
  - Change type of `Config.smart_llm` and `Config.fast_llm` from `str` to `ModelName`

- feat(agent/core): Validate function call arguments in `create_chat_completion`
    - Add `validate_call` method to `CompletionModelFunction` in `model_providers.schema`
    - Add `validate_tool_calls` utility function in `model_providers.utils`
    - Add tool call validation step to `create_chat_completion` in `OpenAIProvider` and `AnthropicProvider`
    - Remove (now redundant) command argument validation logic in agent.py and models/command.py

- refactor(agent): Rename `get_openai_command_specs` to `function_specs_from_commands`

autogpts/autogpt/.env.template

@@ -2,8 +2,11 @@
 ### AutoGPT - GENERAL SETTINGS
 ################################################################################
 
-## OPENAI_API_KEY - OpenAI API Key (Example: my-openai-api-key)
-OPENAI_API_KEY=your-openai-api-key
+## OPENAI_API_KEY - OpenAI API Key (Example: sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx)
+# OPENAI_API_KEY=
+
+## ANTHROPIC_API_KEY - Anthropic API Key (Example: sk-ant-api03-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx)
+# ANTHROPIC_API_KEY=
 
 ## TELEMETRY_OPT_IN - Share telemetry on errors and other issues with the AutoGPT team, e.g. through Sentry.
 ##   This helps us to spot and solve problems earlier & faster. (Default: DISABLED)

autogpts/autogpt/agbenchmark_config/benchmarks.py

@@ -5,8 +5,7 @@
 
 from autogpt.agent_manager.agent_manager import AgentManager
 from autogpt.agents.agent import Agent, AgentConfiguration, AgentSettings
-from autogpt.agents.prompt_strategies.one_shot import OneShotAgentPromptStrategy
-from autogpt.app.main import _configure_openai_provider, run_interaction_loop
+from autogpt.app.main import _configure_llm_provider, run_interaction_loop
 from autogpt.config import AIProfile, ConfigBuilder
 from autogpt.file_storage import FileStorageBackendName, get_storage
 from autogpt.logs.config import configure_logging
@@ -38,10 +37,6 @@ def bootstrap_agent(task: str, continuous_mode: bool) -> Agent:
         ai_goals=[task],
     )
 
-    agent_prompt_config = OneShotAgentPromptStrategy.default_configuration.copy(
-        deep=True
-    )
-    agent_prompt_config.use_functions_api = config.openai_functions
     agent_settings = AgentSettings(
         name=Agent.default_settings.name,
         agent_id=AgentManager.generate_id("AutoGPT-benchmark"),
@@ -53,7 +48,6 @@ def bootstrap_agent(task: str, continuous_mode: bool) -> Agent:
             allow_fs_access=not config.restrict_to_workspace,
             use_functions_api=config.openai_functions,
         ),
-        prompt_config=agent_prompt_config,
         history=Agent.default_settings.history.copy(deep=True),
     )
 
@@ -66,7 +60,7 @@ def bootstrap_agent(task: str, continuous_mode: bool) -> Agent:
 
     agent = Agent(
         settings=agent_settings,
-        llm_provider=_configure_openai_provider(config),
+        llm_provider=_configure_llm_provider(config),
         file_storage=file_storage,
         legacy_config=config,
     )

autogpts/autogpt/autogpt/agents/agent.py

@@ -19,15 +19,14 @@
 from autogpt.core.configuration import Configurable
 from autogpt.core.prompting import ChatPrompt
 from autogpt.core.resource.model_providers import (
-    AssistantChatMessage,
     AssistantFunctionCall,
     ChatMessage,
     ChatModelProvider,
     ChatModelResponse,
 )
 from autogpt.core.runner.client_lib.logging.helpers import dump_prompt
 from autogpt.file_storage.base import FileStorage
-from autogpt.llm.providers.openai import get_openai_command_specs
+from autogpt.llm.providers.openai import function_specs_from_commands
 from autogpt.logs.log_cycle import (
     CURRENT_CONTEXT_FILE_NAME,
     NEXT_ACTION_FILE_NAME,
@@ -46,7 +45,6 @@
     AgentException,
     AgentTerminated,
     CommandExecutionError,
-    InvalidArgumentError,
     UnknownCommandError,
 )
 
@@ -104,7 +102,11 @@ def __init__(
         self.ai_profile = settings.ai_profile
         self.directives = settings.directives
         prompt_config = OneShotAgentPromptStrategy.default_configuration.copy(deep=True)
-        prompt_config.use_functions_api = settings.config.use_functions_api
+        prompt_config.use_functions_api = (
+            settings.config.use_functions_api
+            # Anthropic currently doesn't support tools + prefilling :(
+            and self.llm.provider_name != "anthropic"
+        )
         self.prompt_strategy = OneShotAgentPromptStrategy(prompt_config, logger)
         self.commands: list[Command] = []
 
@@ -172,7 +174,7 @@ async def propose_action(self) -> OneShotAgentActionProposal:
             task=self.state.task,
             ai_profile=self.state.ai_profile,
             ai_directives=directives,
-            commands=get_openai_command_specs(self.commands),
+            commands=function_specs_from_commands(self.commands),
             include_os_info=self.legacy_config.execute_local_commands,
         )
 
@@ -202,12 +204,9 @@ async def complete_and_parse(
         ] = await self.llm_provider.create_chat_completion(
             prompt.messages,
             model_name=self.llm.name,
-            completion_parser=self.parse_and_validate_response,
-            functions=(
-                get_openai_command_specs(self.commands)
-                if self.config.use_functions_api
-                else []
-            ),
+            completion_parser=self.prompt_strategy.parse_response_content,
+            functions=prompt.functions,
+            prefill_response=prompt.prefill_response,
         )
         result = response.parsed_result
 
@@ -223,28 +222,6 @@ async def complete_and_parse(
 
         return result
 
-    def parse_and_validate_response(
-        self, llm_response: AssistantChatMessage
-    ) -> OneShotAgentActionProposal:
-        parsed_response = self.prompt_strategy.parse_response_content(llm_response)
-
-        # Validate command arguments
-        command_name = parsed_response.use_tool.name
-        command = self._get_command(command_name)
-        if arg_errors := command.validate_args(parsed_response.use_tool.arguments)[1]:
-            fmt_errors = [
-                f"{'.'.join(str(p) for p in f.path)}: {f.message}"
-                if f.path
-                else f.message
-                for f in arg_errors
-            ]
-            raise InvalidArgumentError(
-                f"The set of arguments supplied for {command_name} is invalid:\n"
-                + "\n".join(fmt_errors)
-            )
-
-        return parsed_response
-
     async def execute(
         self,
         proposal: OneShotAgentActionProposal,

autogpts/autogpt/autogpt/agents/base.py

@@ -39,11 +39,12 @@
     SystemSettings,
     UserConfigurable,
 )
-from autogpt.core.resource.model_providers import AssistantFunctionCall
-from autogpt.core.resource.model_providers.openai import (
-    OPEN_AI_CHAT_MODELS,
-    OpenAIModelName,
+from autogpt.core.resource.model_providers import (
+    CHAT_MODELS,
+    AssistantFunctionCall,
+    ModelName,
 )
+from autogpt.core.resource.model_providers.openai import OpenAIModelName
 from autogpt.models.utils import ModelWithSummary
 from autogpt.prompts.prompt import DEFAULT_TRIGGERING_PROMPT
 
@@ -56,8 +57,8 @@
 class BaseAgentConfiguration(SystemConfiguration):
     allow_fs_access: bool = UserConfigurable(default=False)
 
-    fast_llm: OpenAIModelName = UserConfigurable(default=OpenAIModelName.GPT3_16k)
-    smart_llm: OpenAIModelName = UserConfigurable(default=OpenAIModelName.GPT4)
+    fast_llm: ModelName = UserConfigurable(default=OpenAIModelName.GPT3_16k)
+    smart_llm: ModelName = UserConfigurable(default=OpenAIModelName.GPT4)
     use_functions_api: bool = UserConfigurable(default=False)
 
     default_cycle_instruction: str = DEFAULT_TRIGGERING_PROMPT
@@ -174,7 +175,7 @@ def llm(self) -> ChatModelInfo:
         llm_name = (
             self.config.smart_llm if self.config.big_brain else self.config.fast_llm
         )
-        return OPEN_AI_CHAT_MODELS[llm_name]
+        return CHAT_MODELS[llm_name]
 
     @property
     def send_token_limit(self) -> int:

autogpts/autogpt/autogpt/agents/prompt_strategies/one_shot.py

@@ -122,7 +122,7 @@ def build_prompt(
         1. System prompt
         3. `cycle_instruction`
         """
-        system_prompt = self.build_system_prompt(
+        system_prompt, response_prefill = self.build_system_prompt(
             ai_profile=ai_profile,
             ai_directives=ai_directives,
             commands=commands,
@@ -131,24 +131,34 @@ def build_prompt(
 
         final_instruction_msg = ChatMessage.user(self.config.choose_action_instruction)
 
-        prompt = ChatPrompt(
+        return ChatPrompt(
             messages=[
                 ChatMessage.system(system_prompt),
                 ChatMessage.user(f'"""{task}"""'),
                 *messages,
                 final_instruction_msg,
             ],
+            prefill_response=response_prefill,
+            functions=commands if self.config.use_functions_api else [],
         )
 
-        return prompt
-
     def build_system_prompt(
         self,
         ai_profile: AIProfile,
         ai_directives: AIDirectives,
         commands: list[CompletionModelFunction],
         include_os_info: bool,
-    ) -> str:
+    ) -> tuple[str, str]:
+        """
+        Builds the system prompt.
+
+        Returns:
+            str: The system prompt body
+            str: The desired start for the LLM's response; used to steer the output
+        """
+        response_fmt_instruction, response_prefill = self.response_format_instruction(
+            self.config.use_functions_api
+        )
         system_prompt_parts = (
             self._generate_intro_prompt(ai_profile)
             + (self._generate_os_info() if include_os_info else [])
@@ -169,16 +179,16 @@ def build_system_prompt(
                 " in the next message. Your job is to complete the task while following"
                 " your directives as given above, and terminate when your task is done."
             ]
-            + [
-                "## RESPONSE FORMAT\n"
-                + self.response_format_instruction(self.config.use_functions_api)
-            ]
+            + ["## RESPONSE FORMAT\n" + response_fmt_instruction]
         )
 
         # Join non-empty parts together into paragraph format
-        return "\n\n".join(filter(None, system_prompt_parts)).strip("\n")
+        return (
+            "\n\n".join(filter(None, system_prompt_parts)).strip("\n"),
+            response_prefill,
+        )
 
-    def response_format_instruction(self, use_functions_api: bool) -> str:
+    def response_format_instruction(self, use_functions_api: bool) -> tuple[str, str]:
         response_schema = self.response_schema.copy(deep=True)
         if (
             use_functions_api
@@ -193,11 +203,15 @@ def response_format_instruction(self, use_functions_api: bool) -> str:
             "\n",
             response_schema.to_typescript_object_interface(_RESPONSE_INTERFACE_NAME),
         )
+        response_prefill = f'{{\n    "{list(response_schema.properties.keys())[0]}":'
 
         return (
-            f"YOU MUST ALWAYS RESPOND WITH A JSON OBJECT OF THE FOLLOWING TYPE:\n"
-            f"{response_format}"
-            + ("\n\nYOU MUST ALSO INVOKE A TOOL!" if use_functions_api else "")
+            (
+                f"YOU MUST ALWAYS RESPOND WITH A JSON OBJECT OF THE FOLLOWING TYPE:\n"
+                f"{response_format}"
+                + ("\n\nYOU MUST ALSO INVOKE A TOOL!" if use_functions_api else "")
+            ),
+            response_prefill,
         )
 
     def _generate_intro_prompt(self, ai_profile: AIProfile) -> list[str]:

autogpts/autogpt/autogpt/app/agent_protocol_server.py

@@ -34,7 +34,6 @@
 from autogpt.app.utils import is_port_free
 from autogpt.config import Config
 from autogpt.core.resource.model_providers import ChatModelProvider, ModelProviderBudget
-from autogpt.core.resource.model_providers.openai import OpenAIProvider
 from autogpt.file_storage import FileStorage
 from autogpt.models.action_history import ActionErrorResult, ActionSuccessResult
 from autogpt.utils.exceptions import AgentFinished
@@ -464,20 +463,18 @@ def _get_task_llm_provider(
         if task.additional_input and (user_id := task.additional_input.get("user_id")):
             _extra_request_headers["AutoGPT-UserID"] = user_id
 
-        task_llm_provider = None
-        if isinstance(self.llm_provider, OpenAIProvider):
-            settings = self.llm_provider._settings.copy()
-            settings.budget = task_llm_budget
-            settings.configuration = task_llm_provider_config  # type: ignore
-            task_llm_provider = OpenAIProvider(
-                settings=settings,
-                logger=logger.getChild(f"Task-{task.task_id}_OpenAIProvider"),
-            )
-
-        if task_llm_provider and task_llm_provider._budget:
-            self._task_budgets[task.task_id] = task_llm_provider._budget
+        settings = self.llm_provider._settings.copy()
+        settings.budget = task_llm_budget
+        settings.configuration = task_llm_provider_config
+        task_llm_provider = self.llm_provider.__class__(
+            settings=settings,
+            logger=logger.getChild(
+                f"Task-{task.task_id}_{self.llm_provider.__class__.__name__}"
+            ),
+        )
+        self._task_budgets[task.task_id] = task_llm_provider._budget  # type: ignore
 
-        return task_llm_provider or self.llm_provider
+        return task_llm_provider
 
 
 def task_agent_id(task_id: str | int) -> str:

autogpts/autogpt/autogpt/app/configurator.py

@@ -10,7 +10,7 @@
 
 from autogpt.config import Config
 from autogpt.config.config import GPT_3_MODEL, GPT_4_MODEL
-from autogpt.core.resource.model_providers.openai import OpenAIModelName, OpenAIProvider
+from autogpt.core.resource.model_providers import ModelName, MultiProvider
 from autogpt.logs.helpers import request_user_double_check
 from autogpt.memory.vector import get_supported_memory_backends
 from autogpt.utils import utils
@@ -150,11 +150,11 @@ async def apply_overrides_to_config(
 
 
 async def check_model(
-    model_name: OpenAIModelName, model_type: Literal["smart_llm", "fast_llm"]
-) -> OpenAIModelName:
+    model_name: ModelName, model_type: Literal["smart_llm", "fast_llm"]
+) -> ModelName:
     """Check if model is available for use. If not, return gpt-3.5-turbo."""
-    openai = OpenAIProvider()
-    models = await openai.get_available_models()
+    multi_provider = MultiProvider()
+    models = await multi_provider.get_available_models()
 
     if any(model_name == m.name for m in models):
         return model_name

autogpts/autogpt/autogpt/app/main.py

@@ -35,7 +35,7 @@
     ConfigBuilder,
     assert_config_has_openai_api_key,
 )
-from autogpt.core.resource.model_providers.openai import OpenAIProvider
+from autogpt.core.resource.model_providers import MultiProvider
 from autogpt.core.runner.client_lib.utils import coroutine
 from autogpt.file_storage import FileStorageBackendName, get_storage
 from autogpt.logs.config import configure_logging
@@ -123,7 +123,7 @@ async def run_auto_gpt(
         skip_news=skip_news,
     )
 
-    llm_provider = _configure_openai_provider(config)
+    llm_provider = _configure_llm_provider(config)
 
     logger = logging.getLogger(__name__)
 
@@ -399,7 +399,7 @@ async def run_auto_gpt_server(
         allow_downloads=allow_downloads,
     )
 
-    llm_provider = _configure_openai_provider(config)
+    llm_provider = _configure_llm_provider(config)
 
     # Set up & start server
     database = AgentDB(
@@ -421,24 +421,12 @@ async def run_auto_gpt_server(
     )
 
 
-def _configure_openai_provider(config: Config) -> OpenAIProvider:
-    """Create a configured OpenAIProvider object.
-
-    Args:
-        config: The program's configuration.
-
-    Returns:
-        A configured OpenAIProvider object.
-    """
-    if config.openai_credentials is None:
-        raise RuntimeError("OpenAI key is not configured")
-
-    openai_settings = OpenAIProvider.default_settings.copy(deep=True)
-    openai_settings.credentials = config.openai_credentials
-    return OpenAIProvider(
-        settings=openai_settings,
-        logger=logging.getLogger("OpenAIProvider"),
-    )
+def _configure_llm_provider(config: Config) -> MultiProvider:
+    multi_provider = MultiProvider()
+    for model in [config.smart_llm, config.fast_llm]:
+        # Ensure model providers for configured LLMs are available
+        multi_provider.get_model_provider(model)
+    return multi_provider
 
 
 def _get_cycle_budget(continuous_mode: bool, continuous_limit: int) -> int | float: