diff --git a/config/config2.yaml.example b/config/config2.yaml.example index 2217f1b2c..6d43ede51 100644 --- a/config/config2.yaml.example +++ b/config/config2.yaml.example @@ -3,6 +3,7 @@ llm: base_url: "YOUR_BASE_URL" api_key: "YOUR_API_KEY" model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview + openai_session_key: "" # If you want to automate the setup for RPM and MAX_TOKENS, please log in through your own browser at https://platform.openai.com/account/limits, enable developer mode, and find 'rate_limits' in the network section. If it's not found, refresh the recording using Ctrl+R. After locating 'rate_limits', find 'Authorization: sess-xxx' in the request header, where 'sess-xxx' is the session key.If you wish to manually set RPM, please set RPM and either do not set OPENAI_SESSION_KEY or set it to "". proxy: "YOUR_PROXY" diff --git a/metagpt/configs/llm_config.py b/metagpt/configs/llm_config.py index fb923d3e4..d35f660df 100644 --- a/metagpt/configs/llm_config.py +++ b/metagpt/configs/llm_config.py @@ -40,6 +40,7 @@ class LLMConfig(YamlModel): api_type: LLMType = LLMType.OPENAI base_url: str = "https://api.openai.com/v1" api_version: Optional[str] = None + openai_session_key: Optional[str] = None model: Optional[str] = None # also stands for DEPLOYMENT_NAME @@ -63,6 +64,7 @@ class LLMConfig(YamlModel): logprobs: Optional[bool] = None # https://cookbook.openai.com/examples/using_logprobs top_logprobs: Optional[int] = None timeout: int = 60 + rpm: int = 10 # For Network proxy: Optional[str] = None diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index fe41fb05f..c8cac073c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -9,6 +9,9 @@ import json from typing import AsyncIterator, Optional, Union +from typing import AsyncIterator, Union + +import openai from openai import APIConnectionError, AsyncOpenAI, AsyncStream from openai._base_client import AsyncHttpxClientWrapper @@ -30,6 +33,8 @@ from metagpt.schema import Message from metagpt.utils.common import CodeParser, decode_image from metagpt.utils.cost_manager import CostManager, Costs +from metagpt.utils.ahttp_client import aget +from metagpt.utils.cost_manager import Costs from metagpt.utils.exceptions import handle_exception from metagpt.utils.token_counter import ( count_message_tokens, @@ -61,7 +66,7 @@ def __init__(self, config: LLMConfig): self.cost_manager: Optional[CostManager] = None def _init_model(self): - self.model = self.config.model # Used in _calc_usage & _cons_kwargs + self.model = self.config.llm.model # Used in _calc_usage & _cons_kwargs def _init_client(self): """https://github.com/openai/openai-python#async-usage""" @@ -69,7 +74,7 @@ def _init_client(self): self.aclient = AsyncOpenAI(**kwargs) def _make_client_kwargs(self) -> dict: - kwargs = {"api_key": self.config.api_key, "base_url": self.config.base_url} + kwargs = {"api_key": self.config.llm.api_key, "base_url": self.config.llm.base_url} # to use proxy, openai v1 needs http_client if proxy_params := self._get_proxy_params(): @@ -290,3 +295,54 @@ async def gen_image( img_url_or_b64 = item.url if resp_format == "url" else item.b64_json imgs.append(decode_image(img_url_or_b64)) return imgs + + async def update_rpm(self): + """ + Asynchronously updates the RPM (requests per minute) limit. + + This method fetches the RPM limit from an external API and updates the rate limiting parameters. + It is designed to be run before making any batched API calls. + """ + self.rpm = await self._aget_rpm() + self.interval = 1.1 * 60 / self.rpm + logger.info(f'Setting rpm to {self.rpm}') + + async def _aget_rpm(self) -> int: + """ + Asynchronously fetches the RPM (requests per minute) limit from an external API. + + This is an internal method used by update_rpm to fetch the current RPM limit. It uses + the OPENAI_SESSION_KEY for authorization and falls back to a default RPM value in case of failure. + + Returns: + int: The fetched or default RPM value. + """ + session_key = self.config.llm.openai_session_key + default_rpm = self.config.llm.rpm + if len(session_key) > 0: + try: + response = await aget( + url="https://api.openai.com/dashboard/rate_limits", + headers={"Authorization": f"Bearer {session_key}"}, + proxy=self.config.llm.proxy + ) + response_content = json.loads(response) + if not "error" in response_content: + if self.model not in response_content: + raise ValueError("Get rpm from api.openai.com error. \ + You have entered a model name that is not supported by OpenAI, or the input is incorrect. \ + Please enter the correct name in the configuration file. \ + Setting rpm to default parameter.") + + limit_dict = response_content[self.model] + return limit_dict["max_requests_per_1_minute"] + else: + error = response_content["error"] + logger.error(f"Connection to api.openai.com failed:{error}.Setting rpm to default parameter.") + return default_rpm + + except Exception as exp: + logger.error(f"Connection to api.openai.com failed, error type:{type(exp).__name__}, error message:{str(exp)}.Setting rpm to default parameter.") + return default_rpm + else: + return default_rpm diff --git a/metagpt/utils/ahttp_client.py b/metagpt/utils/ahttp_client.py index b4a33e9d7..a45bf51a4 100644 --- a/metagpt/utils/ahttp_client.py +++ b/metagpt/utils/ahttp_client.py @@ -28,6 +28,25 @@ async def apost( return data +async def aget( + url: str, + params: Optional[Mapping[str, str]] = None, + headers: Optional[dict] = None, + as_json: bool = False, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total, + proxy: str = None, +) -> Union[str, dict]: + async with aiohttp.ClientSession() as session: + async with session.get(url=url, params=params, headers=headers, timeout=timeout, proxy=proxy) as resp: + if as_json: + data = await resp.json() + else: + data = await resp.read() + data = data.decode(encoding) + return data + + async def apost_stream( url: str, params: Optional[Mapping[str, str]] = None, diff --git a/tests/metagpt/provider/test_openai_api.py b/tests/metagpt/provider/test_openai_api.py new file mode 100644 index 000000000..de26d01fe --- /dev/null +++ b/tests/metagpt/provider/test_openai_api.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/18 19:08 +@Author : zhongyang +@File : test_openai_api.py +""" + +import pytest + +from metagpt.logs import logger +from metagpt.provider.openai_api import OpenAILLM +from metagpt.config2 import config + +@pytest.mark.asyncio +async def test_update_rpm(): + llm = OpenAILLM(config) + await llm.update_rpm() + assert isinstance(llm.rpm, float) \ No newline at end of file