fix: local llm support #100

OpenBMB · Nov 13, 2023 · c16ee0a · c16ee0a
1 parent ffc3d39
commit c16ee0a
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/agentverse/llms/openai.py b/agentverse/llms/openai.py
@@ -113,6 +113,7 @@ def __init__(self, max_retry: int = 3, **kwargs):
             logger.warn(f"Unused arguments: {kwargs}")
         if args["model"] in LOCAL_LLMS:
             openai.api_base = "http://localhost:5000/v1"
+            openai.api_key = "EMPTY"
         super().__init__(args=args, max_retry=max_retry)
 
     @classmethod

diff --git a/scripts/run_local_model_server.sh b/scripts/run_local_model_server.sh
@@ -3,8 +3,12 @@
 # export CUDA_VISIBLE_DEVICES=0
 MODEL_PATH="meta-llama/Llama-2-7b-chat-hf"   # path_to_the_downloaded_model_dir
 MODEL_NAME="llama-2-7b-chat-hf"              # name_of_the_model
-python3 -m fastchat.serve.controller & \
+CONTROLLER_PORT=20002
+python3 -m fastchat.serve.controller --host 127.0.0.1 --port ${CONTROLLER_PORT} & \
 python3 -m fastchat.serve.multi_model_worker \
     --model-path ${MODEL_PATH} \
-    --model-names ${MODEL_NAME} & \
-python3 -m fastchat.serve.openai_api_server --host localhost --port 5000
+    --model-names ${MODEL_NAME} \
+    --host 127.0.0.1 \
+    --controller-address http://127.0.0.1:${CONTROLLER_PORT} \
+    --worker-address http://127.0.0.1:21002 & \
+python3 -m fastchat.serve.openai_api_server --host 127.0.0.1 --port 5000 --controller-address http://127.0.0.1:${CONTROLLER_PORT}