From 6aea574394a67ad31383c2d221162bd9df53cf4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Gon=C3=A7alves?= Date: Fri, 6 Sep 2024 00:19:53 +0100 Subject: [PATCH 1/5] Added event tracking for server startup --- llmstudio/engine/__init__.py | 8 +++++--- llmstudio/llm/__init__.py | 4 +--- llmstudio/server.py | 6 ++++-- llmstudio/tracking/__init__.py | 8 +++++--- llmstudio/ui/__init__.py | 5 +++-- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/llmstudio/engine/__init__.py b/llmstudio/engine/__init__.py index 3f503ae3..83767c87 100644 --- a/llmstudio/engine/__init__.py +++ b/llmstudio/engine/__init__.py @@ -9,6 +9,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse from pydantic import BaseModel, ValidationError +from threading import Event from llmstudio.config import ENGINE_HOST, ENGINE_PORT from llmstudio.engine.providers import * @@ -78,7 +79,7 @@ def _merge_configs(config1, config2): raise RuntimeError(f"Error in configuration data: {e}") -def create_engine_app(config: EngineConfig = _load_engine_config()) -> FastAPI: +def create_engine_app(started_event: Event, config: EngineConfig = _load_engine_config()) -> FastAPI: app = FastAPI( title=ENGINE_TITLE, description=ENGINE_DESCRIPTION, @@ -162,14 +163,15 @@ async def export(request: Request): @app.on_event("startup") async def startup_event(): + started_event.set() print(f"Running LLMstudio Engine on http://{ENGINE_HOST}:{ENGINE_PORT} ") return app -def run_engine_app(): +def run_engine_app(started_event : Event): try: - engine = create_engine_app() + engine = create_engine_app(started_event) uvicorn.run( engine, host=ENGINE_HOST, diff --git a/llmstudio/llm/__init__.py b/llmstudio/llm/__init__.py index 8f42086b..33b2de63 100644 --- a/llmstudio/llm/__init__.py +++ b/llmstudio/llm/__init__.py @@ -9,11 +9,9 @@ from llmstudio.llm.semaphore import DynamicSemaphore from llmstudio.server import start_server -start_server() - - class LLM: def __init__(self, model_id: str, **kwargs): + start_server() self.provider, self.model = model_id.split("/") self.session_id = kwargs.get("session_id") self.api_key = kwargs.get("api_key") diff --git a/llmstudio/server.py b/llmstudio/server.py index e9643d73..1814c3c6 100644 --- a/llmstudio/server.py +++ b/llmstudio/server.py @@ -13,6 +13,7 @@ from llmstudio.engine import run_engine_app from llmstudio.tracking import run_tracking_app from llmstudio.ui import run_ui_app +from threading import Event _servers_started = False @@ -29,8 +30,10 @@ def is_server_running(host, port, path="/health"): def start_server_component(host, port, run_func, server_name): if not is_server_running(host, port): - thread = threading.Thread(target=run_func, daemon=True) + started_event = Event() + thread = threading.Thread(target=run_func, daemon=True, args=(started_event,)) thread.start() + started_event.wait() # wait for startup, this assumes the event is set somewhere return thread else: print(f"{server_name} server already running on {host}:{port}") @@ -53,7 +56,6 @@ def setup_servers(engine, tracking, ui): TRACKING_HOST, TRACKING_PORT, run_tracking_app, "Tracking" ) - ui_thread = None if ui: ui_thread = start_server_component(UI_HOST, UI_PORT, run_ui_app, "UI") diff --git a/llmstudio/tracking/__init__.py b/llmstudio/tracking/__init__.py index 31a7fbe8..9c05ed0d 100644 --- a/llmstudio/tracking/__init__.py +++ b/llmstudio/tracking/__init__.py @@ -1,6 +1,7 @@ import uvicorn from fastapi import APIRouter, FastAPI from fastapi.middleware.cors import CORSMiddleware +from threading import Event from llmstudio.config import TRACKING_HOST, TRACKING_PORT from llmstudio.engine.providers import * @@ -15,7 +16,7 @@ ## Tracking -def create_tracking_app() -> FastAPI: +def create_tracking_app(started_event : Event) -> FastAPI: app = FastAPI( title=TRACKING_TITLE, description=TRACKING_DESCRIPTION, @@ -43,14 +44,15 @@ def health_check(): @app.on_event("startup") async def startup_event(): + started_event.set() print(f"Running LLMstudio Tracking on http://{TRACKING_HOST}:{TRACKING_PORT} ") return app -def run_tracking_app(): +def run_tracking_app(started_event: Event): try: - tracking = create_tracking_app() + tracking = create_tracking_app(started_event) uvicorn.run( tracking, host=TRACKING_HOST, diff --git a/llmstudio/ui/__init__.py b/llmstudio/ui/__init__.py index c2a15c22..1569aa6f 100644 --- a/llmstudio/ui/__init__.py +++ b/llmstudio/ui/__init__.py @@ -2,7 +2,7 @@ import subprocess from pathlib import Path import threading -import webbrowser +from threading import Event from llmstudio.config import UI_PORT @@ -20,6 +20,7 @@ def run_bun_in_thread(): print(f"Error running LLMstudio UI: {e}") -def run_ui_app(): +def run_ui_app(started_event: Event): thread = threading.Thread(target=run_bun_in_thread) thread.start() + started_event.set() #just here for compatibility From fef4c02f8c8a05aef4150d048bc95e7a982407fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Gon=C3=A7alves?= Date: Sun, 8 Sep 2024 20:52:22 +0100 Subject: [PATCH 2/5] pre-commit changes --- llmstudio/engine/__init__.py | 8 +++++--- llmstudio/llm/__init__.py | 1 + llmstudio/server.py | 4 ++-- llmstudio/tracking/__init__.py | 5 +++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/llmstudio/engine/__init__.py b/llmstudio/engine/__init__.py index 83767c87..d541f775 100644 --- a/llmstudio/engine/__init__.py +++ b/llmstudio/engine/__init__.py @@ -1,6 +1,7 @@ import json import os from pathlib import Path +from threading import Event from typing import Any, Dict, List, Optional, Union import uvicorn @@ -9,7 +10,6 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse from pydantic import BaseModel, ValidationError -from threading import Event from llmstudio.config import ENGINE_HOST, ENGINE_PORT from llmstudio.engine.providers import * @@ -79,7 +79,9 @@ def _merge_configs(config1, config2): raise RuntimeError(f"Error in configuration data: {e}") -def create_engine_app(started_event: Event, config: EngineConfig = _load_engine_config()) -> FastAPI: +def create_engine_app( + started_event: Event, config: EngineConfig = _load_engine_config() +) -> FastAPI: app = FastAPI( title=ENGINE_TITLE, description=ENGINE_DESCRIPTION, @@ -169,7 +171,7 @@ async def startup_event(): return app -def run_engine_app(started_event : Event): +def run_engine_app(started_event: Event): try: engine = create_engine_app(started_event) uvicorn.run( diff --git a/llmstudio/llm/__init__.py b/llmstudio/llm/__init__.py index 33b2de63..f6c5c42a 100644 --- a/llmstudio/llm/__init__.py +++ b/llmstudio/llm/__init__.py @@ -9,6 +9,7 @@ from llmstudio.llm.semaphore import DynamicSemaphore from llmstudio.server import start_server + class LLM: def __init__(self, model_id: str, **kwargs): start_server() diff --git a/llmstudio/server.py b/llmstudio/server.py index 1814c3c6..c69a5b96 100644 --- a/llmstudio/server.py +++ b/llmstudio/server.py @@ -1,4 +1,5 @@ import threading +from threading import Event import requests @@ -13,7 +14,6 @@ from llmstudio.engine import run_engine_app from llmstudio.tracking import run_tracking_app from llmstudio.ui import run_ui_app -from threading import Event _servers_started = False @@ -33,7 +33,7 @@ def start_server_component(host, port, run_func, server_name): started_event = Event() thread = threading.Thread(target=run_func, daemon=True, args=(started_event,)) thread.start() - started_event.wait() # wait for startup, this assumes the event is set somewhere + started_event.wait() # wait for startup, this assumes the event is set somewhere return thread else: print(f"{server_name} server already running on {host}:{port}") diff --git a/llmstudio/tracking/__init__.py b/llmstudio/tracking/__init__.py index 9c05ed0d..d32bc768 100644 --- a/llmstudio/tracking/__init__.py +++ b/llmstudio/tracking/__init__.py @@ -1,7 +1,8 @@ +from threading import Event + import uvicorn from fastapi import APIRouter, FastAPI from fastapi.middleware.cors import CORSMiddleware -from threading import Event from llmstudio.config import TRACKING_HOST, TRACKING_PORT from llmstudio.engine.providers import * @@ -16,7 +17,7 @@ ## Tracking -def create_tracking_app(started_event : Event) -> FastAPI: +def create_tracking_app(started_event: Event) -> FastAPI: app = FastAPI( title=TRACKING_TITLE, description=TRACKING_DESCRIPTION, From 902ffb80387957250f45300519d8dc34b2c72985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Gon=C3=A7alves?= Date: Sun, 8 Sep 2024 20:54:05 +0100 Subject: [PATCH 3/5] Added missing status_code for openai APIConnectionError --- llmstudio/engine/providers/azure.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/llmstudio/engine/providers/azure.py b/llmstudio/engine/providers/azure.py index 1bb61516..53df2243 100644 --- a/llmstudio/engine/providers/azure.py +++ b/llmstudio/engine/providers/azure.py @@ -120,13 +120,17 @@ async def generate_client( **function_args, **request.parameters.model_dump(), } - # Perform the asynchronous call return await asyncio.to_thread( client.chat.completions.create, **combined_args ) - except openai._exceptions.APIError as e: + except openai._exceptions.APIConnectionError as e: + raise HTTPException( + status_code=404, detail="There was an error reaching the endpoint" + ) + + except openai._exceptions.APIStatusError as e: raise HTTPException(status_code=e.status_code, detail=e.response.json()) def prepare_messages(self, request: AzureRequest): From f1dd3892b26f10ea64d2cb557434c9e9ca15dfeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Gon=C3=A7alves?= Date: Sun, 8 Sep 2024 20:56:34 +0100 Subject: [PATCH 4/5] Reverted start_server back to class declaration instead of initialization --- llmstudio/llm/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llmstudio/llm/__init__.py b/llmstudio/llm/__init__.py index f6c5c42a..8f42086b 100644 --- a/llmstudio/llm/__init__.py +++ b/llmstudio/llm/__init__.py @@ -9,10 +9,11 @@ from llmstudio.llm.semaphore import DynamicSemaphore from llmstudio.server import start_server +start_server() + class LLM: def __init__(self, model_id: str, **kwargs): - start_server() self.provider, self.model = model_id.split("/") self.session_id = kwargs.get("session_id") self.api_key = kwargs.get("api_key") From ce4dd879011d19cc3007553a552ce9fa9cd27536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Gon=C3=A7alves?= Date: Mon, 9 Sep 2024 11:50:57 +0100 Subject: [PATCH 5/5] Added exception to error message --- llmstudio/engine/providers/azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llmstudio/engine/providers/azure.py b/llmstudio/engine/providers/azure.py index 53df2243..0f6b711d 100644 --- a/llmstudio/engine/providers/azure.py +++ b/llmstudio/engine/providers/azure.py @@ -127,7 +127,7 @@ async def generate_client( except openai._exceptions.APIConnectionError as e: raise HTTPException( - status_code=404, detail="There was an error reaching the endpoint" + status_code=404, detail=f"There was an error reaching the endpoint: {e}" ) except openai._exceptions.APIStatusError as e: