From ec1baef59d01ebbeacb3c626de87dd6b0df41f60 Mon Sep 17 00:00:00 2001 From: Jiayi Ni Date: Sun, 9 Jul 2023 21:10:15 -0700 Subject: [PATCH 1/6] add error handling --- xinference/deploy/cmdline.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/xinference/deploy/cmdline.py b/xinference/deploy/cmdline.py index 6efca345de..e968659d6e 100644 --- a/xinference/deploy/cmdline.py +++ b/xinference/deploy/cmdline.py @@ -14,6 +14,7 @@ import logging +import socket import click from xoscar.utils import get_next_port @@ -43,6 +44,14 @@ def cli( logging_conf = dict(level=log_level.upper()) address = f"{host}:{get_next_port()}" + + try: + skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + skt.bind((host, port)) + skt.close() + except OSError: + raise click.ClickException(f"Port {port} is already in use.") + main( address=address, host=host, @@ -67,6 +76,14 @@ def supervisor( logging_conf = dict(level=log_level.upper()) address = f"{host}:{get_next_port()}" + + try: + skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + skt.bind((host, port)) + skt.close() + except OSError: + raise click.ClickException(f"Port {port} is already in use.") + main(address=address, host=host, port=port, logging_conf=logging_conf) From a929cef317f3d9abcc31e0b90be81c092eefeb62 Mon Sep 17 00:00:00 2001 From: Jiayi Ni Date: Mon, 10 Jul 2023 00:58:35 -0700 Subject: [PATCH 2/6] create socket for RESTful API --- xinference/core/restful_api.py | 15 ++++++--------- xinference/deploy/cmdline.py | 15 --------------- xinference/deploy/supervisor.py | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/xinference/core/restful_api.py b/xinference/core/restful_api.py index 76e03f916c..699a6f459e 100644 --- a/xinference/core/restful_api.py +++ b/xinference/core/restful_api.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -import sys import threading from typing import Any, Dict, List, Literal, Optional, Union @@ -25,7 +24,6 @@ from pydantic import BaseModel, Field from typing_extensions import NotRequired, TypedDict from uvicorn import Config, Server -from xoscar.utils import get_next_port from ..types import ChatCompletion, Completion from .service import SupervisorActor @@ -210,12 +208,10 @@ class Config: class RESTfulAPIActor(xo.Actor): - def __init__(self, host: str, port: int, gradio_block: gr.Blocks): + def __init__(self, sockets, gradio_block: gr.Blocks): super().__init__() self._supervisor_ref: xo.ActorRefType["SupervisorActor"] - default_host = "0.0.0.0" if not sys.platform.startswith("win") else "127.0.0.1" - self._port = port if port else get_next_port() - self._host = host or default_host + self._sockets = sockets self._gradio_block = gradio_block self._router = None @@ -263,11 +259,12 @@ def serve(self): app = gr.mount_gradio_app(app, self._gradio_block, path="/") # run uvicorn in another daemon thread. - config = Config(app=app, host=self._host, port=self._port, log_level="critical") + config = Config(app=app, log_level="critical") server = Server(config) - server_thread = threading.Thread(target=server.run, daemon=True) + server_thread = threading.Thread( + target=server.run, args=[self._sockets], daemon=True + ) server_thread.start() - return f"http://{self._host}:{self._port}" async def list_models(self) -> Dict[str, Dict[str, Any]]: try: diff --git a/xinference/deploy/cmdline.py b/xinference/deploy/cmdline.py index e968659d6e..fb36f19153 100644 --- a/xinference/deploy/cmdline.py +++ b/xinference/deploy/cmdline.py @@ -14,7 +14,6 @@ import logging -import socket import click from xoscar.utils import get_next_port @@ -45,13 +44,6 @@ def cli( address = f"{host}:{get_next_port()}" - try: - skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - skt.bind((host, port)) - skt.close() - except OSError: - raise click.ClickException(f"Port {port} is already in use.") - main( address=address, host=host, @@ -77,13 +69,6 @@ def supervisor( address = f"{host}:{get_next_port()}" - try: - skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - skt.bind((host, port)) - skt.close() - except OSError: - raise click.ClickException(f"Port {port} is already in use.") - main(address=address, host=host, port=port, logging_conf=logging_conf) diff --git a/xinference/deploy/supervisor.py b/xinference/deploy/supervisor.py index b122d181d9..d9080d32d6 100644 --- a/xinference/deploy/supervisor.py +++ b/xinference/deploy/supervisor.py @@ -14,9 +14,12 @@ import asyncio import logging +import socket +import sys from typing import Dict, Optional import xoscar as xo +from xoscar.utils import get_next_port from ..core.gradio import GradioApp from ..core.restful_api import RESTfulAPIActor @@ -28,12 +31,22 @@ async def start_supervisor_components(address: str, host: str, port: int): await xo.create_actor(SupervisorActor, address=address, uid=SupervisorActor.uid()) gradio_block = await GradioApp(address).build() + # create a socket for RESTful API + try: + default_host = "0.0.0.0" if not sys.platform.startswith("win") else "127.0.0.1" + port = port if port else get_next_port() + host = host or default_host + socks = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + socks.bind((host, port)) + sockets = [] + sockets.append(socks) + except OSError: + raise OSError restful_actor = await xo.create_actor( RESTfulAPIActor, address=address, uid=RESTfulAPIActor.uid(), - host=host, - port=port, + sockets=sockets, gradio_block=gradio_block, ) url = await restful_actor.serve() From 8115e04950b19e12e3a505366f4e7b5b637b6c2c Mon Sep 17 00:00:00 2001 From: Jiayi Ni Date: Mon, 10 Jul 2023 01:48:03 -0700 Subject: [PATCH 3/6] rebase --- xinference/deploy/supervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xinference/deploy/supervisor.py b/xinference/deploy/supervisor.py index d9080d32d6..6951120eea 100644 --- a/xinference/deploy/supervisor.py +++ b/xinference/deploy/supervisor.py @@ -36,9 +36,9 @@ async def start_supervisor_components(address: str, host: str, port: int): default_host = "0.0.0.0" if not sys.platform.startswith("win") else "127.0.0.1" port = port if port else get_next_port() host = host or default_host + sockets = [] socks = socket.socket(socket.AF_INET, socket.SOCK_STREAM) socks.bind((host, port)) - sockets = [] sockets.append(socks) except OSError: raise OSError From 7804ff35201351faa2f0a0ed7ee88d795a25ea89 Mon Sep 17 00:00:00 2001 From: Jiayi Ni Date: Mon, 10 Jul 2023 01:54:15 -0700 Subject: [PATCH 4/6] type hint --- xinference/core/restful_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xinference/core/restful_api.py b/xinference/core/restful_api.py index 699a6f459e..6308030ecd 100644 --- a/xinference/core/restful_api.py +++ b/xinference/core/restful_api.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import socket import threading from typing import Any, Dict, List, Literal, Optional, Union @@ -208,7 +209,7 @@ class Config: class RESTfulAPIActor(xo.Actor): - def __init__(self, sockets, gradio_block: gr.Blocks): + def __init__(self, sockets: List[socket.socket], gradio_block: gr.Blocks): super().__init__() self._supervisor_ref: xo.ActorRefType["SupervisorActor"] self._sockets = sockets From 0a948ab2ba82b8dd2181f32e65066e77a49d3875 Mon Sep 17 00:00:00 2001 From: Jiayi Ni Date: Mon, 10 Jul 2023 02:09:33 -0700 Subject: [PATCH 5/6] return url --- xinference/deploy/supervisor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xinference/deploy/supervisor.py b/xinference/deploy/supervisor.py index 6951120eea..94e6971a9b 100644 --- a/xinference/deploy/supervisor.py +++ b/xinference/deploy/supervisor.py @@ -49,7 +49,8 @@ async def start_supervisor_components(address: str, host: str, port: int): sockets=sockets, gradio_block=gradio_block, ) - url = await restful_actor.serve() + await restful_actor.serve() + url = f"http://{host}:{port}" logger.info(f"Server address: {url}") return url From e8415d73621ce6de3d3f397bf548924d38d5ba32 Mon Sep 17 00:00:00 2001 From: Jiayi Ni Date: Mon, 10 Jul 2023 02:39:21 -0700 Subject: [PATCH 6/6] fix --- xinference/deploy/supervisor.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/xinference/deploy/supervisor.py b/xinference/deploy/supervisor.py index 94e6971a9b..fd46c28923 100644 --- a/xinference/deploy/supervisor.py +++ b/xinference/deploy/supervisor.py @@ -15,11 +15,9 @@ import asyncio import logging import socket -import sys from typing import Dict, Optional import xoscar as xo -from xoscar.utils import get_next_port from ..core.gradio import GradioApp from ..core.restful_api import RESTfulAPIActor @@ -32,16 +30,10 @@ async def start_supervisor_components(address: str, host: str, port: int): await xo.create_actor(SupervisorActor, address=address, uid=SupervisorActor.uid()) gradio_block = await GradioApp(address).build() # create a socket for RESTful API - try: - default_host = "0.0.0.0" if not sys.platform.startswith("win") else "127.0.0.1" - port = port if port else get_next_port() - host = host or default_host - sockets = [] - socks = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - socks.bind((host, port)) - sockets.append(socks) - except OSError: - raise OSError + sockets = [] + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind((host, port)) + sockets.append(sock) restful_actor = await xo.create_actor( RESTfulAPIActor, address=address,