BUG: add error handling when the endpoint port is not available

xinference/core/restful_api.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import logging
-import sys
+import socket
 import threading
 from typing import Any, Dict, List, Literal, Optional, Union
 
@@ -25,7 +25,6 @@
 from pydantic import BaseModel, Field
 from typing_extensions import NotRequired, TypedDict
 from uvicorn import Config, Server
-from xoscar.utils import get_next_port
 
 from ..types import ChatCompletion, Completion
 from .service import SupervisorActor
@@ -210,12 +209,10 @@ class Config:
 
 
 class RESTfulAPIActor(xo.Actor):
-    def __init__(self, host: str, port: int, gradio_block: gr.Blocks):
+    def __init__(self, sockets: List[socket.socket], gradio_block: gr.Blocks):
         super().__init__()
         self._supervisor_ref: xo.ActorRefType["SupervisorActor"]
-        default_host = "0.0.0.0" if not sys.platform.startswith("win") else "127.0.0.1"
-        self._port = port if port else get_next_port()
-        self._host = host or default_host
+        self._sockets = sockets
         self._gradio_block = gradio_block
         self._router = None
 
@@ -263,11 +260,12 @@ def serve(self):
         app = gr.mount_gradio_app(app, self._gradio_block, path="/")
 
         # run uvicorn in another daemon thread.
-        config = Config(app=app, host=self._host, port=self._port, log_level="critical")
+        config = Config(app=app, log_level="critical")
         server = Server(config)
-        server_thread = threading.Thread(target=server.run, daemon=True)
+        server_thread = threading.Thread(
+            target=server.run, args=[self._sockets], daemon=True
+        )
         server_thread.start()
-        return f"http://{self._host}:{self._port}"
 
     async def list_models(self) -> Dict[str, Dict[str, Any]]:
         try:

xinference/deploy/cmdline.py

@@ -43,6 +43,7 @@ def cli(
         logging_conf = dict(level=log_level.upper())
 
         address = f"{host}:{get_next_port()}"
+
         main(
             address=address,
             host=host,
@@ -67,6 +68,7 @@ def supervisor(
     logging_conf = dict(level=log_level.upper())
 
     address = f"{host}:{get_next_port()}"
+
     main(address=address, host=host, port=port, logging_conf=logging_conf)
 
 

xinference/deploy/supervisor.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import logging
+import socket
 from typing import Dict, Optional
 
 import xoscar as xo
@@ -28,15 +29,20 @@
 async def start_supervisor_components(address: str, host: str, port: int):
     await xo.create_actor(SupervisorActor, address=address, uid=SupervisorActor.uid())
     gradio_block = GradioApp(address).build()
+    # create a socket for RESTful API
+    sockets = []
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.bind((host, port))
+    sockets.append(sock)
     restful_actor = await xo.create_actor(
         RESTfulAPIActor,
         address=address,
         uid=RESTfulAPIActor.uid(),
-        host=host,
-        port=port,
+        sockets=sockets,
         gradio_block=gradio_block,
     )
-    url = await restful_actor.serve()
+    await restful_actor.serve()
+    url = f"http://{host}:{port}"
     logger.info(f"Server address: {url}")
     return url