From 6348f55337a8f7a7c04d3ec122c8e403d68b44d5 Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Sun, 4 Feb 2024 20:55:51 +0530 Subject: [PATCH] Create a new multiprocess context to set start method Signed-off-by: Sivanantham Chinnaiyan --- python/kserve/kserve/errors.py | 8 -------- python/kserve/kserve/model_server.py | 19 ++++++------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/python/kserve/kserve/errors.py b/python/kserve/kserve/errors.py index 8fcd30ad2b7..1e873f45765 100644 --- a/python/kserve/kserve/errors.py +++ b/python/kserve/kserve/errors.py @@ -88,14 +88,6 @@ def __str__(self): return self.error_msg -class UnsupportedProcessStartMethod(RuntimeError): - def __init__(self, start_method: str): - self.error_msg = f"Unsupported multi processing start method '{start_method}'. Only 'fork' is supported." - - def __str__(self): - return self.error_msg - - async def exception_handler(_, exc): logger.error("Exception:", exc_info=exc) return JSONResponse(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, content={"error": str(exc)}) diff --git a/python/kserve/kserve/model_server.py b/python/kserve/kserve/model_server.py index e4ed2f0c5f9..3dfb0c9f033 100644 --- a/python/kserve/kserve/model_server.py +++ b/python/kserve/kserve/model_server.py @@ -19,14 +19,12 @@ import multiprocessing import signal import socket -from multiprocessing import Process from typing import Dict, List, Optional, Union from ray import serve as rayserve from ray.serve.api import Deployment from ray.serve.handle import RayServeHandle -from .errors import UnsupportedProcessStartMethod from .logging import KSERVE_LOG_CONFIG, logger from .model import Model from .model_repository import ModelRepository @@ -196,26 +194,21 @@ async def serve(): access_log_format=self.access_log_format) await self._rest_server.run() else: - # Since py38 MacOS/Windows defaults to use spawn for starting multiprocessing. - # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods - # Spawn does not work with FastAPI/uvicorn in multiprocessing mode, use fork for multiprocessing - # https://github.com/tiangolo/fastapi/issues/1586 serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) serversocket.bind(('0.0.0.0', self.http_port)) serversocket.listen(5) - start_method = multiprocessing.get_start_method(allow_none=True) - if start_method is None: - logger.info("Setting 'fork' as multiprocessing start method.") - multiprocessing.set_start_method('fork') - elif start_method in ['spawn', 'forkserver']: - raise UnsupportedProcessStartMethod(start_method) self._rest_server = UvicornServer(self.http_port, [serversocket], self.dataplane, self.model_repository_extension, self.enable_docs_url, log_config=self.log_config, access_log_format=self.access_log_format) + # Since py38 MacOS/Windows defaults to use spawn for starting multiprocessing. + # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods + # Spawn does not work with FastAPI/uvicorn in multiprocessing mode, use fork for multiprocessing + # https://github.com/tiangolo/fastapi/issues/1586 + context = multiprocessing.get_context('fork') for _ in range(self.workers): - p = Process(target=self._rest_server.run_sync) + p = context.Process(target=self._rest_server.run_sync) p.start() async def servers_task():