fix: allow only one request in the python backends

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
mudler · Aug 18, 2023 · 547c926 · 547c926
1 parent 91b1902
commit 547c926
Show file tree

Hide file tree

Showing 7 changed files with 7 additions and 5 deletions.
diff --git a/extra/grpc/autogptq/autogptq.py b/extra/grpc/autogptq/autogptq.py
@@ -77,7 +77,7 @@ def PredictStream(self, request, context):
 
 
 def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
     backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
     server.add_insecure_port(address)
     server.start()

diff --git a/extra/grpc/bark/ttsbark.py b/extra/grpc/bark/ttsbark.py
@@ -51,7 +51,7 @@ def TTS(self, request, context):
         return backend_pb2.Result(success=True)
 
 def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
     backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
     server.add_insecure_port(address)
     server.start()

diff --git a/extra/grpc/diffusers/backend_diffusers.py b/extra/grpc/diffusers/backend_diffusers.py
@@ -267,7 +267,7 @@ def GenerateImage(self, request, context):
         return backend_pb2.Result(message="Model loaded successfully", success=True)
 
 def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
     backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
     server.add_insecure_port(address)
     server.start()

diff --git a/extra/grpc/exllama/exllama.py b/extra/grpc/exllama/exllama.py
@@ -110,7 +110,7 @@ def PredictStream(self, request, context):
 
 
 def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
     backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
     server.add_insecure_port(address)
     server.start()

diff --git a/extra/grpc/huggingface/huggingface.py b/extra/grpc/huggingface/huggingface.py
@@ -34,7 +34,7 @@ def Embedding(self, request, context):
 
 
 def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
     backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
     server.add_insecure_port(address)
     server.start()

diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go
@@ -167,6 +167,7 @@ func (ml *ModelLoader) BackendLoader(opts ...Option) (model *grpc.Client, err er
 
 	if o.singleActiveBackend {
 		ml.mu.Lock()
+		log.Debug().Msgf("Stopping all backends except '%s'", o.model)
 		ml.StopAllExcept(o.model)
 		ml.mu.Unlock()
 	}

diff --git a/pkg/model/process.go b/pkg/model/process.go
@@ -19,6 +19,7 @@ func (ml *ModelLoader) StopAllExcept(s string) {
 			for ml.models[id].IsBusy() {
 				log.Debug().Msgf("%s busy. Waiting for  to finish", id)
 			}
+			log.Debug().Msgf("[single-backend] Stopping %s", id)
 			return true
 		}
 		return false