Whisper for 100 concurrent users #2192
Unanswered
saboorniazi
asked this question in
Q&A
Replies: 0 comments
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
-
`import asyncio
import websockets
import numpy as np
import whisper
import torch
import threading
import time
from urllib.parse import parse_qs
Load Whisper model instances globally
model_pool = [whisper.load_model("tiny.en") for _ in range(5)]
print('5 instances of whisper model tiny.en loaded')
Global variables and locks
frames_locks = [threading.Lock() for _ in range(len(model_pool))] # one lock per model instance
threshold = 20 # voice pitch threshold
ms = 500 # milliseconds
connected_clients = set()
user_data = {}
model_counter = 0 # Counter to distribute requests
async def audio_data(websocket, path):
global connected_clients
connected_clients.add(websocket)
query_string = path.split('?')[1]
query_params = dict(parse_qs(query_string))
VAD = query_params.get('vad')
VAD = int(VAD[0]) if VAD else ms
async def handle_audio_stream(websocket, connection_id):
try:
async for message in websocket:
user = user_data.get(connection_id)
if not user:
return
audio_buffer = np.frombuffer(message, dtype=np.int16)
b = ((np.abs(np.fft.rfft(audio_buffer))) * 16000 / len(message) // 2).mean()
async def process_transcription(audio_tensor, websocket, connection_id):
global model_counter
lock_index = model_counter % len(frames_locks)
model_instance = model_pool[model_counter % len(model_pool)]
model_counter += 1
start_server = websockets.serve(audio_data, "localhost", 8001)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()`
this is my code i want to run this for 100 users conccurrently and i am running this on 2 rtx 3090 gpus .
can someone guide me to correctly use asyncio and threads to make this for 100 users?
Beta Was this translation helpful? Give feedback.
All reactions