From 98c9667d8903835cd37147ad13d162ff82cc7199 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 14 Nov 2025 11:39:41 +0200 Subject: [PATCH] server : fix "can batch with" bug --- tools/server/server.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 7dbd8b6a002d1..535d2c450e21e 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -3591,13 +3591,13 @@ struct server_context { // next, batch any pending prompts without exceeding n_batch if (params_base.cont_batching || batch.n_tokens == 0) { for (auto & slot : slots) { + if (!slot.is_processing()) { + continue; + } + // check if we can batch this slot with the previous one - if (slot.is_processing()) { - if (!slot_batched) { - slot_batched = &slot; - } else if (!slot_batched->can_batch_with(slot)) { - continue; - } + if (slot_batched && !slot_batched->can_batch_with(slot)) { + continue; } // this slot still has a prompt to be processed @@ -4028,6 +4028,10 @@ struct server_context { } } + if (!slot_batched) { + slot_batched = &slot; + } + if (batch.n_tokens >= n_batch) { break; }