Skip to content

Commit 2ca824c

Browse files
marcelo-gonzalezgregkh
authored andcommitted
io_uring: flush timeouts that should already have expired
[ Upstream commit f010505 ] Right now io_flush_timeouts() checks if the current number of events is equal to ->timeout.target_seq, but this will miss some timeouts if there have been more than 1 event added since the last time they were flushed (possible in io_submit_flush_completions(), for example). Fix it by recording the last sequence at which timeouts were flushed so that the number of events seen can be compared to the number of events needed without overflow. Signed-off-by: Marcelo Diop-Gonzalez <marcelo827@gmail.com> Reviewed-by: Pavel Begunkov <asml.silence@gmail.com> Signed-off-by: Jens Axboe <axboe@kernel.dk> Signed-off-by: Sasha Levin <sashal@kernel.org>
1 parent f3f906b commit 2ca824c

File tree

1 file changed

+30
-4
lines changed

1 file changed

+30
-4
lines changed

fs/io_uring.c

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,7 @@ struct io_ring_ctx {
353353
unsigned cq_entries;
354354
unsigned cq_mask;
355355
atomic_t cq_timeouts;
356+
unsigned cq_last_tm_flush;
356357
unsigned long cq_check_overflow;
357358
struct wait_queue_head cq_wait;
358359
struct fasync_struct *cq_fasync;
@@ -1521,19 +1522,38 @@ static void __io_queue_deferred(struct io_ring_ctx *ctx)
15211522

15221523
static void io_flush_timeouts(struct io_ring_ctx *ctx)
15231524
{
1524-
while (!list_empty(&ctx->timeout_list)) {
1525+
u32 seq;
1526+
1527+
if (list_empty(&ctx->timeout_list))
1528+
return;
1529+
1530+
seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
1531+
1532+
do {
1533+
u32 events_needed, events_got;
15251534
struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
15261535
struct io_kiocb, timeout.list);
15271536

15281537
if (io_is_timeout_noseq(req))
15291538
break;
1530-
if (req->timeout.target_seq != ctx->cached_cq_tail
1531-
- atomic_read(&ctx->cq_timeouts))
1539+
1540+
/*
1541+
* Since seq can easily wrap around over time, subtract
1542+
* the last seq at which timeouts were flushed before comparing.
1543+
* Assuming not more than 2^31-1 events have happened since,
1544+
* these subtractions won't have wrapped, so we can check if
1545+
* target is in [last_seq, current_seq] by comparing the two.
1546+
*/
1547+
events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1548+
events_got = seq - ctx->cq_last_tm_flush;
1549+
if (events_got < events_needed)
15321550
break;
15331551

15341552
list_del_init(&req->timeout.list);
15351553
io_kill_timeout(req);
1536-
}
1554+
} while (!list_empty(&ctx->timeout_list));
1555+
1556+
ctx->cq_last_tm_flush = seq;
15371557
}
15381558

15391559
static void io_commit_cqring(struct io_ring_ctx *ctx)
@@ -5582,6 +5602,12 @@ static int io_timeout(struct io_kiocb *req)
55825602
tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
55835603
req->timeout.target_seq = tail + off;
55845604

5605+
/* Update the last seq here in case io_flush_timeouts() hasn't.
5606+
* This is safe because ->completion_lock is held, and submissions
5607+
* and completions are never mixed in the same ->completion_lock section.
5608+
*/
5609+
ctx->cq_last_tm_flush = tail;
5610+
55855611
/*
55865612
* Insertion sort, ensuring the first entry in the list is always
55875613
* the one we need first.

0 commit comments

Comments
 (0)