mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
io_uring/poll: cancel all matching requests in one pass
With IORING_ASYNC_CANCEL_ALL, __io_async_cancel() calls io_try_cancel()
until nothing is found, and io_poll_cancel() cancels one request per
call, starting each lookup at the beginning of the hash. A cancelled
request stays hashed until its task work runs, so every lookup walks
past the ones already cancelled. Cancelling K of N pending polls visits
about K*N/2 requests in one io_uring_enter(), with uring_lock held.
A single CANCEL_FD|ALL on 80000 polls on one eventfd takes 17.06s,
0.02s with this change. With voluntary preemption the soft lockup
watchdog fires:
watchdog: BUG: soft lockup - CPU#1 stuck for 26s! [repro:100]
RIP: 0010:io_cancel_req_match+0x19/0xa0
io_poll_cancel+0x73/0x170
io_try_cancel+0x7b/0xf0
__io_async_cancel+0x41/0x110
Walk the hash once and cancel every match, as io_cancel_remove() does
for futex and waitid. Cancel by user_data only needs its own bucket.
__io_async_cancel() still loops until nothing is found, so a request
that io-wq parks in the poll hash meanwhile is still cancelled.
A cancel that matches one request now walks the whole hash twice instead
of stopping at the match.
Cc: [email protected]
Fixes: 8e29da69fe ("io_uring: add support for IORING_ASYNC_CANCEL_ALL")
Signed-off-by: Shang En Sim <[email protected]>
Link: https://patch.msgid.link/[email protected]
Signed-off-by: Jens Axboe <[email protected]>
This commit is contained in:
+1
-1
@@ -198,7 +198,7 @@ static int __io_async_cancel(struct io_cancel_data *cd,
|
||||
break;
|
||||
if (!all)
|
||||
return ret;
|
||||
/* futex and waitid return how many they cancelled */
|
||||
/* poll, futex and waitid return how many they cancelled */
|
||||
nr += max(ret, 1);
|
||||
} while (1);
|
||||
|
||||
|
||||
+37
-2
@@ -827,12 +827,47 @@ static int io_poll_disarm(struct io_kiocb *req)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Cancel every request matching @cd in one pass. Cancelled requests stay
|
||||
* hashed until their task work runs, so finding them one at a time would
|
||||
* walk past all the earlier ones on each lookup.
|
||||
*/
|
||||
static int io_poll_cancel_all(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
|
||||
{
|
||||
unsigned int nr_buckets = 1U << ctx->cancel_table.hash_bits;
|
||||
unsigned int i = 0;
|
||||
struct hlist_node *tmp;
|
||||
struct io_kiocb *req;
|
||||
int nr = 0;
|
||||
|
||||
if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP |
|
||||
IORING_ASYNC_CANCEL_ANY))) {
|
||||
/* keyed by user_data, so only one bucket can match */
|
||||
i = hash_long(cd->data, ctx->cancel_table.hash_bits);
|
||||
nr_buckets = i + 1;
|
||||
}
|
||||
|
||||
for (; i < nr_buckets; i++) {
|
||||
struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i];
|
||||
|
||||
hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) {
|
||||
if (!io_cancel_req_match(req, cd))
|
||||
continue;
|
||||
io_poll_cancel_req(req);
|
||||
nr++;
|
||||
}
|
||||
}
|
||||
return nr ?: -ENOENT;
|
||||
}
|
||||
|
||||
static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
|
||||
{
|
||||
struct io_kiocb *req;
|
||||
|
||||
if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP |
|
||||
IORING_ASYNC_CANCEL_ANY))
|
||||
if (cd->flags & (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_ANY))
|
||||
return io_poll_cancel_all(ctx, cd);
|
||||
|
||||
if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP))
|
||||
req = io_poll_file_find(ctx, cd);
|
||||
else
|
||||
req = io_poll_find(ctx, false, cd);
|
||||
|
||||
Reference in New Issue
Block a user