diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 50a65fbc6ff0..e3d9fe8fc982 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -198,7 +198,7 @@ static int __io_async_cancel(struct io_cancel_data *cd, break; if (!all) return ret; - /* futex and waitid return how many they cancelled */ + /* poll, futex and waitid return how many they cancelled */ nr += max(ret, 1); } while (1); diff --git a/io_uring/poll.c b/io_uring/poll.c index 4f04bda42f7f..9b73b0d88a0e 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -827,12 +827,47 @@ static int io_poll_disarm(struct io_kiocb *req) return 0; } +/* + * Cancel every request matching @cd in one pass. Cancelled requests stay + * hashed until their task work runs, so finding them one at a time would + * walk past all the earlier ones on each lookup. + */ +static int io_poll_cancel_all(struct io_ring_ctx *ctx, struct io_cancel_data *cd) +{ + unsigned int nr_buckets = 1U << ctx->cancel_table.hash_bits; + unsigned int i = 0; + struct hlist_node *tmp; + struct io_kiocb *req; + int nr = 0; + + if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP | + IORING_ASYNC_CANCEL_ANY))) { + /* keyed by user_data, so only one bucket can match */ + i = hash_long(cd->data, ctx->cancel_table.hash_bits); + nr_buckets = i + 1; + } + + for (; i < nr_buckets; i++) { + struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; + + hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { + if (!io_cancel_req_match(req, cd)) + continue; + io_poll_cancel_req(req); + nr++; + } + } + return nr ?: -ENOENT; +} + static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) { struct io_kiocb *req; - if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP | - IORING_ASYNC_CANCEL_ANY)) + if (cd->flags & (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_ANY)) + return io_poll_cancel_all(ctx, cd); + + if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP)) req = io_poll_file_find(ctx, cd); else req = io_poll_find(ctx, false, cd);