Merge tag 'io_uring-7.4-20261106' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull io_uring fixes from Jens Axboe:

 - Only wait for in-flight requests on ring release if the closing task
   actually uses the ring.

 - SQPOLL fixes: do the return-to-userspace work in the SQPOLL thread,
   as memory.high reclaim and blkcg throttling otherwise only ran at
   thread exit, and don't set the stop bit on thread exit, which tripped
   a WARN_ON_ONCE() in the ring teardown.

 - Don't poll a writer for a read side hangup or for urgent data, as a
   half-closed peer or unread OOB data kept the async poll of a send
   ready on every re-arm.

 - Cancel all matching poll requests in one pass for CANCEL_ALL rather
   than restarting the hash walk per request.

 - Hold uring_lock across the put_user() for the index-alloc file
   install.

 - Don't arm an hrtimer and schedule for a wait whose timeout has
   already expired.

 - Add a feature flag for the dma-buf registered buffer support.

 - selftests 32-bit cast fix

* tag 'io_uring-7.4-20261106' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux:
  io_uring: only wait for in-flight requests if the closing task uses the ring
  io_uring: add feature flag for dma-buf support
  io_uring/wait: don't sleep on an already expired timeout
  io_uring/rsrc: hold uring_lock across index-alloc put_user
  selftests/io_uring: cast optval through uintptr_t in mini_liburing.h
  io_uring/poll: don't poll a writer for urgent data
  io_uring/poll: don't poll a writer for a read side hangup
  io_uring/poll: cancel all matching requests in one pass
  io_uring/cancel: count all requests that a CANCEL_ALL cancels
  io_uring/sqpoll: do the return-to-userspace work in the SQPOLL thread
  io_uring/sqpoll: don't set IO_SQ_THREAD_SHOULD_STOP on thread exit
This commit is contained in:
Linus Torvalds
2026-11-06 12:30:19 -08:00
9 changed files with 108 additions and 19 deletions
+1
View File
@@ -646,6 +646,7 @@ struct io_uring_params {
#define IORING_FEAT_MIN_TIMEOUT (1U << 15)
#define IORING_FEAT_RW_ATTR (1U << 16)
#define IORING_FEAT_NO_IOWAIT (1U << 17)
#define IORING_FEAT_REGBUF_TYPES (1U << 18)
/*
* io_uring_register(2) opcodes and arguments
+2 -1
View File
@@ -198,7 +198,8 @@ static int __io_async_cancel(struct io_cancel_data *cd,
break;
if (!all)
return ret;
nr++;
/* poll, futex and waitid return how many they cancelled */
nr += max(ret, 1);
} while (1);
/* slow path, try all io-wq's */
+4 -1
View File
@@ -2547,7 +2547,10 @@ static __cold void io_ring_ctx_wait_inflight(struct io_ring_ctx *ctx)
if (current->flags & (PF_KTHREAD | PF_EXITING))
return;
if (tctx && atomic_read(&tctx->in_cancel))
/* Only wait if this task has attached to the ring */
if (!tctx || !xa_load(&tctx->xa, (unsigned long)ctx))
return;
if (atomic_read(&tctx->in_cancel))
return;
while (io_ring_ctx_inflight(ctx) && !fatal_signal_pending(current)) {
+2 -1
View File
@@ -50,7 +50,8 @@ struct io_ctx_config {
IORING_FEAT_RECVSEND_BUNDLE |\
IORING_FEAT_MIN_TIMEOUT |\
IORING_FEAT_RW_ATTR |\
IORING_FEAT_NO_IOWAIT)
IORING_FEAT_NO_IOWAIT |\
IORING_FEAT_REGBUF_TYPES)
#define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\
IORING_SETUP_SQPOLL |\
+55 -7
View File
@@ -128,12 +128,24 @@ static void io_poll_req_insert(struct io_kiocb *req)
hlist_add_head(&req->hash_node, &table->hbs[index].list);
}
#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL)
static __poll_t io_poll_unmask(__poll_t events)
{
/*
* A peer that stopped sending is an event for a reader, but a write
* only poll must stay armed until the file is writable again.
*/
if (events & (EPOLLIN | EPOLLRDNORM | EPOLLRDBAND))
return IO_POLL_UNMASK | EPOLLRDHUP;
return IO_POLL_UNMASK;
}
static void io_init_poll_iocb(struct io_poll *poll, __poll_t events)
{
poll->head = NULL;
#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
/* mask in events that we always want/need */
poll->events = events | IO_POLL_UNMASK;
poll->events = events | io_poll_unmask(events);
INIT_LIST_HEAD(&poll->wait.entry);
init_waitqueue_func_entry(&poll->wait, io_poll_wake);
}
@@ -727,7 +739,7 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask)
int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)
{
const struct io_issue_def *def = &io_issue_defs[req->opcode];
__poll_t mask = POLLPRI | POLLERR;
__poll_t mask = EPOLLERR;
if (!def->pollin && !def->pollout)
return IO_APOLL_ABORTED;
@@ -735,7 +747,8 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)
return IO_APOLL_ABORTED;
if (def->pollin) {
mask |= EPOLLIN | EPOLLRDNORM;
/* urgent data is an event for a reader only */
mask |= EPOLLIN | EPOLLRDNORM | EPOLLPRI;
/* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
if (req->flags & REQ_F_CLEAR_POLLIN)
@@ -827,12 +840,47 @@ static int io_poll_disarm(struct io_kiocb *req)
return 0;
}
/*
* Cancel every request matching @cd in one pass. Cancelled requests stay
* hashed until their task work runs, so finding them one at a time would
* walk past all the earlier ones on each lookup.
*/
static int io_poll_cancel_all(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
{
unsigned int nr_buckets = 1U << ctx->cancel_table.hash_bits;
unsigned int i = 0;
struct hlist_node *tmp;
struct io_kiocb *req;
int nr = 0;
if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP |
IORING_ASYNC_CANCEL_ANY))) {
/* keyed by user_data, so only one bucket can match */
i = hash_long(cd->data, ctx->cancel_table.hash_bits);
nr_buckets = i + 1;
}
for (; i < nr_buckets; i++) {
struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i];
hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) {
if (!io_cancel_req_match(req, cd))
continue;
io_poll_cancel_req(req);
nr++;
}
}
return nr ?: -ENOENT;
}
static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
{
struct io_kiocb *req;
if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP |
IORING_ASYNC_CANCEL_ANY))
if (cd->flags & (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_ANY))
return io_poll_cancel_all(ctx, cd);
if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP))
req = io_poll_file_find(ctx, cd);
else
req = io_poll_find(ctx, false, cd);
@@ -971,7 +1019,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
poll->events &= ~0xffff;
poll->events |= poll_update->events & 0xffff;
poll->events |= IO_POLL_UNMASK;
poll->events |= io_poll_unmask(poll->events);
}
if (poll_update->update_user_data)
preq->cqe.user_data = poll_update->new_user_data;
+16 -5
View File
@@ -572,11 +572,12 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req,
{
struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
__s32 __user *fds = u64_to_user_ptr(up->arg);
struct io_ring_ctx *ctx = req->ctx;
unsigned int done;
struct file *file;
int ret, fd;
if (!req->ctx->file_table.data.nr)
if (!ctx->file_table.data.nr)
return -ENXIO;
for (done = 0; done < up->nr_args; done++) {
@@ -590,15 +591,25 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req,
ret = -EBADF;
break;
}
ret = io_fixed_fd_install(req, issue_flags, file,
IORING_FILE_INDEX_ALLOC);
if (ret < 0)
/*
* Hold the lock across the put_user(), if it's dropped the
* slot can be closed and reused before the error path runs.
*/
io_ring_submit_lock(ctx, issue_flags);
ret = __io_fixed_fd_install(ctx, file,
IORING_FILE_INDEX_ALLOC);
if (ret < 0) {
io_ring_submit_unlock(ctx, issue_flags);
fput(file);
break;
}
if (put_user(ret, &fds[done])) {
__io_close_fixed(req->ctx, issue_flags, ret);
io_fixed_fd_remove(ctx, ret);
io_ring_submit_unlock(ctx, issue_flags);
ret = -EFAULT;
break;
}
io_ring_submit_unlock(ctx, issue_flags);
}
if (done)
+18 -3
View File
@@ -29,6 +29,7 @@
enum {
IO_SQ_THREAD_SHOULD_STOP = 0,
IO_SQ_THREAD_SHOULD_PARK,
IO_SQ_THREAD_EXITING,
};
void io_sq_thread_unpark(struct io_sq_data *sqd)
@@ -348,6 +349,18 @@ static int io_sq_thread(void *data)
timeout = jiffies + sqd->sq_thread_idle;
}
/*
* The SQPOLL thread never returns to userspace, so do the
* return-to-userspace work, like memory.high reclaim, here.
* It can sleep for a long time, so drop sqd->lock.
*/
if (test_thread_flag(TIF_NOTIFY_RESUME)) {
mutex_unlock(&sqd->lock);
resume_user_mode_work(NULL);
mutex_lock(&sqd->lock);
sqd->sq_cpu = raw_smp_processor_id();
}
cap_entries = !list_is_singular(&sqd->ctx_list);
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
int ret;
@@ -436,9 +449,10 @@ static int io_sq_thread(void *data)
/*
* Cancelation drops sqd->lock while it waits, and a ring could attach
* to a thread that is on its way out. A fatal signal doesn't set the
* bit, so set it here for io_sq_offload_create() to see.
* stop bit, so mark the thread as exiting here for
* io_sq_offload_create() to see.
*/
set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
set_bit(IO_SQ_THREAD_EXITING, &sqd->state);
io_uring_cancel_generic(true, sqd);
rcu_assign_pointer(sqd->thread, NULL);
put_task_struct(current);
@@ -510,7 +524,8 @@ __cold int io_sq_offload_create(struct io_ring_ctx *ctx,
/* don't attach to a dying SQPOLL thread, would be racy */
ret = 0;
if (attached && (!sqd->thread ||
test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)))
test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state) ||
test_bit(IO_SQ_THREAD_EXITING, &sqd->state)))
ret = -ENXIO;
io_sq_thread_unpark(sqd);
+8
View File
@@ -165,6 +165,14 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
{
int ret = 0;
/*
* Don't arm a timer for a timeout that has already passed, it would
* only fire to return -ETIME. NAPI busy polling gets here like that.
*/
if (iowq->timeout != KTIME_MAX &&
ktime_compare(io_get_time(ctx), iowq->timeout) >= 0)
return -ETIME;
/*
* Mark us as being in io_wait if we have pending requests, so cpufreq
* can take into account that the task is waiting for IO - turns out
+2 -1
View File
@@ -3,6 +3,7 @@
#include <linux/io_uring.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
@@ -269,7 +270,7 @@ static inline void io_uring_prep_cmd(struct io_uring_sqe *sqe, int op,
sqe->level = level;
sqe->optname = optname;
sqe->optval = (unsigned long long)optval;
sqe->optval = (unsigned long long)(uintptr_t)optval;
sqe->optlen = optlen;
}