mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
Merge tag 'io_uring-7.4-20261106' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux
Pull io_uring fixes from Jens Axboe: - Only wait for in-flight requests on ring release if the closing task actually uses the ring. - SQPOLL fixes: do the return-to-userspace work in the SQPOLL thread, as memory.high reclaim and blkcg throttling otherwise only ran at thread exit, and don't set the stop bit on thread exit, which tripped a WARN_ON_ONCE() in the ring teardown. - Don't poll a writer for a read side hangup or for urgent data, as a half-closed peer or unread OOB data kept the async poll of a send ready on every re-arm. - Cancel all matching poll requests in one pass for CANCEL_ALL rather than restarting the hash walk per request. - Hold uring_lock across the put_user() for the index-alloc file install. - Don't arm an hrtimer and schedule for a wait whose timeout has already expired. - Add a feature flag for the dma-buf registered buffer support. - selftests 32-bit cast fix * tag 'io_uring-7.4-20261106' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: io_uring: only wait for in-flight requests if the closing task uses the ring io_uring: add feature flag for dma-buf support io_uring/wait: don't sleep on an already expired timeout io_uring/rsrc: hold uring_lock across index-alloc put_user selftests/io_uring: cast optval through uintptr_t in mini_liburing.h io_uring/poll: don't poll a writer for urgent data io_uring/poll: don't poll a writer for a read side hangup io_uring/poll: cancel all matching requests in one pass io_uring/cancel: count all requests that a CANCEL_ALL cancels io_uring/sqpoll: do the return-to-userspace work in the SQPOLL thread io_uring/sqpoll: don't set IO_SQ_THREAD_SHOULD_STOP on thread exit
This commit is contained in:
@@ -646,6 +646,7 @@ struct io_uring_params {
|
||||
#define IORING_FEAT_MIN_TIMEOUT (1U << 15)
|
||||
#define IORING_FEAT_RW_ATTR (1U << 16)
|
||||
#define IORING_FEAT_NO_IOWAIT (1U << 17)
|
||||
#define IORING_FEAT_REGBUF_TYPES (1U << 18)
|
||||
|
||||
/*
|
||||
* io_uring_register(2) opcodes and arguments
|
||||
|
||||
+2
-1
@@ -198,7 +198,8 @@ static int __io_async_cancel(struct io_cancel_data *cd,
|
||||
break;
|
||||
if (!all)
|
||||
return ret;
|
||||
nr++;
|
||||
/* poll, futex and waitid return how many they cancelled */
|
||||
nr += max(ret, 1);
|
||||
} while (1);
|
||||
|
||||
/* slow path, try all io-wq's */
|
||||
|
||||
+4
-1
@@ -2547,7 +2547,10 @@ static __cold void io_ring_ctx_wait_inflight(struct io_ring_ctx *ctx)
|
||||
|
||||
if (current->flags & (PF_KTHREAD | PF_EXITING))
|
||||
return;
|
||||
if (tctx && atomic_read(&tctx->in_cancel))
|
||||
/* Only wait if this task has attached to the ring */
|
||||
if (!tctx || !xa_load(&tctx->xa, (unsigned long)ctx))
|
||||
return;
|
||||
if (atomic_read(&tctx->in_cancel))
|
||||
return;
|
||||
|
||||
while (io_ring_ctx_inflight(ctx) && !fatal_signal_pending(current)) {
|
||||
|
||||
+2
-1
@@ -50,7 +50,8 @@ struct io_ctx_config {
|
||||
IORING_FEAT_RECVSEND_BUNDLE |\
|
||||
IORING_FEAT_MIN_TIMEOUT |\
|
||||
IORING_FEAT_RW_ATTR |\
|
||||
IORING_FEAT_NO_IOWAIT)
|
||||
IORING_FEAT_NO_IOWAIT |\
|
||||
IORING_FEAT_REGBUF_TYPES)
|
||||
|
||||
#define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\
|
||||
IORING_SETUP_SQPOLL |\
|
||||
|
||||
+55
-7
@@ -128,12 +128,24 @@ static void io_poll_req_insert(struct io_kiocb *req)
|
||||
hlist_add_head(&req->hash_node, &table->hbs[index].list);
|
||||
}
|
||||
|
||||
#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL)
|
||||
|
||||
static __poll_t io_poll_unmask(__poll_t events)
|
||||
{
|
||||
/*
|
||||
* A peer that stopped sending is an event for a reader, but a write
|
||||
* only poll must stay armed until the file is writable again.
|
||||
*/
|
||||
if (events & (EPOLLIN | EPOLLRDNORM | EPOLLRDBAND))
|
||||
return IO_POLL_UNMASK | EPOLLRDHUP;
|
||||
return IO_POLL_UNMASK;
|
||||
}
|
||||
|
||||
static void io_init_poll_iocb(struct io_poll *poll, __poll_t events)
|
||||
{
|
||||
poll->head = NULL;
|
||||
#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
|
||||
/* mask in events that we always want/need */
|
||||
poll->events = events | IO_POLL_UNMASK;
|
||||
poll->events = events | io_poll_unmask(events);
|
||||
INIT_LIST_HEAD(&poll->wait.entry);
|
||||
init_waitqueue_func_entry(&poll->wait, io_poll_wake);
|
||||
}
|
||||
@@ -727,7 +739,7 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask)
|
||||
int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)
|
||||
{
|
||||
const struct io_issue_def *def = &io_issue_defs[req->opcode];
|
||||
__poll_t mask = POLLPRI | POLLERR;
|
||||
__poll_t mask = EPOLLERR;
|
||||
|
||||
if (!def->pollin && !def->pollout)
|
||||
return IO_APOLL_ABORTED;
|
||||
@@ -735,7 +747,8 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)
|
||||
return IO_APOLL_ABORTED;
|
||||
|
||||
if (def->pollin) {
|
||||
mask |= EPOLLIN | EPOLLRDNORM;
|
||||
/* urgent data is an event for a reader only */
|
||||
mask |= EPOLLIN | EPOLLRDNORM | EPOLLPRI;
|
||||
|
||||
/* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
|
||||
if (req->flags & REQ_F_CLEAR_POLLIN)
|
||||
@@ -827,12 +840,47 @@ static int io_poll_disarm(struct io_kiocb *req)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Cancel every request matching @cd in one pass. Cancelled requests stay
|
||||
* hashed until their task work runs, so finding them one at a time would
|
||||
* walk past all the earlier ones on each lookup.
|
||||
*/
|
||||
static int io_poll_cancel_all(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
|
||||
{
|
||||
unsigned int nr_buckets = 1U << ctx->cancel_table.hash_bits;
|
||||
unsigned int i = 0;
|
||||
struct hlist_node *tmp;
|
||||
struct io_kiocb *req;
|
||||
int nr = 0;
|
||||
|
||||
if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP |
|
||||
IORING_ASYNC_CANCEL_ANY))) {
|
||||
/* keyed by user_data, so only one bucket can match */
|
||||
i = hash_long(cd->data, ctx->cancel_table.hash_bits);
|
||||
nr_buckets = i + 1;
|
||||
}
|
||||
|
||||
for (; i < nr_buckets; i++) {
|
||||
struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i];
|
||||
|
||||
hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) {
|
||||
if (!io_cancel_req_match(req, cd))
|
||||
continue;
|
||||
io_poll_cancel_req(req);
|
||||
nr++;
|
||||
}
|
||||
}
|
||||
return nr ?: -ENOENT;
|
||||
}
|
||||
|
||||
static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
|
||||
{
|
||||
struct io_kiocb *req;
|
||||
|
||||
if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP |
|
||||
IORING_ASYNC_CANCEL_ANY))
|
||||
if (cd->flags & (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_ANY))
|
||||
return io_poll_cancel_all(ctx, cd);
|
||||
|
||||
if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP))
|
||||
req = io_poll_file_find(ctx, cd);
|
||||
else
|
||||
req = io_poll_find(ctx, false, cd);
|
||||
@@ -971,7 +1019,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
|
||||
|
||||
poll->events &= ~0xffff;
|
||||
poll->events |= poll_update->events & 0xffff;
|
||||
poll->events |= IO_POLL_UNMASK;
|
||||
poll->events |= io_poll_unmask(poll->events);
|
||||
}
|
||||
if (poll_update->update_user_data)
|
||||
preq->cqe.user_data = poll_update->new_user_data;
|
||||
|
||||
+16
-5
@@ -572,11 +572,12 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req,
|
||||
{
|
||||
struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
|
||||
__s32 __user *fds = u64_to_user_ptr(up->arg);
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
unsigned int done;
|
||||
struct file *file;
|
||||
int ret, fd;
|
||||
|
||||
if (!req->ctx->file_table.data.nr)
|
||||
if (!ctx->file_table.data.nr)
|
||||
return -ENXIO;
|
||||
|
||||
for (done = 0; done < up->nr_args; done++) {
|
||||
@@ -590,15 +591,25 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req,
|
||||
ret = -EBADF;
|
||||
break;
|
||||
}
|
||||
ret = io_fixed_fd_install(req, issue_flags, file,
|
||||
IORING_FILE_INDEX_ALLOC);
|
||||
if (ret < 0)
|
||||
/*
|
||||
* Hold the lock across the put_user(), if it's dropped the
|
||||
* slot can be closed and reused before the error path runs.
|
||||
*/
|
||||
io_ring_submit_lock(ctx, issue_flags);
|
||||
ret = __io_fixed_fd_install(ctx, file,
|
||||
IORING_FILE_INDEX_ALLOC);
|
||||
if (ret < 0) {
|
||||
io_ring_submit_unlock(ctx, issue_flags);
|
||||
fput(file);
|
||||
break;
|
||||
}
|
||||
if (put_user(ret, &fds[done])) {
|
||||
__io_close_fixed(req->ctx, issue_flags, ret);
|
||||
io_fixed_fd_remove(ctx, ret);
|
||||
io_ring_submit_unlock(ctx, issue_flags);
|
||||
ret = -EFAULT;
|
||||
break;
|
||||
}
|
||||
io_ring_submit_unlock(ctx, issue_flags);
|
||||
}
|
||||
|
||||
if (done)
|
||||
|
||||
+18
-3
@@ -29,6 +29,7 @@
|
||||
enum {
|
||||
IO_SQ_THREAD_SHOULD_STOP = 0,
|
||||
IO_SQ_THREAD_SHOULD_PARK,
|
||||
IO_SQ_THREAD_EXITING,
|
||||
};
|
||||
|
||||
void io_sq_thread_unpark(struct io_sq_data *sqd)
|
||||
@@ -348,6 +349,18 @@ static int io_sq_thread(void *data)
|
||||
timeout = jiffies + sqd->sq_thread_idle;
|
||||
}
|
||||
|
||||
/*
|
||||
* The SQPOLL thread never returns to userspace, so do the
|
||||
* return-to-userspace work, like memory.high reclaim, here.
|
||||
* It can sleep for a long time, so drop sqd->lock.
|
||||
*/
|
||||
if (test_thread_flag(TIF_NOTIFY_RESUME)) {
|
||||
mutex_unlock(&sqd->lock);
|
||||
resume_user_mode_work(NULL);
|
||||
mutex_lock(&sqd->lock);
|
||||
sqd->sq_cpu = raw_smp_processor_id();
|
||||
}
|
||||
|
||||
cap_entries = !list_is_singular(&sqd->ctx_list);
|
||||
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
|
||||
int ret;
|
||||
@@ -436,9 +449,10 @@ static int io_sq_thread(void *data)
|
||||
/*
|
||||
* Cancelation drops sqd->lock while it waits, and a ring could attach
|
||||
* to a thread that is on its way out. A fatal signal doesn't set the
|
||||
* bit, so set it here for io_sq_offload_create() to see.
|
||||
* stop bit, so mark the thread as exiting here for
|
||||
* io_sq_offload_create() to see.
|
||||
*/
|
||||
set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
|
||||
set_bit(IO_SQ_THREAD_EXITING, &sqd->state);
|
||||
io_uring_cancel_generic(true, sqd);
|
||||
rcu_assign_pointer(sqd->thread, NULL);
|
||||
put_task_struct(current);
|
||||
@@ -510,7 +524,8 @@ __cold int io_sq_offload_create(struct io_ring_ctx *ctx,
|
||||
/* don't attach to a dying SQPOLL thread, would be racy */
|
||||
ret = 0;
|
||||
if (attached && (!sqd->thread ||
|
||||
test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)))
|
||||
test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state) ||
|
||||
test_bit(IO_SQ_THREAD_EXITING, &sqd->state)))
|
||||
ret = -ENXIO;
|
||||
io_sq_thread_unpark(sqd);
|
||||
|
||||
|
||||
@@ -165,6 +165,14 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
/*
|
||||
* Don't arm a timer for a timeout that has already passed, it would
|
||||
* only fire to return -ETIME. NAPI busy polling gets here like that.
|
||||
*/
|
||||
if (iowq->timeout != KTIME_MAX &&
|
||||
ktime_compare(io_get_time(ctx), iowq->timeout) >= 0)
|
||||
return -ETIME;
|
||||
|
||||
/*
|
||||
* Mark us as being in io_wait if we have pending requests, so cpufreq
|
||||
* can take into account that the task is waiting for IO - turns out
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <linux/io_uring.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
@@ -269,7 +270,7 @@ static inline void io_uring_prep_cmd(struct io_uring_sqe *sqe, int op,
|
||||
|
||||
sqe->level = level;
|
||||
sqe->optname = optname;
|
||||
sqe->optval = (unsigned long long)optval;
|
||||
sqe->optval = (unsigned long long)(uintptr_t)optval;
|
||||
sqe->optlen = optlen;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user