Files
Shakeel Butt f013b69550 io_uring/sqpoll: do the return-to-userspace work in the SQPOLL thread
The SQPOLL thread never returns to userspace. io_sq_tw() runs its
task_work, but the rest of resume_user_mode_work(), like memory.high
reclaim and block cgroup throttling, only runs when the thread exits.
So memory.high reclaim only happens inside a charge, with sqd->lock and
ctx->uring_lock held, and not at all if the charges can't block.

This used to work. The loop called io_run_task_work(), which does this
work since commit b5d3ae202f ("io_uring: handle TIF_NOTIFY_RESUME
when checking for task_work"). Commit af5d68f889 ("io_uring/sqpoll:
manage task_work privately") replaced that call with io_sq_tw().

Call resume_user_mode_work() at the top of the loop when
TIF_NOTIFY_RESUME is set, with sqd->lock dropped. It runs once per pass,
so a pass that charges two or more batches still reclaims inside the
charge, as before.

Tested in a VM with memory.high=64M and swap on zram. An SQPOLL ring
reads 256M from /dev/zero into untouched memory, 64K at a time. Before,
the thread made 337 calls to __mem_cgroup_handle_over_high(), all from
inside the charge. After, it makes 704, all from the top of the loop.
memory.peak stays at 64M. With 1M reads, 296 of its 385 calls still come
from inside the charge. Both runs take about 50% longer, and the extra
time is in the handler, which now runs more often. Without memory.high
pressure, the test never calls resume_user_mode_work() and the times
do not change. Lockdep showed no warnings, also with io_uring_register()
racing for sqd->lock.

Fixes: af5d68f889 ("io_uring/sqpoll: manage task_work privately")
Cc: [email protected]
Signed-off-by: Shakeel Butt <[email protected]>
Link: https://patch.msgid.link/[email protected]
Signed-off-by: Jens Axboe <[email protected]>
2026-11-03 07:16:51 -07:00

614 lines
15 KiB
C

// SPDX-License-Identifier: GPL-2.0
/*
* Contains the core associated with submission side polling of the SQ
* ring, offloading submissions from the application to a kernel thread.
*/
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/audit.h>
#include <linux/security.h>
#include <linux/cpuset.h>
#include <linux/sched/cputime.h>
#include <linux/io_uring.h>
#include <linux/kcov.h>
#include <uapi/linux/io_uring.h>
#include "io_uring.h"
#include "tctx.h"
#include "napi.h"
#include "cancel.h"
#include "sqpoll.h"
#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
#define IORING_TW_CAP_ENTRIES_VALUE 32
enum {
IO_SQ_THREAD_SHOULD_STOP = 0,
IO_SQ_THREAD_SHOULD_PARK,
IO_SQ_THREAD_EXITING,
};
void io_sq_thread_unpark(struct io_sq_data *sqd)
__releases(&sqd->lock)
{
WARN_ON_ONCE(sqpoll_task_locked(sqd) == current);
/*
* Do the dance but not conditional clear_bit() because it'd race with
* other threads incrementing park_pending and setting the bit.
*/
clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
if (atomic_dec_return(&sqd->park_pending))
set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
mutex_unlock(&sqd->lock);
wake_up(&sqd->wait);
}
void io_sq_thread_park(struct io_sq_data *sqd)
__acquires(&sqd->lock)
{
struct task_struct *tsk;
atomic_inc(&sqd->park_pending);
set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
mutex_lock(&sqd->lock);
tsk = sqpoll_task_locked(sqd);
if (tsk) {
WARN_ON_ONCE(tsk == current);
wake_up_process(tsk);
}
}
/*
* The sqd has two holders: its rings, as one, and its thread. Neither waits
* for the other to let go. The one that lets go last frees the sqd.
*/
static void io_sq_data_release(struct io_sq_data *sqd)
{
if (refcount_dec_and_test(&sqd->free_refs))
kfree(sqd);
}
/*
* Ask the thread to exit, and don't wait for it. A thread that is stopped or
* frozen exits only when a user lets it run again, and the exit work of the
* ring would keep its place in the workqueue until then. All rings have left
* ctx_list by now and have no request left, so the thread can reach none of
* them. What it still uses is its own or counted: the sqd by free_refs, the
* hash of its io-wq by hash->refs.
*/
void io_sq_thread_stop(struct io_sq_data *sqd)
{
struct task_struct *tsk;
WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state));
set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
mutex_lock(&sqd->lock);
tsk = sqpoll_task_locked(sqd);
if (tsk) {
WARN_ON_ONCE(tsk == current);
wake_up_process(tsk);
}
mutex_unlock(&sqd->lock);
}
void io_put_sq_data(struct io_sq_data *sqd)
{
if (refcount_dec_and_test(&sqd->refs)) {
WARN_ON_ONCE(atomic_read(&sqd->park_pending));
io_sq_thread_stop(sqd);
io_sq_data_release(sqd);
}
}
static __cold void io_sqd_update_thread_idle(struct io_sq_data *sqd)
{
struct io_ring_ctx *ctx;
unsigned sq_thread_idle = 0;
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
sqd->sq_thread_idle = sq_thread_idle;
}
void io_sq_thread_finish(struct io_ring_ctx *ctx)
{
struct io_sq_data *sqd = ctx->sq_data;
if (sqd) {
io_sq_thread_park(sqd);
list_del_init(&ctx->sqd_list);
io_sqd_update_thread_idle(sqd);
io_sq_thread_unpark(sqd);
io_put_sq_data(sqd);
ctx->sq_data = NULL;
}
}
static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
{
struct io_ring_ctx *ctx_attach;
struct io_sq_data *sqd;
CLASS(fd, f)(p->wq_fd);
if (fd_empty(f))
return ERR_PTR(-ENXIO);
if (!io_is_uring_fops(fd_file(f)))
return ERR_PTR(-EINVAL);
ctx_attach = fd_file(f)->private_data;
sqd = ctx_attach->sq_data;
if (!sqd)
return ERR_PTR(-EINVAL);
if (sqd->task_tgid != current->tgid)
return ERR_PTR(-EPERM);
refcount_inc(&sqd->refs);
return sqd;
}
static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
bool *attached)
{
struct io_sq_data *sqd;
*attached = false;
if (p->flags & IORING_SETUP_ATTACH_WQ) {
sqd = io_attach_sq_data(p);
if (!IS_ERR(sqd)) {
*attached = true;
return sqd;
}
/* fall through for EPERM case, setup new sqd/task */
if (PTR_ERR(sqd) != -EPERM)
return sqd;
}
sqd = kzalloc_obj(*sqd);
if (!sqd)
return ERR_PTR(-ENOMEM);
atomic_set(&sqd->park_pending, 0);
refcount_set(&sqd->refs, 1);
refcount_set(&sqd->free_refs, 2);
INIT_LIST_HEAD(&sqd->ctx_list);
mutex_init(&sqd->lock);
init_waitqueue_head(&sqd->wait);
return sqd;
}
static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
{
return READ_ONCE(sqd->state);
}
struct io_sq_time {
bool started;
u64 usec;
};
u64 io_sq_cpu_usec(struct task_struct *tsk)
{
u64 utime, stime;
task_cputime_adjusted(tsk, &utime, &stime);
do_div(stime, 1000);
return stime;
}
static void io_sq_update_worktime(struct io_sq_data *sqd, struct io_sq_time *ist)
{
if (!ist->started)
return;
ist->started = false;
sqd->work_time += io_sq_cpu_usec(current) - ist->usec;
}
static void io_sq_start_worktime(struct io_sq_time *ist)
{
if (ist->started)
return;
ist->started = true;
ist->usec = io_sq_cpu_usec(current);
}
static int __io_sq_thread(struct io_ring_ctx *ctx, struct io_sq_data *sqd,
bool cap_entries, struct io_sq_time *ist)
{
unsigned int to_submit;
int ret = 0;
to_submit = io_sqring_entries(ctx);
/* if we're handling multiple rings, cap submit size for fairness */
if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;
if (to_submit || !list_empty(&ctx->iopoll_list)) {
const struct cred *creds = NULL;
io_sq_start_worktime(ist);
if (ctx->sq_creds != current_cred())
creds = override_creds(ctx->sq_creds);
mutex_lock(&ctx->uring_lock);
if (!list_empty(&ctx->iopoll_list))
io_do_iopoll(ctx, true);
/*
* Don't submit if refs are dying, good for io_uring_register(),
* but also it is relied upon by io_ring_exit_work()
*/
if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
!(ctx->flags & IORING_SETUP_R_DISABLED))
ret = io_submit_sqes(ctx, to_submit);
mutex_unlock(&ctx->uring_lock);
if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
wake_up(&ctx->sqo_sq_wait);
if (creds)
revert_creds(creds);
}
return ret;
}
static bool io_sqd_handle_event(struct io_sq_data *sqd)
{
bool did_sig = false;
struct ksignal ksig;
if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
signal_pending(current)) {
mutex_unlock(&sqd->lock);
if (signal_pending(current))
did_sig = get_signal(&ksig);
wait_event(sqd->wait, !atomic_read(&sqd->park_pending));
mutex_lock(&sqd->lock);
sqd->sq_cpu = raw_smp_processor_id();
}
return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
}
/*
* Run task_work, processing no more than max_entries at a time. If more
* than that is pending, it simply stays on the queue for the next run.
*/
static unsigned int io_sq_tw(int max_entries)
{
struct io_uring_task *tctx = current->io_uring;
unsigned int count = 0;
tctx_task_work_run(tctx, max_entries, &count);
if (task_work_pending(current))
task_work_run();
return count;
}
static bool io_sq_tw_pending(void)
{
struct io_uring_task *tctx = current->io_uring;
return !mpscq_empty(&tctx->task_list);
}
static int io_sq_thread(void *data)
{
struct io_sq_data *sqd = data;
struct io_ring_ctx *ctx;
unsigned long timeout = 0;
char buf[TASK_COMM_LEN] = {};
DEFINE_WAIT(wait);
/* offload context creation failed, just exit */
if (!current->io_uring) {
mutex_lock(&sqd->lock);
rcu_assign_pointer(sqd->thread, NULL);
put_task_struct(current);
mutex_unlock(&sqd->lock);
goto err_out;
}
snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
set_task_comm(current, buf);
/* reset to our pid after we've set task_comm, for fdinfo */
sqd->task_pid = current->pid;
if (sqd->sq_cpu != -1) {
set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
} else {
set_cpus_allowed_ptr(current, cpu_online_mask);
sqd->sq_cpu = raw_smp_processor_id();
}
/*
* Force audit context to get setup, in case we do prep side async
* operations that would trigger an audit call before any issue side
* audit has been done.
*/
audit_uring_entry(IORING_OP_NOP);
audit_uring_exit(true, 0);
mutex_lock(&sqd->lock);
while (1) {
bool cap_entries, sqt_spin = false;
struct io_sq_time ist = { };
if (io_sqd_events_pending(sqd) || signal_pending(current)) {
if (io_sqd_handle_event(sqd))
break;
timeout = jiffies + sqd->sq_thread_idle;
}
/*
* The SQPOLL thread never returns to userspace, so do the
* return-to-userspace work, like memory.high reclaim, here.
* It can sleep for a long time, so drop sqd->lock.
*/
if (test_thread_flag(TIF_NOTIFY_RESUME)) {
mutex_unlock(&sqd->lock);
resume_user_mode_work(NULL);
mutex_lock(&sqd->lock);
sqd->sq_cpu = raw_smp_processor_id();
}
cap_entries = !list_is_singular(&sqd->ctx_list);
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
int ret;
kcov_remote_start_common(ctx->kcov_handle);
ret = __io_sq_thread(ctx, sqd, cap_entries, &ist);
if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
sqt_spin = true;
kcov_remote_stop();
}
if (io_sq_tw(IORING_TW_CAP_ENTRIES_VALUE))
sqt_spin = true;
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
if (io_napi(ctx)) {
io_sq_start_worktime(&ist);
io_napi_sqpoll_busy_poll(ctx);
}
}
io_sq_update_worktime(sqd, &ist);
if (sqt_spin || !time_after(jiffies, timeout)) {
if (sqt_spin)
timeout = jiffies + sqd->sq_thread_idle;
if (unlikely(need_resched())) {
mutex_unlock(&sqd->lock);
cond_resched();
mutex_lock(&sqd->lock);
sqd->sq_cpu = raw_smp_processor_id();
}
continue;
}
prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
if (!io_sqd_events_pending(sqd) && !io_sq_tw_pending()) {
bool needs_sched = true;
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
atomic_or(IORING_SQ_NEED_WAKEUP,
&ctx->rings->sq_flags);
if ((ctx->flags & IORING_SETUP_IOPOLL) &&
!list_empty(&ctx->iopoll_list)) {
needs_sched = false;
break;
}
/*
* Ensure the store of the wakeup flag is not
* reordered with the load of the SQ tail
*/
smp_mb__after_atomic();
/*
* Nothing gets submitted before the ring is
* enabled, which comes with a wake up.
*/
if (READ_ONCE(ctx->flags) & IORING_SETUP_R_DISABLED)
continue;
if (io_sqring_entries(ctx)) {
needs_sched = false;
break;
}
}
if (needs_sched) {
mutex_unlock(&sqd->lock);
schedule();
mutex_lock(&sqd->lock);
sqd->sq_cpu = raw_smp_processor_id();
}
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
atomic_andnot(IORING_SQ_NEED_WAKEUP,
&ctx->rings->sq_flags);
}
finish_wait(&sqd->wait, &wait);
timeout = jiffies + sqd->sq_thread_idle;
}
if (io_sq_tw_pending())
io_sq_tw(UINT_MAX);
/*
* Cancelation drops sqd->lock while it waits, and a ring could attach
* to a thread that is on its way out. A fatal signal doesn't set the
* stop bit, so mark the thread as exiting here for
* io_sq_offload_create() to see.
*/
set_bit(IO_SQ_THREAD_EXITING, &sqd->state);
io_uring_cancel_generic(true, sqd);
rcu_assign_pointer(sqd->thread, NULL);
put_task_struct(current);
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
atomic_or(IORING_SQ_NEED_WAKEUP, &ctx->rings->sq_flags);
io_run_task_work();
mutex_unlock(&sqd->lock);
err_out:
io_sq_data_release(sqd);
do_exit(0);
}
void io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
{
DEFINE_WAIT(wait);
do {
if (!io_sqring_full(ctx))
break;
prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
if (!io_sqring_full(ctx))
break;
schedule();
} while (!signal_pending(current));
finish_wait(&ctx->sqo_sq_wait, &wait);
}
__cold int io_sq_offload_create(struct io_ring_ctx *ctx,
struct io_uring_params *p)
{
int ret;
/* Retain compatibility with failing for an invalid attach attempt */
if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
IORING_SETUP_ATTACH_WQ) {
CLASS(fd, f)(p->wq_fd);
if (fd_empty(f))
return -ENXIO;
if (!io_is_uring_fops(fd_file(f)))
return -EINVAL;
}
if (ctx->flags & IORING_SETUP_SQPOLL) {
struct io_uring_task *tctx;
struct task_struct *tsk;
struct io_sq_data *sqd;
bool attached;
ret = security_uring_sqpoll();
if (ret)
return ret;
sqd = io_get_sq_data(p, &attached);
if (IS_ERR(sqd)) {
ret = PTR_ERR(sqd);
goto err;
}
ctx->sq_creds = get_current_cred();
ctx->sq_data = sqd;
ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
if (!ctx->sq_thread_idle)
ctx->sq_thread_idle = HZ;
io_sq_thread_park(sqd);
list_add(&ctx->sqd_list, &sqd->ctx_list);
io_sqd_update_thread_idle(sqd);
/* don't attach to a dying SQPOLL thread, would be racy */
ret = 0;
if (attached && (!sqd->thread ||
test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state) ||
test_bit(IO_SQ_THREAD_EXITING, &sqd->state)))
ret = -ENXIO;
io_sq_thread_unpark(sqd);
if (ret < 0)
goto err;
if (attached)
return 0;
if (p->flags & IORING_SETUP_SQ_AFF) {
cpumask_var_t allowed_mask;
int cpu = p->sq_thread_cpu;
ret = -EINVAL;
if (cpu >= nr_cpu_ids || !cpu_online(cpu))
goto err_sqpoll;
ret = -ENOMEM;
if (!alloc_cpumask_var(&allowed_mask, GFP_KERNEL))
goto err_sqpoll;
ret = -EINVAL;
cpuset_cpus_allowed(current, allowed_mask);
if (!cpumask_test_cpu(cpu, allowed_mask)) {
free_cpumask_var(allowed_mask);
goto err_sqpoll;
}
free_cpumask_var(allowed_mask);
sqd->sq_cpu = cpu;
} else {
sqd->sq_cpu = -1;
}
sqd->task_pid = current->pid;
sqd->task_tgid = current->tgid;
tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
if (IS_ERR(tsk)) {
ret = PTR_ERR(tsk);
goto err_sqpoll;
}
mutex_lock(&sqd->lock);
rcu_assign_pointer(sqd->thread, tsk);
mutex_unlock(&sqd->lock);
ret = 0;
get_task_struct(tsk);
tctx = io_uring_alloc_task_context(tsk, ctx);
if (!IS_ERR(tctx))
tsk->io_uring = tctx;
else
ret = PTR_ERR(tctx);
wake_up_new_task(tsk);
if (ret)
goto err;
} else if (p->flags & IORING_SETUP_SQ_AFF) {
/* Can't have SQ_AFF without SQPOLL */
ret = -EINVAL;
goto err;
}
return 0;
err_sqpoll:
/* no thread was made: let go of its share */
io_sq_data_release(ctx->sq_data);
err:
io_sq_thread_finish(ctx);
return ret;
}
__cold int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx,
cpumask_var_t mask)
{
struct io_sq_data *sqd = ctx->sq_data;
int ret = -EINVAL;
if (sqd) {
struct task_struct *tsk;
io_sq_thread_park(sqd);
/* Don't set affinity for a dying thread */
tsk = sqpoll_task_locked(sqd);
if (tsk)
ret = io_wq_cpu_affinity(tsk->io_uring, mask);
io_sq_thread_unpark(sqd);
}
return ret;
}