Merge tag 'io_uring-7.4-20261112' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull io_uring fix from Jens Axboe:

 - Don't end the NAPI busy poll early on need_resched() when a single
   NAPI is registered, as that left the rest of the wait timeout
   sleeping in schedule_hrtimeout() rather than busy polling.

* tag 'io_uring-7.4-20261112' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux:
  io_uring/napi: don't end the single NAPI busy poll early on need_resched()
This commit is contained in:
Linus Torvalds
2026-11-13 10:20:23 -08:00
2 changed files with 15 additions and 17 deletions
+1
View File
@@ -140,6 +140,7 @@ struct io_wait_queue {
#ifdef CONFIG_NET_RX_BUSY_POLL
ktime_t napi_busy_poll_dt;
ktime_t napi_busy_poll_end;
bool napi_prefer_busy_poll;
#endif
};
+14 -17
View File
@@ -128,19 +128,15 @@ static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale)
__io_napi_remove_stale(ctx);
}
static inline bool io_napi_busy_loop_timeout(ktime_t start_time,
ktime_t bp)
static inline bool io_napi_busy_loop_timeout(struct io_wait_queue *iowq)
{
if (bp) {
ktime_t end_time = ktime_add(start_time, bp);
ktime_t now = net_to_ktime(busy_loop_current_time());
return ktime_after(now, end_time);
}
return true;
if (!iowq->napi_busy_poll_dt)
return true;
return ktime_after(net_to_ktime(busy_loop_current_time()),
iowq->napi_busy_poll_end);
}
/* Uses an absolute deadline, so the napi loop can be re-entered */
static bool io_napi_busy_loop_should_end(void *data,
unsigned long start_time)
{
@@ -150,8 +146,7 @@ static bool io_napi_busy_loop_should_end(void *data,
return true;
if (io_should_wake(iowq) || io_has_work(iowq->ctx))
return true;
if (io_napi_busy_loop_timeout(net_to_ktime(start_time),
iowq->napi_busy_poll_dt))
if (io_napi_busy_loop_timeout(iowq))
return true;
return false;
@@ -209,13 +204,16 @@ __io_napi_do_busy_loop(struct io_ring_ctx *ctx,
static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx,
struct io_wait_queue *iowq)
{
unsigned long start_time = busy_loop_current_time();
ktime_t now = net_to_ktime(busy_loop_current_time());
bool (*loop_end)(void *, unsigned long) = NULL;
void *loop_end_arg = NULL;
bool is_stale = false;
/* Singular lists use a different napi loop end check function and are
* only executed once.
iowq->napi_busy_poll_end = ktime_add(now, iowq->napi_busy_poll_dt);
/*
* A single NAPI is held by __napi_busy_loop() for the whole spin. It
* may return early on need_resched(), loop until our own deadline.
*/
if (list_is_singular(&ctx->napi_list)) {
loop_end = io_napi_busy_loop_should_end;
@@ -226,8 +224,7 @@ static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx,
do {
is_stale = __io_napi_do_busy_loop(ctx, loop_end,
loop_end_arg);
} while (!io_napi_busy_loop_should_end(iowq, start_time) &&
!loop_end_arg);
} while (!io_napi_busy_loop_should_end(iowq, 0));
}
io_napi_remove_stale(ctx, is_stale);