diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index a0ed03fab123..c8592f4ef510 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -140,6 +140,7 @@ struct io_wait_queue { #ifdef CONFIG_NET_RX_BUSY_POLL ktime_t napi_busy_poll_dt; + ktime_t napi_busy_poll_end; bool napi_prefer_busy_poll; #endif }; diff --git a/io_uring/napi.c b/io_uring/napi.c index d299c89cfc3d..0724c608777d 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -128,19 +128,15 @@ static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale) __io_napi_remove_stale(ctx); } -static inline bool io_napi_busy_loop_timeout(ktime_t start_time, - ktime_t bp) +static inline bool io_napi_busy_loop_timeout(struct io_wait_queue *iowq) { - if (bp) { - ktime_t end_time = ktime_add(start_time, bp); - ktime_t now = net_to_ktime(busy_loop_current_time()); - - return ktime_after(now, end_time); - } - - return true; + if (!iowq->napi_busy_poll_dt) + return true; + return ktime_after(net_to_ktime(busy_loop_current_time()), + iowq->napi_busy_poll_end); } +/* Uses an absolute deadline, so the napi loop can be re-entered */ static bool io_napi_busy_loop_should_end(void *data, unsigned long start_time) { @@ -150,8 +146,7 @@ static bool io_napi_busy_loop_should_end(void *data, return true; if (io_should_wake(iowq) || io_has_work(iowq->ctx)) return true; - if (io_napi_busy_loop_timeout(net_to_ktime(start_time), - iowq->napi_busy_poll_dt)) + if (io_napi_busy_loop_timeout(iowq)) return true; return false; @@ -209,13 +204,16 @@ __io_napi_do_busy_loop(struct io_ring_ctx *ctx, static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) { - unsigned long start_time = busy_loop_current_time(); + ktime_t now = net_to_ktime(busy_loop_current_time()); bool (*loop_end)(void *, unsigned long) = NULL; void *loop_end_arg = NULL; bool is_stale = false; - /* Singular lists use a different napi loop end check function and are - * only executed once. + iowq->napi_busy_poll_end = ktime_add(now, iowq->napi_busy_poll_dt); + + /* + * A single NAPI is held by __napi_busy_loop() for the whole spin. It + * may return early on need_resched(), loop until our own deadline. */ if (list_is_singular(&ctx->napi_list)) { loop_end = io_napi_busy_loop_should_end; @@ -226,8 +224,7 @@ static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, do { is_stale = __io_napi_do_busy_loop(ctx, loop_end, loop_end_arg); - } while (!io_napi_busy_loop_should_end(iowq, start_time) && - !loop_end_arg); + } while (!io_napi_busy_loop_should_end(iowq, 0)); } io_napi_remove_stale(ctx, is_stale);