diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index b456f45fa8de..cf1afd11bb87 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -646,6 +646,7 @@ struct io_uring_params { #define IORING_FEAT_MIN_TIMEOUT (1U << 15) #define IORING_FEAT_RW_ATTR (1U << 16) #define IORING_FEAT_NO_IOWAIT (1U << 17) +#define IORING_FEAT_REGBUF_TYPES (1U << 18) /* * io_uring_register(2) opcodes and arguments diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 79b5c133319b..e3d9fe8fc982 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -198,7 +198,8 @@ static int __io_async_cancel(struct io_cancel_data *cd, break; if (!all) return ret; - nr++; + /* poll, futex and waitid return how many they cancelled */ + nr += max(ret, 1); } while (1); /* slow path, try all io-wq's */ diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 03724419c4e1..5db57bf675f0 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2547,7 +2547,10 @@ static __cold void io_ring_ctx_wait_inflight(struct io_ring_ctx *ctx) if (current->flags & (PF_KTHREAD | PF_EXITING)) return; - if (tctx && atomic_read(&tctx->in_cancel)) + /* Only wait if this task has attached to the ring */ + if (!tctx || !xa_load(&tctx->xa, (unsigned long)ctx)) + return; + if (atomic_read(&tctx->in_cancel)) return; while (io_ring_ctx_inflight(ctx) && !fatal_signal_pending(current)) { diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 32247529076c..a0ed03fab123 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -50,7 +50,8 @@ struct io_ctx_config { IORING_FEAT_RECVSEND_BUNDLE |\ IORING_FEAT_MIN_TIMEOUT |\ IORING_FEAT_RW_ATTR |\ - IORING_FEAT_NO_IOWAIT) + IORING_FEAT_NO_IOWAIT |\ + IORING_FEAT_REGBUF_TYPES) #define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\ IORING_SETUP_SQPOLL |\ diff --git a/io_uring/poll.c b/io_uring/poll.c index 4f04bda42f7f..fc1800c7097b 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -128,12 +128,24 @@ static void io_poll_req_insert(struct io_kiocb *req) hlist_add_head(&req->hash_node, &table->hbs[index].list); } +#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL) + +static __poll_t io_poll_unmask(__poll_t events) +{ + /* + * A peer that stopped sending is an event for a reader, but a write + * only poll must stay armed until the file is writable again. + */ + if (events & (EPOLLIN | EPOLLRDNORM | EPOLLRDBAND)) + return IO_POLL_UNMASK | EPOLLRDHUP; + return IO_POLL_UNMASK; +} + static void io_init_poll_iocb(struct io_poll *poll, __poll_t events) { poll->head = NULL; -#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP) /* mask in events that we always want/need */ - poll->events = events | IO_POLL_UNMASK; + poll->events = events | io_poll_unmask(events); INIT_LIST_HEAD(&poll->wait.entry); init_waitqueue_func_entry(&poll->wait, io_poll_wake); } @@ -727,7 +739,7 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) { const struct io_issue_def *def = &io_issue_defs[req->opcode]; - __poll_t mask = POLLPRI | POLLERR; + __poll_t mask = EPOLLERR; if (!def->pollin && !def->pollout) return IO_APOLL_ABORTED; @@ -735,7 +747,8 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) return IO_APOLL_ABORTED; if (def->pollin) { - mask |= EPOLLIN | EPOLLRDNORM; + /* urgent data is an event for a reader only */ + mask |= EPOLLIN | EPOLLRDNORM | EPOLLPRI; /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ if (req->flags & REQ_F_CLEAR_POLLIN) @@ -827,12 +840,47 @@ static int io_poll_disarm(struct io_kiocb *req) return 0; } +/* + * Cancel every request matching @cd in one pass. Cancelled requests stay + * hashed until their task work runs, so finding them one at a time would + * walk past all the earlier ones on each lookup. + */ +static int io_poll_cancel_all(struct io_ring_ctx *ctx, struct io_cancel_data *cd) +{ + unsigned int nr_buckets = 1U << ctx->cancel_table.hash_bits; + unsigned int i = 0; + struct hlist_node *tmp; + struct io_kiocb *req; + int nr = 0; + + if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP | + IORING_ASYNC_CANCEL_ANY))) { + /* keyed by user_data, so only one bucket can match */ + i = hash_long(cd->data, ctx->cancel_table.hash_bits); + nr_buckets = i + 1; + } + + for (; i < nr_buckets; i++) { + struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; + + hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { + if (!io_cancel_req_match(req, cd)) + continue; + io_poll_cancel_req(req); + nr++; + } + } + return nr ?: -ENOENT; +} + static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) { struct io_kiocb *req; - if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP | - IORING_ASYNC_CANCEL_ANY)) + if (cd->flags & (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_ANY)) + return io_poll_cancel_all(ctx, cd); + + if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP)) req = io_poll_file_find(ctx, cd); else req = io_poll_find(ctx, false, cd); @@ -971,7 +1019,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) poll->events &= ~0xffff; poll->events |= poll_update->events & 0xffff; - poll->events |= IO_POLL_UNMASK; + poll->events |= io_poll_unmask(poll->events); } if (poll_update->update_user_data) preq->cqe.user_data = poll_update->new_user_data; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index e0345c3e764e..8de5737d223e 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -572,11 +572,12 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, { struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); __s32 __user *fds = u64_to_user_ptr(up->arg); + struct io_ring_ctx *ctx = req->ctx; unsigned int done; struct file *file; int ret, fd; - if (!req->ctx->file_table.data.nr) + if (!ctx->file_table.data.nr) return -ENXIO; for (done = 0; done < up->nr_args; done++) { @@ -590,15 +591,25 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, ret = -EBADF; break; } - ret = io_fixed_fd_install(req, issue_flags, file, - IORING_FILE_INDEX_ALLOC); - if (ret < 0) + /* + * Hold the lock across the put_user(), if it's dropped the + * slot can be closed and reused before the error path runs. + */ + io_ring_submit_lock(ctx, issue_flags); + ret = __io_fixed_fd_install(ctx, file, + IORING_FILE_INDEX_ALLOC); + if (ret < 0) { + io_ring_submit_unlock(ctx, issue_flags); + fput(file); break; + } if (put_user(ret, &fds[done])) { - __io_close_fixed(req->ctx, issue_flags, ret); + io_fixed_fd_remove(ctx, ret); + io_ring_submit_unlock(ctx, issue_flags); ret = -EFAULT; break; } + io_ring_submit_unlock(ctx, issue_flags); } if (done) diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index c833e776320d..d387488db2de 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -29,6 +29,7 @@ enum { IO_SQ_THREAD_SHOULD_STOP = 0, IO_SQ_THREAD_SHOULD_PARK, + IO_SQ_THREAD_EXITING, }; void io_sq_thread_unpark(struct io_sq_data *sqd) @@ -348,6 +349,18 @@ static int io_sq_thread(void *data) timeout = jiffies + sqd->sq_thread_idle; } + /* + * The SQPOLL thread never returns to userspace, so do the + * return-to-userspace work, like memory.high reclaim, here. + * It can sleep for a long time, so drop sqd->lock. + */ + if (test_thread_flag(TIF_NOTIFY_RESUME)) { + mutex_unlock(&sqd->lock); + resume_user_mode_work(NULL); + mutex_lock(&sqd->lock); + sqd->sq_cpu = raw_smp_processor_id(); + } + cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { int ret; @@ -436,9 +449,10 @@ static int io_sq_thread(void *data) /* * Cancelation drops sqd->lock while it waits, and a ring could attach * to a thread that is on its way out. A fatal signal doesn't set the - * bit, so set it here for io_sq_offload_create() to see. + * stop bit, so mark the thread as exiting here for + * io_sq_offload_create() to see. */ - set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); + set_bit(IO_SQ_THREAD_EXITING, &sqd->state); io_uring_cancel_generic(true, sqd); rcu_assign_pointer(sqd->thread, NULL); put_task_struct(current); @@ -510,7 +524,8 @@ __cold int io_sq_offload_create(struct io_ring_ctx *ctx, /* don't attach to a dying SQPOLL thread, would be racy */ ret = 0; if (attached && (!sqd->thread || - test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state))) + test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state) || + test_bit(IO_SQ_THREAD_EXITING, &sqd->state))) ret = -ENXIO; io_sq_thread_unpark(sqd); diff --git a/io_uring/wait.c b/io_uring/wait.c index fb1c082993a7..367d49d631ec 100644 --- a/io_uring/wait.c +++ b/io_uring/wait.c @@ -165,6 +165,14 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx, { int ret = 0; + /* + * Don't arm a timer for a timeout that has already passed, it would + * only fire to return -ETIME. NAPI busy polling gets here like that. + */ + if (iowq->timeout != KTIME_MAX && + ktime_compare(io_get_time(ctx), iowq->timeout) >= 0) + return -ETIME; + /* * Mark us as being in io_wait if we have pending requests, so cpufreq * can take into account that the task is waiting for IO - turns out diff --git a/tools/include/io_uring/mini_liburing.h b/tools/include/io_uring/mini_liburing.h index 44be4446feda..6afe7efb4497 100644 --- a/tools/include/io_uring/mini_liburing.h +++ b/tools/include/io_uring/mini_liburing.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -269,7 +270,7 @@ static inline void io_uring_prep_cmd(struct io_uring_sqe *sqe, int op, sqe->level = level; sqe->optname = optname; - sqe->optval = (unsigned long long)optval; + sqe->optval = (unsigned long long)(uintptr_t)optval; sqe->optlen = optlen; }