mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
io_uring: do not charge the SQ/CQ rings to RLIMIT_MEMLOCK
Commit8078486e1d("io_uring: use region api for SQ") and commit81a4058e0c("io_uring: use region api for CQ") made io_uring_setup() allocate the rings with io_create_region(). However, io_create_region() charges the memory to RLIMIT_MEMLOCK, and the rings had been exempt from that limit since commit26bfa89e25("io_uring: place ring SQ/CQ arrays under memcg memory limits"). As of now, a user without CAP_IPC_LOCK gets ENOMEM from io_uring_setup() when their rings exceed the limit, which is 8 MiB by default. PostgreSQL developers have already hit this in their io_uring tests [1]. The issue can be reproduced with a simple liburing program, run as an unprivileged user: #include <liburing.h> #include <stdio.h> int main(void) { static struct io_uring ring[64]; int i; for (i = 0; i < 64; i++) if (io_uring_queue_init(4096, &ring[i], 0) < 0) break; printf("%d rings\n", i); return 0; } Before those commits (v6.13), it prints "64 rings". After those commits (v6.14), it prints "21 rings". This patch makes io_create_region() take the user to charge, and passes no user for the SQ/CQ rings. Link: https://www.postgresql.org/message-id/flat/[email protected] [1] Fixes:8078486e1d("io_uring: use region api for SQ") Fixes:81a4058e0c("io_uring: use region api for CQ") Cc: [email protected] Signed-off-by: Hengyu Liang <[email protected]> Link: https://lore.kernel.org/io-uring/[email protected]/ Signed-off-by: Jens Axboe <[email protected]>
This commit is contained in:
+6
-4
@@ -2070,8 +2070,9 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
|
||||
|
||||
static void io_rings_free(struct io_ring_ctx *ctx)
|
||||
{
|
||||
io_free_region(ctx->user, &ctx->sq_region);
|
||||
io_free_region(ctx->user, &ctx->ring_region);
|
||||
/* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
|
||||
io_free_region(NULL, &ctx->sq_region);
|
||||
io_free_region(NULL, &ctx->ring_region);
|
||||
ctx->rings = NULL;
|
||||
RCU_INIT_POINTER(ctx->rings_rcu, NULL);
|
||||
ctx->sq_sqes = NULL;
|
||||
@@ -2735,7 +2736,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
|
||||
rd.user_addr = p->cq_off.user_addr;
|
||||
rd.flags |= IORING_MEM_REGION_TYPE_USER;
|
||||
}
|
||||
ret = io_create_region(ctx, &ctx->ring_region, &rd, IORING_OFF_CQ_RING);
|
||||
/* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
|
||||
ret = io_create_region(NULL, &ctx->ring_region, &rd, IORING_OFF_CQ_RING);
|
||||
if (ret)
|
||||
return ret;
|
||||
ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
|
||||
@@ -2749,7 +2751,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
|
||||
rd.user_addr = p->sq_off.user_addr;
|
||||
rd.flags |= IORING_MEM_REGION_TYPE_USER;
|
||||
}
|
||||
ret = io_create_region(ctx, &ctx->sq_region, &rd, IORING_OFF_SQES);
|
||||
ret = io_create_region(NULL, &ctx->sq_region, &rd, IORING_OFF_SQES);
|
||||
if (ret) {
|
||||
io_rings_free(ctx);
|
||||
return ret;
|
||||
|
||||
+1
-1
@@ -676,7 +676,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
|
||||
rd.user_addr = reg.ring_addr;
|
||||
rd.flags |= IORING_MEM_REGION_TYPE_USER;
|
||||
}
|
||||
ret = io_create_region(ctx, &bl->region, &rd, mmap_offset);
|
||||
ret = io_create_region(ctx->user, &bl->region, &rd, mmap_offset);
|
||||
if (ret)
|
||||
goto fail;
|
||||
br = io_region_get_ptr(&bl->region);
|
||||
|
||||
+5
-5
@@ -204,7 +204,7 @@ done:
|
||||
return 0;
|
||||
}
|
||||
|
||||
int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
|
||||
int io_create_region(struct user_struct *user, struct io_mapped_region *mr,
|
||||
struct io_uring_region_desc *reg,
|
||||
unsigned long mmap_offset)
|
||||
{
|
||||
@@ -230,8 +230,8 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
|
||||
return -EOVERFLOW;
|
||||
|
||||
nr_pages = reg->size >> PAGE_SHIFT;
|
||||
if (ctx->user) {
|
||||
ret = __io_account_mem(ctx->user, nr_pages);
|
||||
if (user) {
|
||||
ret = __io_account_mem(user, nr_pages);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
@@ -240,7 +240,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
|
||||
if (reg->flags & IORING_MEM_REGION_TYPE_USER)
|
||||
ret = io_region_pin_pages(mr, reg);
|
||||
else
|
||||
ret = io_region_allocate_pages(mr, reg, mmap_offset, ctx->user);
|
||||
ret = io_region_allocate_pages(mr, reg, mmap_offset, user);
|
||||
if (ret)
|
||||
goto out_free;
|
||||
|
||||
@@ -249,7 +249,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
|
||||
goto out_free;
|
||||
return 0;
|
||||
out_free:
|
||||
io_free_region(ctx->user, mr);
|
||||
io_free_region(user, mr);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr,
|
||||
int io_uring_mmap(struct file *file, struct vm_area_struct *vma);
|
||||
|
||||
void io_free_region(struct user_struct *user, struct io_mapped_region *mr);
|
||||
int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
|
||||
int io_create_region(struct user_struct *user, struct io_mapped_region *mr,
|
||||
struct io_uring_region_desc *reg,
|
||||
unsigned long mmap_offset);
|
||||
|
||||
|
||||
+7
-5
@@ -480,8 +480,9 @@ struct io_ring_ctx_rings {
|
||||
static void io_register_free_rings(struct io_ring_ctx *ctx,
|
||||
struct io_ring_ctx_rings *r)
|
||||
{
|
||||
io_free_region(ctx->user, &r->sq_region);
|
||||
io_free_region(ctx->user, &r->ring_region);
|
||||
/* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
|
||||
io_free_region(NULL, &r->sq_region);
|
||||
io_free_region(NULL, &r->ring_region);
|
||||
}
|
||||
|
||||
#define swap_old(ctx, o, n, field) \
|
||||
@@ -529,7 +530,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
|
||||
rd.user_addr = p->cq_off.user_addr;
|
||||
rd.flags |= IORING_MEM_REGION_TYPE_USER;
|
||||
}
|
||||
ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING);
|
||||
ret = io_create_region(NULL, &n.ring_region, &rd, IORING_OFF_CQ_RING);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
@@ -559,7 +560,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
|
||||
rd.user_addr = p->sq_off.user_addr;
|
||||
rd.flags |= IORING_MEM_REGION_TYPE_USER;
|
||||
}
|
||||
ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES);
|
||||
ret = io_create_region(NULL, &n.sq_region, &rd, IORING_OFF_SQES);
|
||||
if (ret) {
|
||||
io_register_free_rings(ctx, &n);
|
||||
return ret;
|
||||
@@ -730,7 +731,8 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
|
||||
!(ctx->flags & IORING_SETUP_R_DISABLED))
|
||||
return -EINVAL;
|
||||
|
||||
ret = io_create_region(ctx, ®ion, &rd, IORING_MAP_OFF_PARAM_REGION);
|
||||
ret = io_create_region(ctx->user, ®ion, &rd,
|
||||
IORING_MAP_OFF_PARAM_REGION);
|
||||
if (ret)
|
||||
return ret;
|
||||
if (copy_to_user(rd_uptr, &rd, sizeof(rd))) {
|
||||
|
||||
+1
-1
@@ -432,7 +432,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
|
||||
mmap_offset = IORING_MAP_OFF_ZCRX_REGION;
|
||||
mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT;
|
||||
|
||||
ret = io_create_region(ctx, &ifq->rq_region, rd, mmap_offset);
|
||||
ret = io_create_region(ctx->user, &ifq->rq_region, rd, mmap_offset);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user