From e7d73cef9726bea49de97ae1fd76405df4050a75 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Sun, 1 Nov 2026 11:57:20 +0100 Subject: [PATCH 01/28] libfs: make i_rwsem the lock of the offset tree The offset tree of a directory is only changed with the directory's i_rwsem held exclusive and offset_readdir() walks it with i_rwsem held shared. But the maple tree doesn't know that. It is set up with its internal spinlock, which every mtree_*() helper takes, and readdir takes rcu_read_lock() around each lookup. The next patch makes readdir keep one ma_state across dir_emit(), which can sleep. That is safe because no writer can run while readdir holds i_rwsem. But the maple tree only allows keeping a ma_state across a sleep if it is protected by the tree's lock. A reader that relies on rcu_read_lock() has to mas_pause() before it drops it, and that means a walk from the root for every entry again. So tell the tree what protects it. Initialize it with MT_FLAGS_LOCK_EXTERN and the directory's i_rwsem as its lock and switch the writers from the mtree_*() helpers to the mas_*() api. They all run with i_rwsem held exclusive already. With lockdep enabled the maple tree now also checks that every writer holds i_rwsem exclusive. Nothing can reach the offset tree anymore when the directory inode is destroyed. So clear the external lock with mt_on_stack() and destroy the tree with __mt_destroy() instead of taking i_rwsem in ->destroy_inode(). Cc: stable@vger.kernel.org # v6.14+ Link: https://patch.msgid.link/20261101-work-libfs-offset_readdir-v2-1-c621125b56d1@kernel.org Reviewed-by: Chuck Lever Signed-off-by: Christian Brauner (Amutable) --- fs/libfs.c | 46 +++++++++++++++++++++++++++++----------------- include/linux/fs.h | 2 +- mm/shmem.c | 2 +- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/fs/libfs.c b/fs/libfs.c index 96a6e9b1535f..653ee8461a8f 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -263,17 +263,18 @@ static long dentry2offset(struct dentry *dentry) return (long)dentry->d_fsdata; } -static struct lock_class_key simple_offset_lock_class; - /** * simple_offset_init - initialize an offset_ctx * @octx: directory offset map to be initialized + * @dir: the directory @octx belongs to * + * The directory's i_rwsem is the lock of the offset map. Writers hold it + * exclusive, offset_readdir() holds it shared. */ -void simple_offset_init(struct offset_ctx *octx) +void simple_offset_init(struct offset_ctx *octx, struct inode *dir) { - mt_init_flags(&octx->mt, MT_FLAGS_ALLOC_RANGE); - lockdep_set_class(&octx->mt.ma_lock, &simple_offset_lock_class); + mt_init_flags(&octx->mt, MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN); + mt_set_external_lock(&octx->mt, &dir->i_rwsem); octx->next_offset = DIR_OFFSET_MIN; } @@ -290,12 +291,13 @@ int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry) unsigned long offset; int ret; + MA_STATE(mas, &octx->mt, 0, 0); + if (dentry2offset(dentry) != 0) return -EBUSY; - ret = mtree_alloc_cyclic(&octx->mt, &offset, dentry, DIR_OFFSET_MIN, - DIR_OFFSET_MAX, &octx->next_offset, - GFP_KERNEL); + ret = mas_alloc_cyclic(&mas, &offset, dentry, DIR_OFFSET_MIN, + DIR_OFFSET_MAX, &octx->next_offset, GFP_KERNEL); if (unlikely(ret < 0)) return ret == -EBUSY ? -ENOSPC : ret; @@ -308,7 +310,9 @@ static int simple_offset_replace(struct offset_ctx *octx, struct dentry *dentry, { int ret; - ret = mtree_store(&octx->mt, offset, dentry, GFP_KERNEL); + MA_STATE(mas, &octx->mt, offset, offset); + + ret = mas_store_gfp(&mas, dentry, GFP_KERNEL); if (ret) return ret; offset_set(dentry, offset); @@ -323,13 +327,14 @@ static int simple_offset_replace(struct offset_ctx *octx, struct dentry *dentry, */ void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry) { - long offset; + long offset = dentry2offset(dentry); + + MA_STATE(mas, &octx->mt, offset, offset); - offset = dentry2offset(dentry); if (offset == 0) return; - mtree_erase(&octx->mt, offset); + mas_erase(&mas); offset_set(dentry, 0); } @@ -387,16 +392,20 @@ int simple_offset_rename_exchange(struct inode *old_dir, long new_index = dentry2offset(new_dentry); int ret; + MA_STATE(old_mas, &old_ctx->mt, old_index, old_index); + MA_STATE(new_mas, &new_ctx->mt, new_index, new_index); + if (WARN_ON(!old_index || !new_index)) return -EINVAL; - ret = mtree_store(&new_ctx->mt, new_index, old_dentry, GFP_KERNEL); + ret = mas_store_gfp(&new_mas, old_dentry, GFP_KERNEL); if (WARN_ON(ret)) return ret; - ret = mtree_store(&old_ctx->mt, old_index, new_dentry, GFP_KERNEL); + ret = mas_store_gfp(&old_mas, new_dentry, GFP_KERNEL); if (WARN_ON(ret)) { - mtree_store(&new_ctx->mt, new_index, new_dentry, GFP_KERNEL); + mas_set(&new_mas, new_index); + mas_store_gfp(&new_mas, new_dentry, GFP_KERNEL); return ret; } @@ -411,11 +420,14 @@ int simple_offset_rename_exchange(struct inode *old_dir, * @octx: directory offset ctx that is about to be destroyed * * During fs teardown (eg. umount), a directory's offset map might still - * contain entries. xa_destroy() cleans out anything that remains. + * contain entries. __mt_destroy() cleans out anything that remains. + * Nothing can reach the map of a directory that is being destroyed, so + * this runs without the directory's i_rwsem. */ void simple_offset_destroy(struct offset_ctx *octx) { - mtree_destroy(&octx->mt); + mt_on_stack(octx->mt); + __mt_destroy(&octx->mt); } /** diff --git a/include/linux/fs.h b/include/linux/fs.h index 99caef43ff3c..acea6c40caaa 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3332,7 +3332,7 @@ struct offset_ctx { unsigned long next_offset; }; -void simple_offset_init(struct offset_ctx *octx); +void simple_offset_init(struct offset_ctx *octx, struct inode *dir); int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry); void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry); void simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry, diff --git a/mm/shmem.c b/mm/shmem.c index af21a89abb0f..580ef4226f98 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -3287,7 +3287,7 @@ static struct inode *__shmem_get_inode(const struct mnt_idmap *idmap, inode->i_size = 2 * BOGO_DIRENT_SIZE; inode->i_op = &shmem_dir_inode_operations; inode->i_fop = &simple_offset_dir_operations; - simple_offset_init(shmem_get_offset_ctx(inode)); + simple_offset_init(shmem_get_offset_ctx(inode), inode); break; case S_IFLNK: /* From 309f7a5585aeaf77d3d7c27304ac97e515447dac Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Sun, 1 Nov 2026 11:57:21 +0100 Subject: [PATCH 02/28] libfs: walk offset directories in offset order offset_iterate_dir() walks d_children from the head and only uses the maple tree to map ctx->pos back to a dentry when a getdents64() call resumes. offset_dir_lookup() finds the entry at ctx->pos, or the one with the next lower offset if it is gone, and the list walk continues from there. That only works if list order and offset order agree and quite often they don't. A rename over an existing name gives the moved dentry the offset of the name it replaced but __d_move() puts it at the head of d_children. RENAME_EXCHANGE does the same to both dentries. And once the cyclic offset allocator wraps new entries get low offsets at the head of the list. So if the entry at ctx->pos is unlinked between two getdents64() calls we resume at the entry with the next lower offset and that one can sit anywhere in the list: (1) If it sits further down the list everything in between is skipped. Create y, t, a, b and rename b over t. The list is now t(4) a(5) y(3). Stop the listing at t and unlink t. The next call resumes at y and a is never returned although nobody touched it. (2) If there is no lower entry we restart at the head and return the whole directory again. Nickolai reported this for the simple case of unlinking the oldest entry while a listing is about to reach it. Ending the walk there instead turns this into (1): create a, b, c, rename c over a, stop the listing at the renamed entry, unlink it, and b is lost. Instead of the list let's walk the maple tree itself in descending offset order. Then ctx->pos really means that everything above it has been emitted and resuming at the highest offset at or below it is right no matter where the dentry sits in d_children. Entries created or renamed to a new name after the listing started get an offset above ctx->pos and aren't returned, same as with the list walk which never went back to the head. So a readdir loop that renames or creates an entry for each one it reads still terminates. The maple tree can't change under us as all modifications happen with the directory's i_rwsem held exclusive and offset_readdir() holds it shared. That also covers the dentries in the tree. They are all persistent and only unlink, rmdir and rename remove or rename them. And since the previous patch i_rwsem is the lock of the tree. So keep a single ma_state across the walk and emit directly from the dentry without taking rcu_read_lock(), a reference or d_lock. This also makes the walk cheaper than the list walk was. With a million entries on tmpfs a getdents64() listing goes from 146ms to 114ms. With four processes listing the directory at the same time it goes from 470ms to 113ms per listing as they don't bounce the parent's d_lock and the dentries' lockrefs between them anymore. base v2 this 1M entries, getdents64 32k buffer (ms) 146 127 114 1M entries, getdents64 1k buffer (ms) 184 156 141 1M entries, getdents64 128b buffer (ms) 429 364 335 1M entries, ls -f (s) 0.259 0.239 0.228 1M entries, LC_ALL=C ls (s) 0.482 0.467 0.455 1M entries, 4 parallel listers (ms) 470 125 113 18 entries, rewind and list (ns) 4107 3594 3323 250k of 1M left, getdents64 32k (ms) 58.8 57.8 53.8 250k of 1M left, getdents64 128b (ms) 128 114 108 Fixes: b9b588f22a0c ("libfs: Use d_children list to iterate simple_offset directories") Fixes: f70681e9e606 ("libfs: Fix duplicate directory entry in offset_dir_lookup") Reported-by: Nickolai Zeldovich Link: https://patch.msgid.link/20261018024549.798968-1-nickolai@csail.mit.edu Link: https://patch.msgid.link/20261101-work-libfs-offset_readdir-v2-2-c621125b56d1@kernel.org Reviewed-by: Chuck Lever Signed-off-by: Christian Brauner (Amutable) --- fs/libfs.c | 85 ++++++++++++++---------------------------------------- 1 file changed, 22 insertions(+), 63 deletions(-) diff --git a/fs/libfs.c b/fs/libfs.c index 653ee8461a8f..654f487b05c0 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -283,6 +283,12 @@ void simple_offset_init(struct offset_ctx *octx, struct inode *dir) * @octx: directory offset ctx to be updated * @dentry: new dentry being added * + * The offset map holds no reference to @dentry. Once positive, @dentry + * must be pinned with d_make_persistent() for as long as it is in the + * map, and entries may only be added, removed or renamed with the + * directory's i_rwsem held exclusive: offset_readdir() emits straight + * from the map under the shared lock. + * * Returns zero on success. @octx and the dentry's offset are updated. * Otherwise, a negative errno value is returned. */ @@ -458,51 +464,6 @@ static loff_t offset_dir_llseek(struct file *file, loff_t offset, int whence) return vfs_setpos(file, offset, LONG_MAX); } -static struct dentry *find_positive_dentry(struct dentry *parent, - struct dentry *dentry, - bool next) -{ - struct dentry *found = NULL; - - spin_lock(&parent->d_lock); - if (next) - dentry = d_next_sibling(dentry); - else if (!dentry) - dentry = d_first_child(parent); - hlist_for_each_entry_from(dentry, d_sib) { - if (!simple_positive(dentry)) - continue; - spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); - if (simple_positive(dentry)) - found = dget_dlock(dentry); - spin_unlock(&dentry->d_lock); - if (likely(found)) - break; - } - spin_unlock(&parent->d_lock); - return found; -} - -static noinline_for_stack struct dentry * -offset_dir_lookup(struct dentry *parent, loff_t offset) -{ - struct inode *inode = d_inode(parent); - struct offset_ctx *octx = inode->i_op->get_offset_ctx(inode); - struct dentry *child, *found = NULL; - - MA_STATE(mas, &octx->mt, offset, offset); - - if (offset == DIR_OFFSET_FIRST) - found = find_positive_dentry(parent, NULL, false); - else { - rcu_read_lock(); - child = mas_find_rev(&mas, DIR_OFFSET_MIN); - found = find_positive_dentry(parent, child, false); - rcu_read_unlock(); - } - return found; -} - static bool offset_dir_emit(struct dir_context *ctx, struct dentry *dentry) { struct inode *inode = d_inode(dentry); @@ -511,32 +472,30 @@ static bool offset_dir_emit(struct dir_context *ctx, struct dentry *dentry) inode->i_ino, fs_umode_to_dtype(inode->i_mode)); } +/* + * Emit entries in descending offset order. The directory's i_rwsem is the + * lock of the offset tree and the persistent dentries in it only change + * with it held exclusive. We hold it shared, so @mas stays valid across + * dir_emit() and the dentries need no references. + */ static void offset_iterate_dir(struct file *file, struct dir_context *ctx) { - struct dentry *dir = file->f_path.dentry; + struct inode *inode = file_inode(file); + struct offset_ctx *octx = inode->i_op->get_offset_ctx(inode); + loff_t start = ctx->pos == DIR_OFFSET_FIRST ? DIR_OFFSET_MAX : ctx->pos; struct dentry *dentry; - dentry = offset_dir_lookup(dir, ctx->pos); - if (!dentry) - goto out_eod; - while (true) { - struct dentry *next; + MA_STATE(mas, &octx->mt, start, start); + for (;;) { + dentry = mas_find_rev(&mas, DIR_OFFSET_MIN); + if (!dentry) + break; + VFS_WARN_ON_ONCE(!(dentry->d_flags & DCACHE_PERSISTENT)); ctx->pos = dentry2offset(dentry); if (!offset_dir_emit(ctx, dentry)) - break; - - next = find_positive_dentry(dir, dentry, true); - dput(dentry); - - if (!next) - goto out_eod; - dentry = next; + return; } - dput(dentry); - return; - -out_eod: ctx->pos = DIR_OFFSET_EOD; } From 45eb9737058fa78a73f17a5da55c8cb482c6d5bb Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Sun, 1 Nov 2026 11:57:22 +0100 Subject: [PATCH 03/28] libfs: end offset readdir at positions past DIR_OFFSET_EOD offset_dir_llseek() takes any position up to LONG_MAX but offset_readdir() only skips the walk for DIR_OFFSET_EOD itself. For a position past it mas_find_rev() starts the walk at the highest offset in the directory. So an lseek() past the end makes the next getdents64() return the whole directory again. Before commit b9b588f22a0c ("libfs: Use d_children list to iterate simple_offset directories") the walk went up from the position and found nothing past the end, and dcache_readdir() returns nothing there either. So only walk for positions below DIR_OFFSET_EOD. Fixes: b9b588f22a0c ("libfs: Use d_children list to iterate simple_offset directories") Link: https://patch.msgid.link/20261101-work-libfs-offset_readdir-v2-3-c621125b56d1@kernel.org Reviewed-by: Chuck Lever Signed-off-by: Christian Brauner (Amutable) --- fs/libfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/libfs.c b/fs/libfs.c index 654f487b05c0..46e0205154e7 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -530,7 +530,7 @@ static int offset_readdir(struct file *file, struct dir_context *ctx) if (!dir_emit_dots(file, ctx)) return 0; - if (ctx->pos != DIR_OFFSET_EOD) + if (ctx->pos < DIR_OFFSET_EOD) offset_iterate_dir(file, ctx); return 0; } From a210eb87cd3d19e929d1a35e2430bdc85f3eb59b Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 27 Oct 2026 14:53:22 -0700 Subject: [PATCH 04/28] nilfs2: convert iomap ops to ->iomap_next() Convert nilfs2 iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20261027215326.2693300-2-joannelkoong@gmail.com Acked-by: Ryusuke Konishi Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/nilfs2/iomap.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/nilfs2/iomap.c b/fs/nilfs2/iomap.c index e130ed63abd9..359786c97f2b 100644 --- a/fs/nilfs2/iomap.c +++ b/fs/nilfs2/iomap.c @@ -67,6 +67,8 @@ static int nilfs_iomap_begin(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(nilfs_iomap_next, nilfs_iomap_begin); + const struct iomap_ops nilfs_iomap_ops = { - .iomap_begin = nilfs_iomap_begin, + .iomap_next = nilfs_iomap_next, }; From f6c38b3d56c23dc80635477a83535c2b886c3275 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 27 Oct 2026 14:53:23 -0700 Subject: [PATCH 05/28] fuse: convert ext_map iomap ops to ->iomap_next() Convert the fuse ext_map iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20261027215326.2693300-3-joannelkoong@gmail.com Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/fuse/ext_map.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/fuse/ext_map.c b/fs/fuse/ext_map.c index 7f04edc33fe3..edb19498d496 100644 --- a/fs/fuse/ext_map.c +++ b/fs/fuse/ext_map.c @@ -210,8 +210,11 @@ static int fuse_ext_map_iomap_begin(struct inode *inode, loff_t offset, loff_t l return 0; } +static DEFINE_IOMAP_ITER_NEXT(fuse_ext_map_iomap_next, + fuse_ext_map_iomap_begin); + static const struct iomap_ops fuse_ext_map_iomap_ops = { - .iomap_begin = fuse_ext_map_iomap_begin, + .iomap_next = fuse_ext_map_iomap_next, }; static vm_fault_t fuse_ext_map_huge_fault(struct vm_fault *vmf, unsigned int order) From 82f677376a90791acfdf5223e6c2f18d3c3c8732 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 27 Oct 2026 14:53:24 -0700 Subject: [PATCH 06/28] iomap: remove ->iomap_begin()/->iomap_end() legacy path With all in-tree filesystems implementing ->iomap_next(), drop the ->iomap_begin()/->iomap_end() path. Reviewed-by: Fengnan Chang Reviewed-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20261027215326.2693300-4-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/iter.c | 6 +----- include/linux/iomap.h | 6 ++---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index fd8791c4963b..8b6da9750a93 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -119,11 +119,7 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) trace_iomap_iter(iter, ops, _RET_IP_); - if (ops->iomap_next) - ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap); - else - ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap, - ops->iomap_begin, ops->iomap_end); + ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap); iter->status = 0; if (ret > 0) diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 5254430844b2..66211ddd96cb 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -195,7 +195,7 @@ struct iomap_write_ops { }; /* - * Flags for iomap_begin / iomap_end. No flag implies a read. + * Flags for iomap_next. No flag implies a read. */ #define IOMAP_WRITE (1 << 0) /* writing, must allocate blocks */ #define IOMAP_ZERO (1 << 1) /* zeroing operation, may skip holes */ @@ -240,8 +240,6 @@ typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter, struct iomap *iomap, struct iomap *srcmap); struct iomap_ops { - iomap_iter_begin_fn iomap_begin; - iomap_iter_end_fn iomap_end; iomap_iter_next_fn iomap_next; }; @@ -256,7 +254,7 @@ struct iomap_ops { * incremental iter advance. * @status: Status of the most recent iteration. Zero on success or a negative * errno on error. - * @flags: Zero or more of the iomap_begin flags above. + * @flags: Zero or more of the iomap_next flags above. * @iomap: Map describing the I/O iteration * @srcmap: Source map for COW operations */ From 9e8fc16dc6b474b9a87843c3d3ee12fc83f2bdcc Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 27 Oct 2026 14:53:25 -0700 Subject: [PATCH 07/28] iomap: pass iomap_iter_next_fn directly instead of struct iomap_ops Now that all filesystems implement ->iomap_next() and the legacy ->iomap_begin()/->iomap_end() fallback is gone, struct iomap_ops only wraps a single iomap_next function pointer. Drop the struct entirely and pass the iomap_iter_next_fn directly to iomap_iter() and all the iomap/dax entry points. Filesystems pass their iomap_iter_next_fn callback directly instead of an ops struct. Several iomap_iter_next_fn callbacks become non-static and gain header declarations since callers now reference them directly. The iomap_iter tracepoint's ops field is renamed to next to match. These tracepoints are explicitly not stable ABI (see the note in iomap/trace.h). No functional change intended other than the tracepoint field rename noted above. Acked-by: Jan Kara Reviewed-by: Christoph Hellwig Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20261027215326.2693300-5-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- block/fops.c | 10 ++----- fs/btrfs/direct-io.c | 8 ++---- fs/dax.c | 48 ++++++++++++++++---------------- fs/erofs/data.c | 28 ++++++++----------- fs/erofs/internal.h | 3 +- fs/erofs/zmap.c | 7 +---- fs/exfat/file.c | 14 +++++----- fs/exfat/inode.c | 6 ++-- fs/exfat/iomap.c | 22 ++++----------- fs/exfat/iomap.h | 6 ++-- fs/ext2/ext2.h | 3 +- fs/ext2/file.c | 4 +-- fs/ext2/inode.c | 9 ++---- fs/ext4/ext4.h | 10 ++++--- fs/ext4/extents.c | 8 ++---- fs/ext4/file.c | 22 +++++++-------- fs/ext4/inode.c | 58 ++++++++++++--------------------------- fs/f2fs/data.c | 6 +--- fs/f2fs/f2fs.h | 3 +- fs/f2fs/file.c | 4 +-- fs/fuse/dax.c | 10 ++----- fs/fuse/ext_map.c | 10 ++----- fs/fuse/file.c | 10 ++----- fs/fuse/virtio_fs.c | 3 +- fs/gfs2/aops.c | 6 ++-- fs/gfs2/bmap.c | 9 ++---- fs/gfs2/bmap.h | 2 +- fs/gfs2/file.c | 6 ++-- fs/gfs2/inode.c | 6 ++-- fs/hpfs/file.c | 6 +--- fs/internal.h | 1 - fs/iomap/buffered-io.c | 40 +++++++++++++-------------- fs/iomap/direct-io.c | 8 +++--- fs/iomap/fiemap.c | 8 +++--- fs/iomap/iter.c | 9 +++--- fs/iomap/seek.c | 8 +++--- fs/iomap/swapfile.c | 4 +-- fs/iomap/trace.h | 12 ++++---- fs/nilfs2/file.c | 2 +- fs/nilfs2/iomap.c | 6 +--- fs/nilfs2/iomap.h | 4 ++- fs/ntfs/aops.c | 6 ++-- fs/ntfs/file.c | 20 +++++++------- fs/ntfs/inode.c | 2 +- fs/ntfs/iomap.c | 42 ++++++---------------------- fs/ntfs/iomap.h | 10 +++---- fs/ntfs3/file.c | 16 +++++------ fs/ntfs3/inode.c | 13 +++------ fs/ntfs3/ntfs_fs.h | 3 +- fs/remap_range.c | 6 ++-- fs/xfs/xfs_aops.c | 8 +++--- fs/xfs/xfs_file.c | 53 ++++++++++++++++++------------------ fs/xfs/xfs_iomap.c | 55 ++++++++----------------------------- fs/xfs/xfs_iomap.h | 16 +++++------ fs/xfs/xfs_iops.c | 4 +-- fs/xfs/xfs_reflink.c | 6 ++-- fs/zonefs/file.c | 22 +++++---------- include/linux/dax.h | 18 ++++++------ include/linux/fs.h | 7 +++-- include/linux/iomap.h | 62 ++++++++++++++++++++++-------------------- 60 files changed, 338 insertions(+), 480 deletions(-) diff --git a/block/fops.c b/block/fops.c index e5c27cc35ca1..806c32216d80 100644 --- a/block/fops.c +++ b/block/fops.c @@ -467,10 +467,6 @@ static int blkdev_iomap_begin(struct inode *inode, loff_t offset, loff_t length, static DEFINE_IOMAP_ITER_NEXT(blkdev_iomap_next, blkdev_iomap_begin); -static const struct iomap_ops blkdev_iomap_ops = { - .iomap_next = blkdev_iomap_next, -}; - #ifdef CONFIG_BUFFER_HEAD static int blkdev_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh, int create) @@ -524,13 +520,13 @@ const struct address_space_operations def_blk_aops = { #else /* CONFIG_BUFFER_HEAD */ static int blkdev_read_folio(struct file *file, struct folio *folio) { - iomap_bio_read_folio(folio, &blkdev_iomap_ops); + iomap_bio_read_folio(folio, blkdev_iomap_next); return 0; } static void blkdev_readahead(struct readahead_control *rac) { - iomap_bio_readahead(rac, &blkdev_iomap_ops); + iomap_bio_readahead(rac, blkdev_iomap_next); } static ssize_t blkdev_writeback_range(struct iomap_writepage_ctx *wpc, @@ -736,7 +732,7 @@ static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from) * i_blkbits/folio order and truncating the page cache. */ inode_lock_shared(inode); - ret = iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL, + ret = iomap_file_buffered_write(iocb, from, blkdev_iomap_next, NULL, NULL); inode_unlock_shared(inode); return ret; diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index ed3914257d39..90919d79d069 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -807,10 +807,6 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, static DEFINE_IOMAP_ITER_NEXT_END(btrfs_dio_iomap_next, btrfs_dio_iomap_begin, btrfs_dio_iomap_end); -static const struct iomap_ops btrfs_dio_iomap_ops = { - .iomap_next = btrfs_dio_iomap_next, -}; - static const struct iomap_dio_ops btrfs_dio_ops = { .submit_io = btrfs_dio_submit_io, .bio_set = &btrfs_dio_bioset, @@ -821,7 +817,7 @@ static ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, { struct btrfs_dio_data data = { 0 }; - return iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, + return iomap_dio_rw(iocb, iter, btrfs_dio_iomap_next, &btrfs_dio_ops, IOMAP_DIO_PARTIAL | IOMAP_DIO_FSBLOCK_ALIGNED, &data, done_before); } @@ -858,7 +854,7 @@ static struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *it dio_flags |= IOMAP_DIO_BOUNCE; } - return __iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, + return __iomap_dio_rw(iocb, iter, btrfs_dio_iomap_next, &btrfs_dio_ops, dio_flags, &data, done_before); } diff --git a/fs/dax.c b/fs/dax.c index 94ba41273347..9d161d7ef3ef 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1485,7 +1485,7 @@ out_unlock: } int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct iomap_iter iter = { .inode = inode, @@ -1499,7 +1499,7 @@ int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len, return 0; iter.len = min(len, size - pos); - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = dax_unshare_iter(&iter); return ret; } @@ -1577,7 +1577,7 @@ static int dax_zero_iter(struct iomap_iter *iter, bool *did_zero) } int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct iomap_iter iter = { .inode = inode, @@ -1587,14 +1587,14 @@ int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, }; int ret; - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = dax_zero_iter(&iter, did_zero); return ret; } EXPORT_SYMBOL_GPL(dax_zero_range); int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { unsigned int blocksize = i_blocksize(inode); unsigned int off = pos & (blocksize - 1); @@ -1602,7 +1602,7 @@ int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, /* Block boundary? Nothing to do */ if (!off) return 0; - return dax_zero_range(inode, pos, blocksize - off, did_zero, ops); + return dax_zero_range(inode, pos, blocksize - off, did_zero, next); } EXPORT_SYMBOL_GPL(dax_truncate_page); @@ -1727,7 +1727,8 @@ static int dax_iomap_iter(struct iomap_iter *iomi, struct iov_iter *iter) * dax_iomap_rw - Perform I/O to a DAX file * @iocb: The control block for this I/O * @iter: The addresses to do I/O from or to - * @ops: iomap ops passed from the file system + * @next: Callback passed from the file system for advancing to next + * iteration * * This function performs read and write operations to directly mapped * persistent memory. The callers needs to take care of read/write exclusion @@ -1735,7 +1736,7 @@ static int dax_iomap_iter(struct iomap_iter *iomi, struct iov_iter *iter) */ ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct iomap_iter iomi = { .inode = iocb->ki_filp->f_mapping->host, @@ -1762,7 +1763,7 @@ dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, if (iocb->ki_flags & IOCB_NOWAIT) iomi.flags |= IOMAP_NOWAIT; - while ((ret = iomap_iter(&iomi, ops)) > 0) + while ((ret = iomap_iter(&iomi, next)) > 0) iomi.status = dax_iomap_iter(&iomi, iter); done = iomi.pos - iocb->ki_pos; @@ -1890,7 +1891,7 @@ static vm_fault_t dax_fault_iter(struct vm_fault *vmf, } static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, unsigned long *pfnp, - int *iomap_errp, const struct iomap_ops *ops) + int *iomap_errp, iomap_iter_next_fn next) { struct address_space *mapping = vmf->vma->vm_file->f_mapping; XA_STATE(xas, &mapping->i_pages, vmf->pgoff); @@ -1935,7 +1936,7 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, unsigned long *pfnp, goto unlock_entry; } - while ((error = iomap_iter(&iter, ops)) > 0) { + while ((error = iomap_iter(&iter, next)) > 0) { if (WARN_ON_ONCE(iomap_length(&iter) < PAGE_SIZE)) { iter.status = -EIO; /* fs corruption? */ continue; @@ -2000,7 +2001,7 @@ static bool dax_fault_check_fallback(struct vm_fault *vmf, struct xa_state *xas, } static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct address_space *mapping = vmf->vma->vm_file->f_mapping; XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER); @@ -2057,7 +2058,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp, } iter.pos = (loff_t)xas.xa_index << PAGE_SHIFT; - while (iomap_iter(&iter, ops) > 0) { + while (iomap_iter(&iter, next) > 0) { if (iomap_length(&iter) < PMD_SIZE) continue; /* actually breaks out of the loop */ @@ -2079,7 +2080,7 @@ out: } #else static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { return VM_FAULT_FALLBACK; } @@ -2091,7 +2092,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp, * @order: Order of the page to fault in * @pfnp: PFN to insert for synchronous faults if fsync is required * @iomap_errp: Storage for detailed error code in case of error - * @ops: Iomap ops passed from the file system + * @next: The iomap iteration function for the filesystem * * When a page fault occurs, filesystems may call this helper in * their fault handler for DAX files. dax_iomap_fault() assumes the caller @@ -2100,12 +2101,12 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp, */ vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order, unsigned long *pfnp, int *iomap_errp, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { if (order == 0) - return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops); + return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, next); else if (order == PMD_ORDER) - return dax_iomap_pmd_fault(vmf, pfnp, ops); + return dax_iomap_pmd_fault(vmf, pfnp, next); else return VM_FAULT_FALLBACK; } @@ -2235,7 +2236,7 @@ out_unlock: int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff, struct inode *dst, loff_t dstoff, loff_t len, bool *same, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct iomap_iter src_iter = { .inode = src, @@ -2251,8 +2252,8 @@ int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff, }; int ret, status; - while ((ret = iomap_iter(&src_iter, ops)) > 0 && - (ret = iomap_iter(&dst_iter, ops)) > 0) { + while ((ret = iomap_iter(&src_iter, next)) > 0 && + (ret = iomap_iter(&dst_iter, next)) > 0) { status = dax_range_compare_iter(&src_iter, &dst_iter, min(src_iter.len, dst_iter.len), same); if (status < 0) @@ -2265,9 +2266,10 @@ int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff, int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, loff_t *len, unsigned int remap_flags, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { return __generic_remap_file_range_prep(file_in, pos_in, file_out, - pos_out, len, remap_flags, ops); + pos_out, len, remap_flags, + next); } EXPORT_SYMBOL_GPL(dax_remap_file_range_prep); diff --git a/fs/erofs/data.c b/fs/erofs/data.c index 968303afd620..1f3b06464880 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -374,10 +374,6 @@ static int erofs_iomap_end(struct inode *inode, loff_t pos, loff_t length, static DEFINE_IOMAP_ITER_NEXT_END(erofs_iomap_next, erofs_iomap_begin, erofs_iomap_end); -static const struct iomap_ops erofs_iomap_ops = { - .iomap_next = erofs_iomap_next, -}; - int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { @@ -385,9 +381,9 @@ int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP)) return -EOPNOTSUPP; return iomap_fiemap(inode, fieinfo, start, len, - &z_erofs_iomap_report_ops); + z_erofs_iomap_next_report); } - return iomap_fiemap(inode, fieinfo, start, len, &erofs_iomap_ops); + return iomap_fiemap(inode, fieinfo, start, len, erofs_iomap_next); } /* @@ -408,7 +404,7 @@ static int erofs_read_folio(struct file *file, struct folio *folio) }; trace_erofs_read_folio(iter_ctx.realinode, folio, true); - iomap_read_folio(&erofs_iomap_ops, &read_ctx, &iter_ctx); + iomap_read_folio(erofs_iomap_next, &read_ctx, &iter_ctx); if (need_iput) iput(iter_ctx.realinode); return 0; @@ -429,14 +425,14 @@ static void erofs_readahead(struct readahead_control *rac) trace_erofs_readahead(iter_ctx.realinode, readahead_index(rac), readahead_count(rac), true); - iomap_readahead(&erofs_iomap_ops, &read_ctx, &iter_ctx); + iomap_readahead(erofs_iomap_next, &read_ctx, &iter_ctx); if (need_iput) iput(iter_ctx.realinode); } static sector_t erofs_bmap(struct address_space *mapping, sector_t block) { - return iomap_bmap(mapping, block, &erofs_iomap_ops); + return iomap_bmap(mapping, block, erofs_iomap_next); } static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) @@ -448,14 +444,14 @@ static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return 0; if (IS_ENABLED(CONFIG_FS_DAX) && IS_DAX(inode)) - return dax_iomap_rw(iocb, to, &erofs_iomap_ops); + return dax_iomap_rw(iocb, to, erofs_iomap_next); if ((iocb->ki_flags & IOCB_DIRECT) && inode->i_sb->s_bdev) { struct erofs_iomap_iter_ctx iter_ctx = { .realinode = inode, }; - return iomap_dio_rw(iocb, to, &erofs_iomap_ops, + return iomap_dio_rw(iocb, to, erofs_iomap_next, NULL, 0, &iter_ctx, 0); } return filemap_read(iocb, to, 0); @@ -474,7 +470,7 @@ const struct address_space_operations erofs_aops = { static vm_fault_t erofs_dax_huge_fault(struct vm_fault *vmf, unsigned int order) { - return dax_iomap_fault(vmf, order, NULL, NULL, &erofs_iomap_ops); + return dax_iomap_fault(vmf, order, NULL, NULL, erofs_iomap_next); } static vm_fault_t erofs_dax_fault(struct vm_fault *vmf) @@ -506,18 +502,18 @@ static int erofs_file_mmap_prepare(struct vm_area_desc *desc) loff_t erofs_file_llseek(struct file *file, loff_t offset, int whence) { struct inode *inode = file->f_mapping->host; - const struct iomap_ops *ops = &erofs_iomap_ops; + iomap_iter_next_fn next = erofs_iomap_next; if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) { if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP)) return generic_file_llseek(file, offset, whence); - ops = &z_erofs_iomap_report_ops; + next = z_erofs_iomap_next_report; } if (whence == SEEK_HOLE) - offset = iomap_seek_hole(inode, offset, ops); + offset = iomap_seek_hole(inode, offset, next); else if (whence == SEEK_DATA) - offset = iomap_seek_data(inode, offset, ops); + offset = iomap_seek_data(inode, offset, next); else return generic_file_llseek(file, offset, whence); diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 8363c0c051f6..c16dfa752cf9 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -397,9 +397,10 @@ struct erofs_iomap_iter_ctx { struct inode *realinode; }; -extern const struct iomap_ops z_erofs_iomap_report_ops; extern const struct iomap_read_ops erofs_fileio_iomap_read_ops; +DECLARE_IOMAP_ITER_NEXT(z_erofs_iomap_next_report); + int erofs_setup_managed_cache(struct super_block *sb); #ifdef CONFIG_EROFS_FS_BACKED_BY_FILE int erofs_read_meta_folio(struct file *file, struct folio *folio); diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index f8981fc74246..e0d5987b742b 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -830,9 +830,4 @@ static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, return 0; } -static DEFINE_IOMAP_ITER_NEXT(z_erofs_iomap_next_report, - z_erofs_iomap_begin_report); - -const struct iomap_ops z_erofs_iomap_report_ops = { - .iomap_next = z_erofs_iomap_next_report, -}; +DEFINE_IOMAP_ITER_NEXT(z_erofs_iomap_next_report, z_erofs_iomap_begin_report); diff --git a/fs/exfat/file.c b/fs/exfat/file.c index 1386149916bb..435bbef288b8 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -828,7 +828,7 @@ static ssize_t exfat_fallback_buffered_write(struct kiocb *iocb, inode_dio_wait(file_inode(iocb->ki_filp)); - written = iomap_file_buffered_write(iocb, from, &exfat_write_iomap_ops, + written = iomap_file_buffered_write(iocb, from, exfat_write_iomap_next, NULL, NULL); if (written < 0) return written; @@ -850,7 +850,7 @@ static ssize_t exfat_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) { ssize_t ret; - ret = iomap_dio_rw(iocb, from, &exfat_write_iomap_ops, + ret = iomap_dio_rw(iocb, from, exfat_write_iomap_next, &exfat_write_dio_ops, 0, NULL, 0); if (ret == -ENOTBLK) ret = 0; @@ -921,7 +921,7 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter) */ inode_dio_wait(inode); ret = iomap_file_buffered_write(iocb, iter, - &exfat_write_iomap_ops, NULL, NULL); + exfat_write_iomap_next, NULL, NULL); } if (ret != -EIOCBQUEUED && end > i_size_read(inode)) exfat_write_failed(inode); @@ -957,7 +957,7 @@ static ssize_t exfat_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) if (iocb->ki_flags & IOCB_DIRECT) { file_accessed(iocb->ki_filp); - ret = iomap_dio_rw(iocb, iter, &exfat_iomap_ops, NULL, 0, + ret = iomap_dio_rw(iocb, iter, exfat_iomap_next, NULL, 0, NULL, 0); } else { ret = generic_file_read_iter(iocb, iter); @@ -1016,7 +1016,7 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) file_update_time(vmf->vma->vm_file); filemap_invalidate_lock_shared(inode->i_mapping); - ret = iomap_page_mkwrite(vmf, &exfat_iomap_ops, NULL); + ret = iomap_page_mkwrite(vmf, exfat_iomap_next, NULL); filemap_invalidate_unlock_shared(inode->i_mapping); sb_end_pagefault(inode->i_sb); inode_unlock(inode); @@ -1073,12 +1073,12 @@ static loff_t exfat_file_llseek(struct file *file, loff_t offset, int whence) switch (whence) { case SEEK_HOLE: inode_lock_shared(inode); - offset = iomap_seek_hole(inode, offset, &exfat_iomap_ops); + offset = iomap_seek_hole(inode, offset, exfat_iomap_next); inode_unlock_shared(inode); break; case SEEK_DATA: inode_lock_shared(inode); - offset = iomap_seek_data(inode, offset, &exfat_iomap_ops); + offset = iomap_seek_data(inode, offset, exfat_iomap_next); inode_unlock_shared(inode); break; default: diff --git a/fs/exfat/inode.c b/fs/exfat/inode.c index 5129df1e79b4..c1d15d4fc5e7 100644 --- a/fs/exfat/inode.c +++ b/fs/exfat/inode.c @@ -249,7 +249,7 @@ static int exfat_read_folio(struct file *file, struct folio *folio) .ops = &exfat_iomap_bio_read_ops, }; - iomap_read_folio(&exfat_iomap_ops, &ctx, NULL); + iomap_read_folio(exfat_iomap_next, &ctx, NULL); return 0; } @@ -271,7 +271,7 @@ static void exfat_readahead(struct readahead_control *rac) valid_size < pos + readahead_length(rac)) return; - iomap_readahead(&exfat_iomap_ops, &ctx, NULL); + iomap_readahead(exfat_iomap_next, &ctx, NULL); } static int exfat_writepages(struct address_space *mapping, @@ -294,7 +294,7 @@ static sector_t exfat_aop_bmap(struct address_space *mapping, sector_t block) sector_t blocknr; inode_lock_shared(mapping->host); - blocknr = iomap_bmap(mapping, block, &exfat_iomap_ops); + blocknr = iomap_bmap(mapping, block, exfat_iomap_next); inode_unlock_shared(mapping->host); return blocknr; } diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c index e50ecbfbfbe0..4e6f3ed9dcc8 100644 --- a/fs/exfat/iomap.c +++ b/fs/exfat/iomap.c @@ -179,15 +179,11 @@ static int exfat_write_iomap_begin(struct inode *inode, loff_t offset, loff_t le * Zero the invalid bytes from valid_size to the end of the block * to prevent stale data from being exposed through the page cache. */ - return iomap_truncate_page(inode, valid_size, NULL, &exfat_iomap_ops, + return iomap_truncate_page(inode, valid_size, NULL, exfat_iomap_next, NULL, NULL); } -static DEFINE_IOMAP_ITER_NEXT(exfat_iomap_next, exfat_iomap_begin); - -const struct iomap_ops exfat_iomap_ops = { - .iomap_next = exfat_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT(exfat_iomap_next, exfat_iomap_begin); #ifdef CONFIG_SWAP static int exfat_swap_iomap_begin(struct inode *inode, loff_t offset, @@ -204,10 +200,6 @@ static int exfat_swap_iomap_begin(struct inode *inode, loff_t offset, static DEFINE_IOMAP_ITER_NEXT(exfat_swap_iomap_next, exfat_swap_iomap_begin); - -static const struct iomap_ops exfat_swap_iomap_ops = { - .iomap_next = exfat_swap_iomap_next, -}; #endif /* @@ -245,12 +237,8 @@ static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } -static DEFINE_IOMAP_ITER_NEXT_END(exfat_write_iomap_next, - exfat_write_iomap_begin, exfat_write_iomap_end); - -const struct iomap_ops exfat_write_iomap_ops = { - .iomap_next = exfat_write_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT_END(exfat_write_iomap_next, exfat_write_iomap_begin, + exfat_write_iomap_end); /* * exfat_writeback_range - Map folio during writeback @@ -327,7 +315,7 @@ int exfat_iomap_swap_activate(struct swap_info_struct *sis, { #ifdef CONFIG_SWAP return iomap_swapfile_activate(sis, file, span, - &exfat_swap_iomap_ops); + exfat_swap_iomap_next); #else return -EIO; #endif diff --git a/fs/exfat/iomap.h b/fs/exfat/iomap.h index fd8a913f7794..519132367b65 100644 --- a/fs/exfat/iomap.h +++ b/fs/exfat/iomap.h @@ -6,9 +6,11 @@ #ifndef _LINUX_EXFAT_IOMAP_H #define _LINUX_EXFAT_IOMAP_H +#include + extern const struct iomap_dio_ops exfat_write_dio_ops; -extern const struct iomap_ops exfat_iomap_ops; -extern const struct iomap_ops exfat_write_iomap_ops; +DECLARE_IOMAP_ITER_NEXT(exfat_iomap_next); +DECLARE_IOMAP_ITER_NEXT(exfat_write_iomap_next); extern const struct iomap_writeback_ops exfat_writeback_ops; extern const struct iomap_read_ops exfat_iomap_bio_read_ops; diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h index 7bdada93dd06..b46649c510a5 100644 --- a/fs/ext2/ext2.h +++ b/fs/ext2/ext2.h @@ -18,6 +18,7 @@ #include #include #include +#include /* XXX Here for now... not interested in restructing headers JUST now */ @@ -782,7 +783,7 @@ extern const struct file_operations ext2_file_operations; /* inode.c */ extern void ext2_set_file_ops(struct inode *inode); extern const struct address_space_operations ext2_aops; -extern const struct iomap_ops ext2_iomap_ops; +DECLARE_IOMAP_ITER_NEXT(ext2_iomap_next); /* namei.c */ extern const struct inode_operations ext2_dir_inode_operations; diff --git a/fs/ext2/file.c b/fs/ext2/file.c index 67fe423c3828..168c8246add9 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -55,7 +55,7 @@ static ssize_t ext2_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) trace_ext2_dio_read_begin(iocb, to, 0); inode_lock_shared(inode); - ret = iomap_dio_rw(iocb, to, &ext2_iomap_ops, NULL, 0, NULL, 0); + ret = iomap_dio_rw(iocb, to, ext2_iomap_next, NULL, 0, NULL, 0); inode_unlock_shared(inode); trace_ext2_dio_read_end(iocb, to, ret); @@ -119,7 +119,7 @@ static ssize_t ext2_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) (!IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(from), blocksize))) flags |= IOMAP_DIO_FORCE_WAIT; - ret = iomap_dio_rw(iocb, from, &ext2_iomap_ops, &ext2_dio_write_ops, + ret = iomap_dio_rw(iocb, from, ext2_iomap_next, &ext2_dio_write_ops, flags, NULL, 0); /* ENOTBLK is magic return value for fallback to buffered-io */ diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 12ac4cfe1500..0b8187c1fcd0 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -862,12 +862,7 @@ ext2_iomap_end(struct inode *inode, loff_t offset, loff_t length, return 0; } -static DEFINE_IOMAP_ITER_NEXT_END(ext2_iomap_next, ext2_iomap_begin, - ext2_iomap_end); - -const struct iomap_ops ext2_iomap_ops = { - .iomap_next = ext2_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT_END(ext2_iomap_next, ext2_iomap_begin, ext2_iomap_end); int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) @@ -886,7 +881,7 @@ int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, if (i_size == 0) i_size = 1; len = min_t(u64, len, i_size); - ret = iomap_fiemap(inode, fieinfo, start, len, &ext2_iomap_ops); + ret = iomap_fiemap(inode, fieinfo, start, len, ext2_iomap_next); inode_unlock(inode); return ret; diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index f38fec3167fb..bf0325ecbec7 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -4105,10 +4106,11 @@ static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) io_end->flag &= ~EXT4_IO_END_UNWRITTEN; } -extern const struct iomap_ops ext4_iomap_ops; -extern const struct iomap_ops ext4_iomap_report_ops; -extern const struct iomap_ops ext4_iomap_buffered_write_ops; -extern const struct iomap_ops ext4_iomap_buffered_da_write_ops; +DECLARE_IOMAP_ITER_NEXT(ext4_iomap_next); +DECLARE_IOMAP_ITER_NEXT(ext4_iomap_next_report); +DECLARE_IOMAP_ITER_NEXT(ext4_iomap_buffered_write_next); +DECLARE_IOMAP_ITER_NEXT(ext4_iomap_buffered_da_write_next); + extern const struct iomap_write_ops ext4_iomap_write_ops; int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index ad0d80e33f60..cd35e88bc372 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -5315,10 +5315,6 @@ static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset, static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_xattr_next, ext4_iomap_xattr_begin); -static const struct iomap_ops ext4_iomap_xattr_ops = { - .iomap_next = ext4_iomap_xattr_next, -}; - static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len) { u64 maxbytes = ext4_get_maxbytes(inode); @@ -5361,10 +5357,10 @@ int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { fieinfo->fi_flags &= ~FIEMAP_FLAG_XATTR; error = iomap_fiemap(inode, fieinfo, start, len, - &ext4_iomap_xattr_ops); + ext4_iomap_xattr_next); } else { error = iomap_fiemap(inode, fieinfo, start, len, - &ext4_iomap_report_ops); + ext4_iomap_next_report); } unlock: inode_unlock_shared(inode); diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 50d3c92709c8..ad8985c529c6 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -93,7 +93,7 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin); if (ret == -ENOTBLK) - ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); + ret = iomap_dio_rw(iocb, to, ext4_iomap_next, NULL, 0, NULL, 0); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); @@ -121,7 +121,7 @@ static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) /* Fallback to buffered IO in case we cannot support DAX */ return generic_file_read_iter(iocb, to); } - ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops); + ret = dax_iomap_rw(iocb, to, ext4_iomap_next); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); @@ -334,14 +334,14 @@ static ssize_t ext4_iomap_buffered_write(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); - const struct iomap_ops *iomap_ops; + iomap_iter_next_fn next; if (test_opt(inode->i_sb, DELALLOC) && !ext4_nonda_switch(inode->i_sb)) - iomap_ops = &ext4_iomap_buffered_da_write_ops; + next = ext4_iomap_buffered_da_write_next; else - iomap_ops = &ext4_iomap_buffered_write_ops; + next = ext4_iomap_buffered_write_next; - return iomap_file_buffered_write(iocb, from, iomap_ops, + return iomap_file_buffered_write(iocb, from, next, &ext4_iomap_write_ops, NULL); } @@ -663,7 +663,7 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; } - ret = iomap_dio_rw(iocb, from, &ext4_iomap_ops, &ext4_dio_write_ops, + ret = iomap_dio_rw(iocb, from, ext4_iomap_next, &ext4_dio_write_ops, dio_flags, NULL, 0); if (ret == -ENOTBLK) ret = 0; @@ -767,7 +767,7 @@ ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) ext4_journal_stop(handle); } - ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops); + ret = dax_iomap_rw(iocb, from, ext4_iomap_next); if (extend) { ret = ext4_handle_inode_extension(inode, offset, ret, count); @@ -855,7 +855,7 @@ retry: } else { filemap_invalidate_lock_shared(mapping); } - result = dax_iomap_fault(vmf, order, &pfn, &error, &ext4_iomap_ops); + result = dax_iomap_fault(vmf, order, &pfn, &error, ext4_iomap_next); if (write) { ext4_journal_stop(handle); @@ -1034,13 +1034,13 @@ loff_t ext4_llseek(struct file *file, loff_t offset, int whence) case SEEK_HOLE: inode_lock_shared(inode); offset = iomap_seek_hole(inode, offset, - &ext4_iomap_report_ops); + ext4_iomap_next_report); inode_unlock_shared(inode); break; case SEEK_DATA: inode_lock_shared(inode); offset = iomap_seek_data(inode, offset, - &ext4_iomap_report_ops); + ext4_iomap_next_report); inode_unlock_shared(inode); break; } diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 10cd85d4c721..134e6d8538dd 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3519,7 +3519,7 @@ static sector_t ext4_bmap(struct address_space *mapping, sector_t block) filemap_write_and_wait(mapping); } - ret = iomap_bmap(mapping, block, &ext4_iomap_ops); + ret = iomap_bmap(mapping, block, ext4_iomap_next); out: inode_unlock_shared(inode); @@ -3994,11 +3994,7 @@ out: return 0; } -static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin); - -const struct iomap_ops ext4_iomap_ops = { - .iomap_next = ext4_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin); static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, @@ -4051,11 +4047,7 @@ set_iomap: return 0; } -static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report); - -const struct iomap_ops ext4_iomap_report_ops = { - .iomap_next = ext4_iomap_next_report, -}; +DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report); static int ext4_iomap_map_blocks(struct inode *inode, loff_t offset, loff_t length, struct ext4_map_blocks *map, @@ -4260,45 +4252,29 @@ again: static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_zero_next, ext4_iomap_zero_begin); -static const struct iomap_ops ext4_iomap_zero_ops = { - .iomap_next = ext4_iomap_zero_next, -}; - /* * Since we always allocate unwritten extents, there is no need for * iomap_end to clean up allocated blocks on a short write. */ -static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_buffered_write_next, - ext4_iomap_buffered_write_begin); +DEFINE_IOMAP_ITER_NEXT(ext4_iomap_buffered_write_next, + ext4_iomap_buffered_write_begin); -const struct iomap_ops ext4_iomap_buffered_write_ops = { - .iomap_next = ext4_iomap_buffered_write_next, -}; - -static DEFINE_IOMAP_ITER_NEXT_END(ext4_iomap_buffered_da_write_next, - ext4_iomap_buffered_da_write_begin, - ext4_iomap_buffered_da_write_end); - -const struct iomap_ops ext4_iomap_buffered_da_write_ops = { - .iomap_next = ext4_iomap_buffered_da_write_next, -}; +DEFINE_IOMAP_ITER_NEXT_END(ext4_iomap_buffered_da_write_next, + ext4_iomap_buffered_da_write_begin, + ext4_iomap_buffered_da_write_end); static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_buffered_read_next, ext4_iomap_buffered_read_begin); -static const struct iomap_ops ext4_iomap_buffered_read_ops = { - .iomap_next = ext4_iomap_buffered_read_next, -}; - static int ext4_iomap_read_folio(struct file *file, struct folio *folio) { - iomap_bio_read_folio(folio, &ext4_iomap_buffered_read_ops); + iomap_bio_read_folio(folio, ext4_iomap_buffered_read_next); return 0; } static void ext4_iomap_readahead(struct readahead_control *rac) { - iomap_bio_readahead(rac, &ext4_iomap_buffered_read_ops); + iomap_bio_readahead(rac, ext4_iomap_buffered_read_next); } @@ -4561,7 +4537,7 @@ static int ext4_iomap_swap_activate(struct swap_info_struct *sis, struct file *file, sector_t *span) { return iomap_swapfile_activate(sis, file, span, - &ext4_iomap_report_ops); + ext4_iomap_next_report); } static const struct address_space_operations ext4_aops = { @@ -4827,7 +4803,7 @@ static int ext4_block_iomap_zero_range(struct inode *inode, loff_t from, !rwsem_is_locked(&inode->i_mapping->invalidate_lock)); ret = iomap_zero_range(inode, from, length, did_zero, - &ext4_iomap_zero_ops, &ext4_iomap_write_ops, + ext4_iomap_zero_next, &ext4_iomap_write_ops, NULL); if (ret) return ret; @@ -4866,7 +4842,7 @@ static int ext4_block_zero_range(struct inode *inode, if (IS_DAX(inode)) { return dax_zero_range(inode, from, length, did_zero, - &ext4_iomap_ops); + ext4_iomap_next); } else if (ext4_should_journal_data(inode)) { return ext4_block_journalled_zero_range(inode, from, length, did_zero); @@ -7829,18 +7805,18 @@ out_error: static vm_fault_t ext4_iomap_page_mkwrite(struct vm_fault *vmf) { struct inode *inode = file_inode(vmf->vma->vm_file); - const struct iomap_ops *iomap_ops; + iomap_iter_next_fn next; /* * ext4_nonda_switch() could writeback this folio, so have to * call it before lock folio. */ if (test_opt(inode->i_sb, DELALLOC) && !ext4_nonda_switch(inode->i_sb)) - iomap_ops = &ext4_iomap_buffered_da_write_ops; + next = ext4_iomap_buffered_da_write_next; else - iomap_ops = &ext4_iomap_buffered_write_ops; + next = ext4_iomap_buffered_write_next; - return iomap_page_mkwrite(vmf, iomap_ops, NULL); + return iomap_page_mkwrite(vmf, next, NULL); } vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 28fd8afff5d4..7c5aa95b7da8 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -4853,8 +4853,4 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } -static DEFINE_IOMAP_ITER_NEXT(f2fs_iomap_next, f2fs_iomap_begin); - -const struct iomap_ops f2fs_iomap_ops = { - .iomap_next = f2fs_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT(f2fs_iomap_next, f2fs_iomap_begin); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index f7423b3f6e4a..6941891d7cae 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -4397,7 +4398,7 @@ int f2fs_init_post_read_processing(void); void f2fs_destroy_post_read_processing(void); int f2fs_init_wq(struct f2fs_sb_info *sbi); void f2fs_destroy_wq(struct f2fs_sb_info *sbi); -extern const struct iomap_ops f2fs_iomap_ops; +DECLARE_IOMAP_ITER_NEXT(f2fs_iomap_next); /* * gc.c diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index e2c0f5083313..cf095978fad4 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -5245,7 +5245,7 @@ static ssize_t f2fs_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) * F2FS_DIO_READ counter will be decremented correctly in all cases. */ inc_cache_count(sbi, F2FS_DIO_READ); - dio = __iomap_dio_rw(iocb, to, &f2fs_iomap_ops, + dio = __iomap_dio_rw(iocb, to, f2fs_iomap_next, &f2fs_iomap_dio_read_ops, 0, NULL, 0); if (IS_ERR_OR_NULL(dio)) { ret = PTR_ERR_OR_ZERO(dio); @@ -5584,7 +5584,7 @@ static ssize_t f2fs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from, dio_flags = 0; if (pos + count > inode->i_size) dio_flags |= IOMAP_DIO_FORCE_WAIT; - dio = __iomap_dio_rw(iocb, from, &f2fs_iomap_ops, + dio = __iomap_dio_rw(iocb, from, f2fs_iomap_next, &f2fs_iomap_dio_write_ops, dio_flags, iocb, 0); if (IS_ERR_OR_NULL(dio)) { ret = PTR_ERR_OR_ZERO(dio); diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c index cae217ff89ba..e20a5ae6fecc 100644 --- a/fs/fuse/dax.c +++ b/fs/fuse/dax.c @@ -656,10 +656,6 @@ static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length, static DEFINE_IOMAP_ITER_NEXT_END(fuse_iomap_next, fuse_iomap_begin, fuse_iomap_end); -static const struct iomap_ops fuse_iomap_ops = { - .iomap_next = fuse_iomap_next, -}; - static void fuse_wait_dax_page(struct inode *inode) { filemap_invalidate_unlock(inode->i_mapping); @@ -687,7 +683,7 @@ ssize_t fuse_vdax_read_iter(struct kiocb *iocb, struct iov_iter *to) inode_lock_shared(inode); } - ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops); + ret = dax_iomap_rw(iocb, to, fuse_iomap_next); inode_unlock_shared(inode); /* TODO file_accessed(iocb->f_filp) */ @@ -742,7 +738,7 @@ ssize_t fuse_vdax_write_iter(struct kiocb *iocb, struct iov_iter *from) if (file_extending_write(iocb, from)) ret = fuse_dax_direct_write(iocb, from); else - ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops); + ret = dax_iomap_rw(iocb, from, fuse_iomap_next); out: inode_unlock(inode); @@ -777,7 +773,7 @@ retry: * to populate page cache or access memory we are trying to free. */ filemap_invalidate_lock_shared(inode->i_mapping); - ret = dax_iomap_fault(vmf, order, &pfn, &error, &fuse_iomap_ops); + ret = dax_iomap_fault(vmf, order, &pfn, &error, fuse_iomap_next); if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) { error = 0; retry = true; diff --git a/fs/fuse/ext_map.c b/fs/fuse/ext_map.c index edb19498d496..ce97377391ad 100644 --- a/fs/fuse/ext_map.c +++ b/fs/fuse/ext_map.c @@ -213,10 +213,6 @@ static int fuse_ext_map_iomap_begin(struct inode *inode, loff_t offset, loff_t l static DEFINE_IOMAP_ITER_NEXT(fuse_ext_map_iomap_next, fuse_ext_map_iomap_begin); -static const struct iomap_ops fuse_ext_map_iomap_ops = { - .iomap_next = fuse_ext_map_iomap_next, -}; - static vm_fault_t fuse_ext_map_huge_fault(struct vm_fault *vmf, unsigned int order) { struct inode *inode = file_inode(vmf->vma->vm_file); @@ -237,7 +233,7 @@ static vm_fault_t fuse_ext_map_huge_fault(struct vm_fault *vmf, unsigned int ord filemap_invalidate_lock_shared(inode->i_mapping); - ret = dax_iomap_fault(vmf, order, &pfn, NULL, &fuse_ext_map_iomap_ops); + ret = dax_iomap_fault(vmf, order, &pfn, NULL, fuse_ext_map_iomap_next); if (ret & VM_FAULT_NEEDDSYNC) ret = dax_finish_sync_fault(vmf, order, pfn); @@ -286,7 +282,7 @@ ssize_t fuse_ext_map_read_iter(struct kiocb *iocb, struct iov_iter *to) if (!IS_ENABLED(CONFIG_FS_DAX)) return -EIO; - res = dax_iomap_rw(iocb, to, &fuse_ext_map_iomap_ops); + res = dax_iomap_rw(iocb, to, fuse_ext_map_iomap_next); file_accessed(iocb->ki_filp); return res; @@ -312,7 +308,7 @@ ssize_t fuse_ext_map_write_iter(struct kiocb *iocb, struct iov_iter *from) if (!IS_ENABLED(CONFIG_FS_DAX)) return -EIO; - return dax_iomap_rw(iocb, from, &fuse_ext_map_iomap_ops); + return dax_iomap_rw(iocb, from, fuse_ext_map_iomap_next); } int fuse_ext_map_mmap(struct file *file, struct vm_area_struct *vma) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index d5e1cb53d86a..6b279ff5ec05 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -923,10 +923,6 @@ static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, static DEFINE_IOMAP_ITER_NEXT(fuse_iomap_next, fuse_iomap_begin); -static const struct iomap_ops fuse_iomap_ops = { - .iomap_next = fuse_iomap_next, -}; - struct fuse_fill_read_data { struct file *file; @@ -1036,7 +1032,7 @@ static int fuse_read_folio(struct file *file, struct folio *folio) return -EIO; } - iomap_read_folio(&fuse_iomap_ops, &ctx, NULL); + iomap_read_folio(fuse_iomap_next, &ctx, NULL); fuse_invalidate_atime(inode); return 0; } @@ -1146,7 +1142,7 @@ static void fuse_readahead(struct readahead_control *rac) if (fuse_is_bad(inode)) return; - iomap_readahead(&fuse_iomap_ops, &ctx, NULL); + iomap_readahead(fuse_iomap_next, &ctx, NULL); if (data.ia) fuse_send_readpages(data.ia, data.file, data.nr_bytes, fc->async_read); @@ -1570,7 +1566,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * and granular dirty tracking for large folios. */ written = iomap_file_buffered_write(iocb, from, - &fuse_iomap_ops, + fuse_iomap_next, &fuse_iomap_write_ops, file); } else { diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index ba8d79c4bcac..df069b070bf3 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -1053,7 +1053,8 @@ static void virtio_fs_cleanup_vqs(struct virtio_device *vdev) } /* Map a window offset to a page frame number. The window offset will have - * been produced by .iomap_next(), which maps a file offset to a window offset. + * been produced by fuse_iomap_next(), which maps a file offset to a window + * offset. */ static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c index ee11d494e47c..d31615f49754 100644 --- a/fs/gfs2/aops.c +++ b/fs/gfs2/aops.c @@ -425,7 +425,7 @@ static int gfs2_read_folio(struct file *file, struct folio *folio) if (!gfs2_is_jdata(ip) || (i_blocksize(inode) == PAGE_SIZE && !folio_buffers(folio))) { - iomap_bio_read_folio(folio, &gfs2_iomap_ops); + iomap_bio_read_folio(folio, gfs2_iomap_next); } else if (gfs2_is_stuffed(ip)) { error = stuffed_read_folio(ip, folio); } else { @@ -500,7 +500,7 @@ static void gfs2_readahead(struct readahead_control *rac) else if (gfs2_is_jdata(ip)) mpage_readahead(rac, gfs2_block_map); else - iomap_bio_readahead(rac, &gfs2_iomap_ops); + iomap_bio_readahead(rac, gfs2_iomap_next); } /** @@ -572,7 +572,7 @@ static sector_t gfs2_bmap(struct address_space *mapping, sector_t lblock) return 0; if (!gfs2_is_stuffed(ip)) - dblock = iomap_bmap(mapping, lblock, &gfs2_iomap_ops); + dblock = iomap_bmap(mapping, lblock, gfs2_iomap_next); gfs2_glock_dq_uninit(&i_gh); diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 264170f91c5d..09884f698d91 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1205,12 +1205,7 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, return 0; } -static DEFINE_IOMAP_ITER_NEXT_END(gfs2_iomap_next, gfs2_iomap_begin, - gfs2_iomap_end); - -const struct iomap_ops gfs2_iomap_ops = { - .iomap_next = gfs2_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT_END(gfs2_iomap_next, gfs2_iomap_begin, gfs2_iomap_end); /** * gfs2_block_map - Map one or more blocks of an inode to a disk block @@ -1325,7 +1320,7 @@ static int gfs2_block_zero_range(struct inode *inode, loff_t from, loff_t length if (from >= inode->i_size) return 0; length = min(length, inode->i_size - from); - return iomap_zero_range(inode, from, length, NULL, &gfs2_iomap_ops, + return iomap_zero_range(inode, from, length, NULL, gfs2_iomap_next, &gfs2_iomap_write_ops, NULL); } diff --git a/fs/gfs2/bmap.h b/fs/gfs2/bmap.h index e3d6efdfd890..cceb47950b7b 100644 --- a/fs/gfs2/bmap.h +++ b/fs/gfs2/bmap.h @@ -43,7 +43,7 @@ static inline void gfs2_write_calc_reserv(const struct gfs2_inode *ip, } } -extern const struct iomap_ops gfs2_iomap_ops; +DECLARE_IOMAP_ITER_NEXT(gfs2_iomap_next); extern const struct iomap_write_ops gfs2_iomap_write_ops; extern const struct iomap_writeback_ops gfs2_writeback_ops; diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index 1efd0679badd..cea24d5de91c 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -851,7 +851,7 @@ retry: goto out_uninit; pagefault_disable(); to->nofault = true; - ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL, + ret = iomap_dio_rw(iocb, to, gfs2_iomap_next, NULL, IOMAP_DIO_PARTIAL, NULL, read); to->nofault = false; pagefault_enable(); @@ -917,7 +917,7 @@ retry: goto out_unlock; from->nofault = true; - ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL, + ret = iomap_dio_rw(iocb, from, gfs2_iomap_next, NULL, IOMAP_DIO_PARTIAL, NULL, written); from->nofault = false; if (ret <= 0) { @@ -1069,7 +1069,7 @@ retry: goto out_unlock; pagefault_disable(); - ret = iomap_file_buffered_write(iocb, from, &gfs2_iomap_ops, + ret = iomap_file_buffered_write(iocb, from, gfs2_iomap_next, &gfs2_iomap_write_ops, NULL); pagefault_enable(); if (ret > 0) diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 2c2dc459c037..75fb1a0c9142 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -2229,7 +2229,7 @@ retry: goto out; pagefault_disable(); - ret = iomap_fiemap(inode, fieinfo, start, len, &gfs2_iomap_ops); + ret = iomap_fiemap(inode, fieinfo, start, len, gfs2_iomap_next); pagefault_enable(); gfs2_glock_dq_uninit(&gh); @@ -2254,7 +2254,7 @@ loff_t gfs2_seek_data(struct file *file, loff_t offset) inode_lock_shared(inode); ret = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &gh); if (!ret) - ret = iomap_seek_data(inode, offset, &gfs2_iomap_ops); + ret = iomap_seek_data(inode, offset, gfs2_iomap_next); gfs2_glock_dq_uninit(&gh); inode_unlock_shared(inode); @@ -2273,7 +2273,7 @@ loff_t gfs2_seek_hole(struct file *file, loff_t offset) inode_lock_shared(inode); ret = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &gh); if (!ret) - ret = iomap_seek_hole(inode, offset, &gfs2_iomap_ops); + ret = iomap_seek_hole(inode, offset, gfs2_iomap_next); gfs2_glock_dq_uninit(&gh); inode_unlock_shared(inode); diff --git a/fs/hpfs/file.c b/fs/hpfs/file.c index 6a629ab956fc..ef7eb601be09 100644 --- a/fs/hpfs/file.c +++ b/fs/hpfs/file.c @@ -158,10 +158,6 @@ static int hpfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, static DEFINE_IOMAP_ITER_NEXT(hpfs_iomap_next, hpfs_iomap_begin); -static const struct iomap_ops hpfs_iomap_ops = { - .iomap_next = hpfs_iomap_next, -}; - static int hpfs_read_folio(struct file *file, struct folio *folio) { return mpage_read_folio(folio, hpfs_get_block); @@ -238,7 +234,7 @@ static int hpfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, inode_lock(inode); len = min_t(u64, len, i_size_read(inode)); - ret = iomap_fiemap(inode, fieinfo, start, len, &hpfs_iomap_ops); + ret = iomap_fiemap(inode, fieinfo, start, len, hpfs_iomap_next); inode_unlock(inode); return ret; diff --git a/fs/internal.h b/fs/internal.h index 967c8556c17f..3f2fbac40f8e 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -8,7 +8,6 @@ struct super_block; struct file_system_type; struct iomap; -struct iomap_ops; struct linux_binprm; struct path; struct mount; diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index d6e2f1ded5ca..6d843b168d5f 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -648,8 +648,8 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, return 0; } -void iomap_read_folio(const struct iomap_ops *ops, - struct iomap_read_folio_ctx *ctx, void *private) +void iomap_read_folio(iomap_iter_next_fn next, struct iomap_read_folio_ctx *ctx, + void *private) { struct folio *folio = ctx->cur_folio; struct iomap_iter iter = { @@ -672,7 +672,7 @@ void iomap_read_folio(const struct iomap_ops *ops, fsverity_readahead(ctx->vi, folio->index, folio_nr_pages(folio)); - while ((ret = iomap_iter(&iter, ops)) > 0) { + while ((ret = iomap_iter(&iter, next)) > 0) { iter.status = iomap_read_folio_iter(&iter, ctx, &bytes_submitted); if (ctx->read_ctx && ctx->ops->submit_read) @@ -711,22 +711,22 @@ static int iomap_readahead_iter(struct iomap_iter *iter, /** * iomap_readahead - Attempt to read pages from a file. - * @ops: The operations vector for the filesystem. + * @next: The iomap iteration function for the filesystem. * @ctx: The ctx used for issuing readahead. * @private: The filesystem-specific information for issuing iomap_iter. * * This function is for filesystems to call to implement their readahead * address_space operation. * - * Context: The @ops callbacks may submit I/O (eg to read the addresses of + * Context: The @next callback may submit I/O (eg to read the addresses of * blocks from disc), and may wait for it. The caller may be trying to * access a different page, and so sleeping excessively should be avoided. * It may allocate memory, but should avoid costly allocations. This * function is called with memalloc_nofs set, so allocations will not cause * the filesystem to be reentered. */ -void iomap_readahead(const struct iomap_ops *ops, - struct iomap_read_folio_ctx *ctx, void *private) +void iomap_readahead(iomap_iter_next_fn next, struct iomap_read_folio_ctx *ctx, + void *private) { struct readahead_control *rac = ctx->rac; struct iomap_iter iter = { @@ -748,7 +748,7 @@ void iomap_readahead(const struct iomap_ops *ops, fsverity_readahead(ctx->vi, readahead_index(rac), readahead_count(rac)); - while (iomap_iter(&iter, ops) > 0) { + while (iomap_iter(&iter, next) > 0) { iter.status = iomap_readahead_iter(&iter, ctx, &cur_bytes_submitted); if (ctx->read_ctx && ctx->ops->submit_read) @@ -1293,7 +1293,7 @@ retry: ssize_t iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i, - const struct iomap_ops *ops, + iomap_iter_next_fn next, const struct iomap_write_ops *write_ops, void *private) { struct iomap_iter iter = { @@ -1314,7 +1314,7 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i, if (iocb->ki_flags & IOCB_DONTCACHE) iter.flags |= IOMAP_DONTCACHE; - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = iomap_write_iter(&iter, i, write_ops); if (unlikely(iter.pos == iocb->ki_pos)) @@ -1326,7 +1326,7 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i, EXPORT_SYMBOL_GPL(iomap_file_buffered_write); int iomap_fsverity_write(struct file *file, loff_t pos, size_t length, - const void *buf, const struct iomap_ops *ops, + const void *buf, iomap_iter_next_fn next, const struct iomap_write_ops *write_ops) { int ret; @@ -1343,7 +1343,7 @@ int iomap_fsverity_write(struct file *file, loff_t pos, size_t length, iov_iter_kvec(&iiter, WRITE, &kvec, 1, length); - ret = iomap_file_buffered_write(&iocb, &iiter, ops, write_ops, NULL); + ret = iomap_file_buffered_write(&iocb, &iiter, next, write_ops, NULL); if (ret < 0) return ret; return ret == length ? 0 : -EIO; @@ -1615,7 +1615,7 @@ static int iomap_unshare_iter(struct iomap_iter *iter, int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, - const struct iomap_ops *ops, + iomap_iter_next_fn next, const struct iomap_write_ops *write_ops) { struct iomap_iter iter = { @@ -1630,7 +1630,7 @@ iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, return 0; iter.len = min(len, size - pos); - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = iomap_unshare_iter(&iter, write_ops); return ret; } @@ -1743,7 +1743,7 @@ EXPORT_SYMBOL_GPL(iomap_fill_dirty_folios); int iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, - const struct iomap_ops *ops, + iomap_iter_next_fn next, const struct iomap_write_ops *write_ops, void *private) { struct folio_batch fbatch; @@ -1768,7 +1768,7 @@ iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, */ range_dirty = filemap_range_needs_writeback(mapping, iter.pos, iter.pos + iter.len - 1); - while ((ret = iomap_iter(&iter, ops)) > 0) { + while ((ret = iomap_iter(&iter, next)) > 0) { const struct iomap *srcmap = iomap_iter_srcmap(&iter); if (!(iter.iomap.flags & IOMAP_F_FOLIO_BATCH) && @@ -1794,7 +1794,7 @@ EXPORT_SYMBOL_GPL(iomap_zero_range); int iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, - const struct iomap_ops *ops, + iomap_iter_next_fn next, const struct iomap_write_ops *write_ops, void *private) { unsigned int blocksize = i_blocksize(inode); @@ -1803,7 +1803,7 @@ iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, /* Block boundary? Nothing to do */ if (!off) return 0; - return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops, + return iomap_zero_range(inode, pos, blocksize - off, did_zero, next, write_ops, private); } EXPORT_SYMBOL_GPL(iomap_truncate_page); @@ -1828,7 +1828,7 @@ static int iomap_folio_mkwrite_iter(struct iomap_iter *iter, return iomap_iter_advance(iter, length); } -vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops, +vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, iomap_iter_next_fn next, void *private) { struct iomap_iter iter = { @@ -1845,7 +1845,7 @@ vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops, goto out_unlock; iter.pos = folio_pos(folio); iter.len = ret; - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = iomap_folio_mkwrite_iter(&iter, folio); if (ret < 0) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index ad041ac3252a..f00b0841fdd0 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -695,7 +695,7 @@ static int iomap_dio_iter(struct iomap_iter *iter, struct iomap_dio *dio) */ struct iomap_dio * __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, const struct iomap_dio_ops *dops, + iomap_iter_next_fn next, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before) { struct inode *inode = file_inode(iocb->ki_filp); @@ -819,7 +819,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, inode_dio_begin(inode); blk_start_plug(&plug); - while ((ret = iomap_iter(&iomi, ops)) > 0) { + while ((ret = iomap_iter(&iomi, next)) > 0) { iomi.status = iomap_dio_iter(&iomi, dio); /* @@ -909,12 +909,12 @@ EXPORT_SYMBOL_GPL(__iomap_dio_rw); ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, const struct iomap_dio_ops *dops, + iomap_iter_next_fn next, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before) { struct iomap_dio *dio; - dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, + dio = __iomap_dio_rw(iocb, iter, next, dops, dio_flags, private, done_before); if (IS_ERR_OR_NULL(dio)) return PTR_ERR_OR_ZERO(dio); diff --git a/fs/iomap/fiemap.c b/fs/iomap/fiemap.c index 54b824b7edb5..a9e33c7af910 100644 --- a/fs/iomap/fiemap.c +++ b/fs/iomap/fiemap.c @@ -56,7 +56,7 @@ advance: } int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi, - u64 start, u64 len, const struct iomap_ops *ops) + u64 start, u64 len, iomap_iter_next_fn next) { struct iomap_iter iter = { .inode = inode, @@ -73,7 +73,7 @@ int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi, if (ret) return ret; - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = iomap_fiemap_iter(&iter, fi, &prev); /* inode with no (attribute) mapping will give ENOENT */ @@ -92,7 +92,7 @@ EXPORT_SYMBOL_GPL(iomap_fiemap); /* legacy ->bmap interface. 0 is the error return (!) */ sector_t iomap_bmap(struct address_space *mapping, sector_t bno, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct iomap_iter iter = { .inode = mapping->host, @@ -107,7 +107,7 @@ iomap_bmap(struct address_space *mapping, sector_t bno, return 0; bno = 0; - while ((ret = iomap_iter(&iter, ops)) > 0) { + while ((ret = iomap_iter(&iter, next)) > 0) { if (iter.iomap.type == IOMAP_MAPPED) bno = iomap_sector(&iter.iomap, iter.pos) >> blkshift; /* leave iter.status unset to abort loop */ diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index 8b6da9750a93..771b60fd1022 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -101,7 +101,7 @@ EXPORT_SYMBOL_GPL(iomap_iter_continue); /** * iomap_iter - iterate over ranges in a file * @iter: iteration structure - * @ops: iomap ops provided by the filesystem + * @next: the iomap iteration function for the filesystem * * Iterate over filesystem-provided space mappings for the provided file range. * @@ -113,14 +113,13 @@ EXPORT_SYMBOL_GPL(iomap_iter_continue); * of the loop body: leave @iter.status unchanged, or set it to a negative * errno. */ -int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) +int iomap_iter(struct iomap_iter *iter, iomap_iter_next_fn next) { int ret; - trace_iomap_iter(iter, ops, _RET_IP_); - - ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap); + trace_iomap_iter(iter, next, _RET_IP_); + ret = next(iter, &iter->iomap, &iter->srcmap); iter->status = 0; if (ret > 0) iomap_iter_done(iter); diff --git a/fs/iomap/seek.c b/fs/iomap/seek.c index 6cbc587c93da..6fd3173d25f5 100644 --- a/fs/iomap/seek.c +++ b/fs/iomap/seek.c @@ -27,7 +27,7 @@ static int iomap_seek_hole_iter(struct iomap_iter *iter, } loff_t -iomap_seek_hole(struct inode *inode, loff_t pos, const struct iomap_ops *ops) +iomap_seek_hole(struct inode *inode, loff_t pos, iomap_iter_next_fn next) { loff_t size = i_size_read(inode); struct iomap_iter iter = { @@ -42,7 +42,7 @@ iomap_seek_hole(struct inode *inode, loff_t pos, const struct iomap_ops *ops) return -ENXIO; iter.len = size - pos; - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = iomap_seek_hole_iter(&iter, &pos); if (ret < 0) return ret; @@ -73,7 +73,7 @@ static int iomap_seek_data_iter(struct iomap_iter *iter, } loff_t -iomap_seek_data(struct inode *inode, loff_t pos, const struct iomap_ops *ops) +iomap_seek_data(struct inode *inode, loff_t pos, iomap_iter_next_fn next) { loff_t size = i_size_read(inode); struct iomap_iter iter = { @@ -88,7 +88,7 @@ iomap_seek_data(struct inode *inode, loff_t pos, const struct iomap_ops *ops) return -ENXIO; iter.len = size - pos; - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = iomap_seek_data_iter(&iter, &pos); if (ret < 0) return ret; diff --git a/fs/iomap/swapfile.c b/fs/iomap/swapfile.c index 0db77c449467..c59f67a7de09 100644 --- a/fs/iomap/swapfile.c +++ b/fs/iomap/swapfile.c @@ -139,7 +139,7 @@ static int iomap_swapfile_iter(struct iomap_iter *iter, */ int iomap_swapfile_activate(struct swap_info_struct *sis, struct file *swap_file, sector_t *pagespan, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct inode *inode = swap_file->f_mapping->host; struct iomap_iter iter = { @@ -163,7 +163,7 @@ int iomap_swapfile_activate(struct swap_info_struct *sis, if (ret) return ret; - while ((ret = iomap_iter(&iter, ops)) > 0) + while ((ret = iomap_iter(&iter, next)) > 0) iter.status = iomap_swapfile_iter(&iter, &iter.iomap, &isi); if (ret < 0) return ret; diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h index e1ea8392cf47..d36a77b854aa 100644 --- a/fs/iomap/trace.h +++ b/fs/iomap/trace.h @@ -218,9 +218,9 @@ TRACE_EVENT(iomap_add_to_ioend, ); TRACE_EVENT(iomap_iter, - TP_PROTO(struct iomap_iter *iter, const void *ops, + TP_PROTO(struct iomap_iter *iter, const void *next, unsigned long caller), - TP_ARGS(iter, ops, caller), + TP_ARGS(iter, next, caller), TP_STRUCT__entry( __field(dev_t, dev) __field(u64, ino) @@ -228,7 +228,7 @@ TRACE_EVENT(iomap_iter, __field(u64, length) __field(int, status) __field(unsigned int, flags) - __field(const void *, ops) + __field(const void *, next) __field(unsigned long, caller) ), TP_fast_assign( @@ -238,10 +238,10 @@ TRACE_EVENT(iomap_iter, __entry->length = iomap_length(iter); __entry->status = iter->status; __entry->flags = iter->flags; - __entry->ops = ops; + __entry->next = next; __entry->caller = caller; ), - TP_printk("dev %d:%d ino 0x%llx pos 0x%llx length 0x%llx status %d flags %s (0x%x) ops %ps caller %pS", + TP_printk("dev %d:%d ino 0x%llx pos 0x%llx length 0x%llx status %d flags %s (0x%x) next %ps caller %pS", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->pos, @@ -249,7 +249,7 @@ TRACE_EVENT(iomap_iter, __entry->status, __print_flags(__entry->flags, "|", IOMAP_FLAGS_STRINGS), __entry->flags, - __entry->ops, + __entry->next, (void *)__entry->caller) ); diff --git a/fs/nilfs2/file.c b/fs/nilfs2/file.c index 30dd8e5ebe5b..d1bf52b00349 100644 --- a/fs/nilfs2/file.c +++ b/fs/nilfs2/file.c @@ -149,7 +149,7 @@ static ssize_t nilfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) if (iocb->ki_flags & IOCB_DIRECT) { inode_lock_shared(inode); - ret = iomap_dio_rw(iocb, to, &nilfs_iomap_ops, + ret = iomap_dio_rw(iocb, to, nilfs_iomap_next, NULL, 0, NULL, 0); inode_unlock_shared(inode); } else diff --git a/fs/nilfs2/iomap.c b/fs/nilfs2/iomap.c index 359786c97f2b..c00189ade861 100644 --- a/fs/nilfs2/iomap.c +++ b/fs/nilfs2/iomap.c @@ -67,8 +67,4 @@ static int nilfs_iomap_begin(struct inode *inode, loff_t offset, return 0; } -static DEFINE_IOMAP_ITER_NEXT(nilfs_iomap_next, nilfs_iomap_begin); - -const struct iomap_ops nilfs_iomap_ops = { - .iomap_next = nilfs_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT(nilfs_iomap_next, nilfs_iomap_begin); diff --git a/fs/nilfs2/iomap.h b/fs/nilfs2/iomap.h index adef3e22346d..5f9c05bd1230 100644 --- a/fs/nilfs2/iomap.h +++ b/fs/nilfs2/iomap.h @@ -8,6 +8,8 @@ #ifndef _NILFS_IOMAP_H #define _NILFS_IOMAP_H -extern const struct iomap_ops nilfs_iomap_ops; +#include + +DECLARE_IOMAP_ITER_NEXT(nilfs_iomap_next); #endif /* _NILFS_IOMAP_H */ diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c index ad781674fef1..cb3d5443ee1d 100644 --- a/fs/ntfs/aops.c +++ b/fs/ntfs/aops.c @@ -103,7 +103,7 @@ static int ntfs_read_folio(struct file *file, struct folio *folio) return ntfs_read_compressed_block(folio); } - iomap_read_folio(&ntfs_read_iomap_ops, &ctx, NULL); + iomap_read_folio(ntfs_read_iomap_next, &ctx, NULL); return 0; } @@ -247,7 +247,7 @@ static void ntfs_readahead(struct readahead_control *rac) if (!NInoNonResident(ni) || NInoEncrypted(ni) || NInoCompressed(ni) || NInoWofCompressed(ni)) return; - iomap_readahead(&ntfs_read_iomap_ops, &ctx, NULL); + iomap_readahead(ntfs_read_iomap_next, &ctx, NULL); } static int ntfs_writepages(struct address_space *mapping, @@ -301,7 +301,7 @@ static int ntfs_swap_activate(struct swap_info_struct *sis, return -EOPNOTSUPP; return iomap_swapfile_activate(sis, swap_file, span, - &ntfs_read_iomap_ops); + ntfs_read_iomap_next); } const struct address_space_operations ntfs_aops = { diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index b2f5882d9990..1861d0f07fef 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -686,12 +686,12 @@ loff_t ntfs_file_llseek(struct file *file, loff_t offset, int whence) switch (whence) { case SEEK_HOLE: inode_lock_shared(inode); - offset = iomap_seek_hole(inode, offset, &ntfs_seek_iomap_ops); + offset = iomap_seek_hole(inode, offset, ntfs_seek_iomap_next); inode_unlock_shared(inode); break; case SEEK_DATA: inode_lock_shared(inode); - offset = iomap_seek_data(inode, offset, &ntfs_seek_iomap_ops); + offset = iomap_seek_data(inode, offset, ntfs_seek_iomap_next); inode_unlock_shared(inode); break; default: @@ -728,7 +728,7 @@ ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) } file_accessed(iocb->ki_filp); - ret = iomap_dio_rw(iocb, to, &ntfs_read_iomap_ops, NULL, 0, + ret = iomap_dio_rw(iocb, to, ntfs_read_iomap_next, NULL, 0, NULL, 0); } else { ret = generic_file_read_iter(iocb, to); @@ -770,7 +770,7 @@ static ssize_t ntfs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) { ssize_t ret; - ret = iomap_dio_rw(iocb, from, &ntfs_dio_iomap_ops, + ret = iomap_dio_rw(iocb, from, ntfs_dio_iomap_next, &ntfs_write_dio_ops, 0, NULL, 0); if (ret == -ENOTBLK) ret = 0; @@ -798,7 +798,7 @@ static ssize_t ntfs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) inode_dio_wait(file_inode(iocb->ki_filp)); written = iomap_file_buffered_write(iocb, from, - &ntfs_write_iomap_ops, &ntfs_iomap_folio_ops, + ntfs_write_iomap_next, &ntfs_iomap_folio_ops, NULL); if (written < 0) { ret = written; @@ -930,7 +930,7 @@ ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (NInoNonResident(ni) && iocb->ki_flags & IOCB_DIRECT) ret = ntfs_dio_write_iter(iocb, from); else - ret = iomap_file_buffered_write(iocb, from, &ntfs_write_iomap_ops, + ret = iomap_file_buffered_write(iocb, from, ntfs_write_iomap_next, &ntfs_iomap_folio_ops, NULL); out: if (ret < 0 && ret != -EIOCBQUEUED) { @@ -1017,7 +1017,7 @@ static vm_fault_t ntfs_filemap_page_mkwrite(struct vm_fault *vmf) * exclusively while invalidating pagecache and changing extents. */ filemap_invalidate_lock_shared(mapping); - ret = iomap_page_mkwrite(vmf, &ntfs_page_mkwrite_iomap_ops, NULL); + ret = iomap_page_mkwrite(vmf, ntfs_page_mkwrite_iomap_next, NULL); filemap_invalidate_unlock_shared(mapping); sb_end_pagefault(inode->i_sb); return ret; @@ -1072,7 +1072,7 @@ int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, if (NInoWofCompressed(NTFS_I(inode))) return -EOPNOTSUPP; - return iomap_fiemap(inode, fieinfo, start, len, &ntfs_read_iomap_ops); + return iomap_fiemap(inode, fieinfo, start, len, ntfs_read_iomap_next); } static const char *ntfs_get_link(struct dentry *dentry, struct inode *inode, @@ -1322,7 +1322,7 @@ static int ntfs_punch_hole(struct ntfs_inode *ni, int mode, loff_t offset, ntfs_cluster_to_bytes(vol, start_vcn + 1), end_offset); err = iomap_zero_range(vi, offset, to - offset, - NULL, &ntfs_seek_iomap_ops, + NULL, ntfs_seek_iomap_next, &ntfs_iomap_folio_ops, NULL); if (err < 0) goto out; @@ -1338,7 +1338,7 @@ static int ntfs_punch_hole(struct ntfs_inode *ni, int mode, loff_t offset, from = ntfs_cluster_to_bytes(vol, end_vcn - 1); if (from < ni->initialized_size) { err = iomap_zero_range(vi, from, end_offset - from, - NULL, &ntfs_seek_iomap_ops, + NULL, ntfs_seek_iomap_next, &ntfs_iomap_folio_ops, NULL); if (err < 0) goto out; diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 206db4e99ef1..a453d103445b 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -2589,7 +2589,7 @@ int ntfs_extend_initialized_size(struct inode *vi, const loff_t offset, if (old_init_size < offset) { err = iomap_zero_range(vi, old_init_size, offset - old_init_size, - NULL, &ntfs_seek_iomap_ops, + NULL, ntfs_seek_iomap_next, &ntfs_iomap_folio_ops, NULL); if (err) goto out; diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c index 6e51cce3aeba..72f85fc07b6e 100644 --- a/fs/ntfs/iomap.c +++ b/fs/ntfs/iomap.c @@ -293,12 +293,8 @@ static int ntfs_read_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } -static DEFINE_IOMAP_ITER_NEXT_END(ntfs_read_iomap_next, - ntfs_read_iomap_begin, ntfs_read_iomap_end); - -const struct iomap_ops ntfs_read_iomap_ops = { - .iomap_next = ntfs_read_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT_END(ntfs_read_iomap_next, ntfs_read_iomap_begin, + ntfs_read_iomap_end); /* * Check that the cached iomap still matches the NTFS runlist before @@ -351,15 +347,7 @@ static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t leng static DEFINE_IOMAP_ITER_NEXT_END(ntfs_zero_read_iomap_next, ntfs_seek_iomap_begin, ntfs_zero_read_iomap_end); -static const struct iomap_ops ntfs_zero_read_iomap_ops = { - .iomap_next = ntfs_zero_read_iomap_next, -}; - -static DEFINE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next, ntfs_seek_iomap_begin); - -const struct iomap_ops ntfs_seek_iomap_ops = { - .iomap_next = ntfs_seek_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next, ntfs_seek_iomap_begin); int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length) { @@ -378,7 +366,7 @@ static int ntfs_zero_range(struct inode *inode, loff_t offset, loff_t length) return iomap_zero_range(inode, offset, length, NULL, - &ntfs_zero_read_iomap_ops, + ntfs_zero_read_iomap_next, &ntfs_zero_iomap_folio_ops, NULL); } @@ -782,12 +770,8 @@ static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } -static DEFINE_IOMAP_ITER_NEXT_END(ntfs_write_iomap_next, - ntfs_write_iomap_begin, ntfs_write_iomap_end); - -const struct iomap_ops ntfs_write_iomap_ops = { - .iomap_next = ntfs_write_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT_END(ntfs_write_iomap_next, ntfs_write_iomap_begin, + ntfs_write_iomap_end); static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, @@ -797,13 +781,9 @@ static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_MKWRITE); } -static DEFINE_IOMAP_ITER_NEXT_END(ntfs_page_mkwrite_iomap_next, +DEFINE_IOMAP_ITER_NEXT_END(ntfs_page_mkwrite_iomap_next, ntfs_page_mkwrite_iomap_begin, ntfs_write_iomap_end); -const struct iomap_ops ntfs_page_mkwrite_iomap_ops = { - .iomap_next = ntfs_page_mkwrite_iomap_next, -}; - static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) @@ -812,12 +792,8 @@ static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_DIO); } -static DEFINE_IOMAP_ITER_NEXT_END(ntfs_dio_iomap_next, - ntfs_dio_iomap_begin, ntfs_write_iomap_end); - -const struct iomap_ops ntfs_dio_iomap_ops = { - .iomap_next = ntfs_dio_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT_END(ntfs_dio_iomap_next, ntfs_dio_iomap_begin, + ntfs_write_iomap_end); static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, struct folio *folio, u64 offset, unsigned int len, u64 end_pos) diff --git a/fs/ntfs/iomap.h b/fs/ntfs/iomap.h index 3abc1d493e91..fec8b907c48f 100644 --- a/fs/ntfs/iomap.h +++ b/fs/ntfs/iomap.h @@ -12,11 +12,11 @@ #include "volume.h" #include "inode.h" -extern const struct iomap_ops ntfs_write_iomap_ops; -extern const struct iomap_ops ntfs_read_iomap_ops; -extern const struct iomap_ops ntfs_seek_iomap_ops; -extern const struct iomap_ops ntfs_page_mkwrite_iomap_ops; -extern const struct iomap_ops ntfs_dio_iomap_ops; +DECLARE_IOMAP_ITER_NEXT(ntfs_write_iomap_next); +DECLARE_IOMAP_ITER_NEXT(ntfs_read_iomap_next); +DECLARE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next); +DECLARE_IOMAP_ITER_NEXT(ntfs_page_mkwrite_iomap_next); +DECLARE_IOMAP_ITER_NEXT(ntfs_dio_iomap_next); extern const struct iomap_writeback_ops ntfs_writeback_ops; extern const struct iomap_write_ops ntfs_iomap_folio_ops; extern int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length); diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index 1624f9b014e0..6c84a3e1c7fe 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -319,7 +319,7 @@ static int ntfs_extend_initialized_size(struct file *file, } err = iomap_zero_range(inode, valid, new_valid - valid, NULL, - &ntfs_iomap_ops, &ntfs_iomap_folio_ops, NULL); + ntfs_iomap_next, &ntfs_iomap_folio_ops, NULL); if (err) { ni->i_valid = valid; ntfs_inode_warn(inode, @@ -558,7 +558,7 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) /* Zero head of punch. */ if (tmp > from) { err = iomap_zero_range(inode, from, tmp - from, NULL, - &ntfs_iomap_ops, + ntfs_iomap_next, &ntfs_iomap_folio_ops, NULL); if (err) goto out; @@ -576,7 +576,7 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) /* Zero tail of punch. */ if (vbo < end_a && end_a < end) { err = iomap_zero_range(inode, end_a, end - end_a, NULL, - &ntfs_iomap_ops, + ntfs_iomap_next, &ntfs_iomap_folio_ops, NULL); if (err) goto out; @@ -896,7 +896,7 @@ static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) goto out; } - ret = iomap_dio_rw(iocb, iter, &ntfs_iomap_ops, NULL, dio_flags, + ret = iomap_dio_rw(iocb, iter, ntfs_iomap_next, NULL, dio_flags, NULL, 0); if (ret <= 0) @@ -1313,7 +1313,7 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) !ntfs_should_use_dio(iocb, from)) { iocb->ki_flags &= ~IOCB_DIRECT; - ret = iomap_file_buffered_write(iocb, from, &ntfs_iomap_ops, + ret = iomap_file_buffered_write(iocb, from, ntfs_iomap_next, &ntfs_iomap_folio_ops, NULL); inode_unlock(inode); @@ -1330,7 +1330,7 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; } - ret = iomap_dio_rw(iocb, from, &ntfs_iomap_ops, NULL, + ret = iomap_dio_rw(iocb, from, ntfs_iomap_next, NULL, IOMAP_DIO_FORCE_WAIT, NULL, 0); if (ret == -ENOTBLK) { @@ -1343,7 +1343,7 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) vbo = iocb->ki_pos; iocb->ki_flags &= ~IOCB_DIRECT; - err = iomap_file_buffered_write(iocb, from, &ntfs_iomap_ops, + err = iomap_file_buffered_write(iocb, from, ntfs_iomap_next, &ntfs_iomap_folio_ops, NULL); if (err < 0) { ret = err; @@ -1493,7 +1493,7 @@ int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, inode_lock_shared(inode); - err = iomap_fiemap(inode, fieinfo, start, len, &ntfs_iomap_ops); + err = iomap_fiemap(inode, fieinfo, start, len, ntfs_iomap_next); inode_unlock_shared(inode); return err; diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 43d5e83403aa..45f894d182dd 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -788,7 +788,7 @@ static sector_t ntfs_bmap(struct address_space *mapping, sector_t block) ni_allocate_da_blocks(ni); } - return iomap_bmap(mapping, block, &ntfs_iomap_ops); + return iomap_bmap(mapping, block, ntfs_iomap_next); } static void ntfs_iomap_read_end_io(struct bio *bio) @@ -860,7 +860,7 @@ static int ntfs_read_folio(struct file *file, struct folio *folio) return err; } - iomap_read_folio(&ntfs_iomap_ops, &ctx, NULL); + iomap_read_folio(ntfs_iomap_next, &ctx, NULL); return 0; } @@ -884,7 +884,7 @@ static void ntfs_readahead(struct readahead_control *rac) return; } - iomap_readahead(&ntfs_iomap_ops, &ctx, NULL); + iomap_readahead(ntfs_iomap_next, &ctx, NULL); } int ntfs_set_size(struct inode *inode, u64 new_size) @@ -2336,12 +2336,7 @@ const struct address_space_operations ntfs_aops_cmpr = { .invalidate_folio = iomap_invalidate_folio, }; -static DEFINE_IOMAP_ITER_NEXT_END(ntfs_iomap_next, ntfs_iomap_begin, - ntfs_iomap_end); - -const struct iomap_ops ntfs_iomap_ops = { - .iomap_next = ntfs_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT_END(ntfs_iomap_next, ntfs_iomap_begin, ntfs_iomap_end); const struct iomap_write_ops ntfs_iomap_folio_ops = { .put_folio = ntfs_iomap_put_folio, diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index f400cf94d05b..16e7eab3e16d 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -814,7 +815,7 @@ int ntfs_create_inode(const struct mnt_idmap *idmap, struct inode *dir, int ntfs_link_inode(struct inode *inode, struct dentry *dentry); int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry); void ntfs_evict_inode(struct inode *inode); -extern const struct iomap_ops ntfs_iomap_ops; +DECLARE_IOMAP_ITER_NEXT(ntfs_iomap_next); extern const struct iomap_write_ops ntfs_iomap_folio_ops; extern const struct inode_operations ntfs_link_inode_operations; extern const struct address_space_operations ntfs_aops; diff --git a/fs/remap_range.c b/fs/remap_range.c index 6eb7d845de5d..7d269837f63f 100644 --- a/fs/remap_range.c +++ b/fs/remap_range.c @@ -277,7 +277,7 @@ int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, loff_t *len, unsigned int remap_flags, - const struct iomap_ops *dax_read_ops) + iomap_iter_next_fn dax_read_next) { struct inode *inode_in = file_inode(file_in); struct inode *inode_out = file_inode(file_out); @@ -340,10 +340,10 @@ __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, if (!IS_DAX(inode_in)) ret = vfs_dedupe_file_range_compare(file_in, pos_in, file_out, pos_out, *len, &is_same); - else if (dax_read_ops) + else if (dax_read_next) ret = dax_dedupe_file_range_compare(inode_in, pos_in, inode_out, pos_out, *len, &is_same, - dax_read_ops); + dax_read_next); else return -EINVAL; if (ret) diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index c30e688cfc9f..1db6f3640c53 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -575,7 +575,7 @@ xfs_vm_bmap( */ if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip)) return 0; - return iomap_bmap(mapping, block, &xfs_read_iomap_ops); + return iomap_bmap(mapping, block, xfs_read_iomap_next); } static void @@ -614,7 +614,7 @@ xfs_vm_read_folio( struct iomap_read_folio_ctx ctx = { .cur_folio = folio }; ctx.ops = xfs_get_iomap_read_ops(folio->mapping); - iomap_read_folio(&xfs_read_iomap_ops, &ctx, NULL); + iomap_read_folio(xfs_read_iomap_next, &ctx, NULL); return 0; } @@ -625,7 +625,7 @@ xfs_vm_readahead( struct iomap_read_folio_ctx ctx = { .rac = rac }; ctx.ops = xfs_get_iomap_read_ops(rac->mapping), - iomap_readahead(&xfs_read_iomap_ops, &ctx, NULL); + iomap_readahead(xfs_read_iomap_next, &ctx, NULL); } static int @@ -671,7 +671,7 @@ xfs_vm_swap_activate( sis->bdev = xfs_inode_buftarg(ip)->bt_bdev; return iomap_swapfile_activate(sis, swap_file, span, - &xfs_read_iomap_ops); + xfs_read_iomap_next); } const struct address_space_operations xfs_address_space_operations = { diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 7d0c8a197575..e772347aff9a 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -250,12 +250,12 @@ xfs_file_dio_read( if (ret) return ret; if (mapping_stable_writes(iocb->ki_filp->f_mapping)) { - ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, + ret = iomap_dio_rw(iocb, to, xfs_read_iomap_next, &xfs_dio_read_bounce_ops, 0, NULL, 0); } else { ret = iomap_dio_read_simple(iocb, to, xfs_read_iomap_begin); if (ret == -ENOTBLK) - ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, + ret = iomap_dio_rw(iocb, to, xfs_read_iomap_next, NULL, 0, NULL, 0); } xfs_iunlock(ip, XFS_IOLOCK_SHARED); @@ -279,7 +279,7 @@ xfs_file_dax_read( ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); if (ret) return ret; - ret = dax_iomap_rw(iocb, to, &xfs_read_iomap_ops); + ret = dax_iomap_rw(iocb, to, xfs_read_iomap_next); xfs_iunlock(ip, XFS_IOLOCK_SHARED); file_accessed(iocb->ki_filp); @@ -729,7 +729,7 @@ xfs_file_dio_write_aligned( struct xfs_inode *ip, struct kiocb *iocb, struct iov_iter *from, - const struct iomap_ops *ops, + iomap_iter_next_fn next, const struct iomap_dio_ops *dops, struct xfs_zone_alloc_ctx *ac) { @@ -764,7 +764,7 @@ xfs_file_dio_write_aligned( if (mapping_stable_writes(iocb->ki_filp->f_mapping)) dio_flags |= IOMAP_DIO_BOUNCE; trace_xfs_file_direct_write(iocb, from); - ret = iomap_dio_rw(iocb, from, ops, dops, dio_flags, ac, 0); + ret = iomap_dio_rw(iocb, from, next, dops, dio_flags, ac, 0); out_unlock: xfs_iunlock(ip, iolock); return ret; @@ -786,7 +786,7 @@ xfs_file_dio_write_zoned( if (ret < 0) return ret; ret = xfs_file_dio_write_aligned(ip, iocb, from, - &xfs_zoned_direct_write_iomap_ops, + xfs_zoned_direct_write_iomap_next, &xfs_dio_zoned_write_ops, &ac); xfs_zoned_space_unreserve(ip->i_mount, &ac); return ret; @@ -811,16 +811,16 @@ xfs_file_dio_write_atomic( unsigned int iolock = XFS_IOLOCK_SHARED; ssize_t ret, ocount = iov_iter_count(from); unsigned int dio_flags = 0; - const struct iomap_ops *dops; + iomap_iter_next_fn next; /* * HW offload should be faster, so try that first if it is already * known that the write length is not too large. */ if (ocount > xfs_inode_buftarg(ip)->bt_awu_max) - dops = &xfs_atomic_write_cow_iomap_ops; + next = xfs_atomic_write_cow_iomap_next; else - dops = &xfs_direct_write_iomap_ops; + next = xfs_direct_write_iomap_next; retry: ret = xfs_ilock_iocb_for_write(iocb, &iolock); @@ -840,18 +840,19 @@ retry: trace_xfs_file_direct_write(iocb, from); if (mapping_stable_writes(iocb->ki_filp->f_mapping)) dio_flags |= IOMAP_DIO_BOUNCE; - ret = iomap_dio_rw(iocb, from, dops, &xfs_dio_write_ops, dio_flags, + ret = iomap_dio_rw(iocb, from, next, &xfs_dio_write_ops, dio_flags, NULL, 0); /* - * The retry mechanism is based on the ->iomap_next method returning - * -ENOPROTOOPT, which would be when the REQ_ATOMIC-based write is not - * possible. The REQ_ATOMIC-based method is typically not possible if - * the write spans multiple extents or the disk blocks are misaligned. + * The retry mechanism is based on the iomap_iter_next_fn callback + * returning -ENOPROTOOPT, which would be when the REQ_ATOMIC-based + * write is not possible. The REQ_ATOMIC-based method is typically not + * possible if the write spans multiple extents or the disk blocks are + * misaligned. */ - if (ret == -ENOPROTOOPT && dops == &xfs_direct_write_iomap_ops) { + if (ret == -ENOPROTOOPT && next == xfs_direct_write_iomap_next) { xfs_iunlock(ip, iolock); - dops = &xfs_atomic_write_cow_iomap_ops; + next = xfs_atomic_write_cow_iomap_next; goto retry; } @@ -934,7 +935,7 @@ retry_exclusive: flags |= IOMAP_DIO_BOUNCE; trace_xfs_file_direct_write(iocb, from); - ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops, + ret = iomap_dio_rw(iocb, from, xfs_direct_write_iomap_next, &xfs_dio_write_ops, flags, NULL, 0); /* @@ -974,7 +975,7 @@ xfs_file_dio_write( if (iocb->ki_flags & IOCB_ATOMIC) return xfs_file_dio_write_atomic(ip, iocb, from); return xfs_file_dio_write_aligned(ip, iocb, from, - &xfs_direct_write_iomap_ops, &xfs_dio_write_ops, NULL); + xfs_direct_write_iomap_next, &xfs_dio_write_ops, NULL); } static noinline ssize_t @@ -998,7 +999,7 @@ xfs_file_dax_write( pos = iocb->ki_pos; trace_xfs_file_dax_write(iocb, from); - ret = dax_iomap_rw(iocb, from, &xfs_dax_write_iomap_ops); + ret = dax_iomap_rw(iocb, from, xfs_dax_write_iomap_next); if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { i_size_write(inode, iocb->ki_pos); error = xfs_setfilesize(ip, pos, ret); @@ -1041,7 +1042,7 @@ write_retry: trace_xfs_file_buffered_write(iocb, from); ret = iomap_file_buffered_write(iocb, from, - &xfs_buffered_write_iomap_ops, &xfs_iomap_write_ops, + xfs_buffered_write_iomap_next, &xfs_iomap_write_ops, NULL); /* @@ -1122,7 +1123,7 @@ xfs_file_buffered_write_zoned( retry: trace_xfs_file_buffered_write(iocb, from); ret = iomap_file_buffered_write(iocb, from, - &xfs_buffered_write_iomap_ops, &xfs_iomap_write_ops, + xfs_buffered_write_iomap_next, &xfs_iomap_write_ops, &ac); if (ret == -ENOSPC && !cleared_space) { /* @@ -1931,10 +1932,10 @@ xfs_file_llseek( default: return generic_file_llseek(file, offset, whence); case SEEK_HOLE: - offset = iomap_seek_hole(inode, offset, &xfs_seek_iomap_ops); + offset = iomap_seek_hole(inode, offset, xfs_seek_iomap_next); break; case SEEK_DATA: - offset = iomap_seek_data(inode, offset, &xfs_seek_iomap_ops); + offset = iomap_seek_data(inode, offset, xfs_seek_iomap_next); break; } @@ -1958,8 +1959,8 @@ xfs_dax_fault_locked( } ret = dax_iomap_fault(vmf, order, &pfn, NULL, (write_fault && !vmf->cow_page) ? - &xfs_dax_write_iomap_ops : - &xfs_read_iomap_ops); + xfs_dax_write_iomap_next : + xfs_read_iomap_next); if (ret & VM_FAULT_NEEDDSYNC) ret = dax_finish_sync_fault(vmf, order, pfn); return ret; @@ -2023,7 +2024,7 @@ __xfs_write_fault( if (IS_DAX(inode)) ret = xfs_dax_fault_locked(vmf, order, true); else - ret = iomap_page_mkwrite(vmf, &xfs_buffered_write_iomap_ops, + ret = iomap_page_mkwrite(vmf, xfs_buffered_write_iomap_next, ac); xfs_iunlock(ip, lock_mode); diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 329c745e3120..c591fb07a62d 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1037,13 +1037,9 @@ out_unlock: return error; } -static DEFINE_IOMAP_ITER_NEXT(xfs_direct_write_iomap_next, +DEFINE_IOMAP_ITER_NEXT(xfs_direct_write_iomap_next, xfs_direct_write_iomap_begin); -const struct iomap_ops xfs_direct_write_iomap_ops = { - .iomap_next = xfs_direct_write_iomap_next, -}; - #ifdef CONFIG_XFS_RT /* * This is really simple. The space has already been reserved before taking the @@ -1087,12 +1083,9 @@ xfs_zoned_direct_write_iomap_begin( return 0; } -static DEFINE_IOMAP_ITER_NEXT(xfs_zoned_direct_write_iomap_next, +DEFINE_IOMAP_ITER_NEXT(xfs_zoned_direct_write_iomap_next, xfs_zoned_direct_write_iomap_begin); -const struct iomap_ops xfs_zoned_direct_write_iomap_ops = { - .iomap_next = xfs_zoned_direct_write_iomap_next, -}; #endif /* CONFIG_XFS_RT */ #ifdef DEBUG @@ -1275,13 +1268,9 @@ out_unlock: return error; } -static DEFINE_IOMAP_ITER_NEXT(xfs_atomic_write_cow_iomap_next, +DEFINE_IOMAP_ITER_NEXT(xfs_atomic_write_cow_iomap_next, xfs_atomic_write_cow_iomap_begin); -const struct iomap_ops xfs_atomic_write_cow_iomap_ops = { - .iomap_next = xfs_atomic_write_cow_iomap_next, -}; - static int xfs_dax_write_iomap_end( struct inode *inode, @@ -1302,13 +1291,9 @@ xfs_dax_write_iomap_end( return xfs_reflink_end_cow(ip, pos, written); } -static DEFINE_IOMAP_ITER_NEXT_END(xfs_dax_write_iomap_next, +DEFINE_IOMAP_ITER_NEXT_END(xfs_dax_write_iomap_next, xfs_direct_write_iomap_begin, xfs_dax_write_iomap_end); -const struct iomap_ops xfs_dax_write_iomap_ops = { - .iomap_next = xfs_dax_write_iomap_next, -}; - /* * Convert a hole to a delayed allocation. */ @@ -2191,13 +2176,9 @@ xfs_buffered_write_iomap_end( return 0; } -static DEFINE_IOMAP_ITER_NEXT_END(xfs_buffered_write_iomap_next, +DEFINE_IOMAP_ITER_NEXT_END(xfs_buffered_write_iomap_next, xfs_buffered_write_iomap_begin, xfs_buffered_write_iomap_end); -const struct iomap_ops xfs_buffered_write_iomap_ops = { - .iomap_next = xfs_buffered_write_iomap_next, -}; - int xfs_read_iomap_begin( struct inode *inode, @@ -2239,11 +2220,7 @@ xfs_read_iomap_begin( shared ? IOMAP_F_SHARED : 0, seq); } -static DEFINE_IOMAP_ITER_NEXT(xfs_read_iomap_next, xfs_read_iomap_begin); - -const struct iomap_ops xfs_read_iomap_ops = { - .iomap_next = xfs_read_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT(xfs_read_iomap_next, xfs_read_iomap_begin); static int xfs_seek_iomap_begin( @@ -2329,11 +2306,7 @@ out_unlock: return error; } -static DEFINE_IOMAP_ITER_NEXT(xfs_seek_iomap_next, xfs_seek_iomap_begin); - -const struct iomap_ops xfs_seek_iomap_ops = { - .iomap_next = xfs_seek_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT(xfs_seek_iomap_next, xfs_seek_iomap_begin); static int xfs_xattr_iomap_begin( @@ -2378,11 +2351,7 @@ out_unlock: return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq); } -static DEFINE_IOMAP_ITER_NEXT(xfs_xattr_iomap_next, xfs_xattr_iomap_begin); - -const struct iomap_ops xfs_xattr_iomap_ops = { - .iomap_next = xfs_xattr_iomap_next, -}; +DEFINE_IOMAP_ITER_NEXT(xfs_xattr_iomap_next, xfs_xattr_iomap_begin); int xfs_zero_range( @@ -2398,9 +2367,9 @@ xfs_zero_range( if (IS_DAX(inode)) return dax_zero_range(inode, pos, len, did_zero, - &xfs_dax_write_iomap_ops); + xfs_dax_write_iomap_next); return iomap_zero_range(inode, pos, len, did_zero, - &xfs_buffered_write_iomap_ops, &xfs_iomap_write_ops, + xfs_buffered_write_iomap_next, &xfs_iomap_write_ops, ac); } @@ -2415,8 +2384,8 @@ xfs_truncate_page( if (IS_DAX(inode)) return dax_truncate_page(inode, pos, did_zero, - &xfs_dax_write_iomap_ops); + xfs_dax_write_iomap_next); return iomap_truncate_page(inode, pos, did_zero, - &xfs_buffered_write_iomap_ops, &xfs_iomap_write_ops, + xfs_buffered_write_iomap_next, &xfs_iomap_write_ops, ac); } diff --git a/fs/xfs/xfs_iomap.h b/fs/xfs/xfs_iomap.h index f2520a9b3a13..974e4ed34f1d 100644 --- a/fs/xfs/xfs_iomap.h +++ b/fs/xfs/xfs_iomap.h @@ -69,14 +69,14 @@ int xfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned flags, struct iomap *iomap, struct iomap *srcmap); -extern const struct iomap_ops xfs_buffered_write_iomap_ops; -extern const struct iomap_ops xfs_direct_write_iomap_ops; -extern const struct iomap_ops xfs_zoned_direct_write_iomap_ops; -extern const struct iomap_ops xfs_read_iomap_ops; -extern const struct iomap_ops xfs_seek_iomap_ops; -extern const struct iomap_ops xfs_xattr_iomap_ops; -extern const struct iomap_ops xfs_dax_write_iomap_ops; -extern const struct iomap_ops xfs_atomic_write_cow_iomap_ops; +DECLARE_IOMAP_ITER_NEXT(xfs_buffered_write_iomap_next); +DECLARE_IOMAP_ITER_NEXT(xfs_direct_write_iomap_next); +DECLARE_IOMAP_ITER_NEXT(xfs_zoned_direct_write_iomap_next); +DECLARE_IOMAP_ITER_NEXT(xfs_read_iomap_next); +DECLARE_IOMAP_ITER_NEXT(xfs_seek_iomap_next); +DECLARE_IOMAP_ITER_NEXT(xfs_xattr_iomap_next); +DECLARE_IOMAP_ITER_NEXT(xfs_dax_write_iomap_next); +DECLARE_IOMAP_ITER_NEXT(xfs_atomic_write_cow_iomap_next); extern const struct iomap_write_ops xfs_iomap_write_ops; #endif /* __XFS_IOMAP_H__*/ diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index d307ec97fac8..6c5633db30da 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -1256,10 +1256,10 @@ xfs_vn_fiemap( if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { fieinfo->fi_flags &= ~FIEMAP_FLAG_XATTR; error = iomap_fiemap(inode, fieinfo, start, length, - &xfs_xattr_iomap_ops); + xfs_xattr_iomap_next); } else { error = iomap_fiemap(inode, fieinfo, start, length, - &xfs_read_iomap_ops); + xfs_read_iomap_next); } xfs_iunlock(XFS_I(inode), XFS_IOLOCK_SHARED); diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 104542b10d9e..c5c796e2b83d 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -1724,7 +1724,7 @@ xfs_reflink_remap_prep( pos_out, len, remap_flags); else ret = dax_remap_file_range_prep(file_in, pos_in, file_out, - pos_out, len, remap_flags, &xfs_read_iomap_ops); + pos_out, len, remap_flags, xfs_read_iomap_next); if (ret || *len == 0) goto out_unlock; @@ -1919,10 +1919,10 @@ xfs_reflink_unshare( if (IS_DAX(inode)) error = dax_file_unshare(inode, offset, len, - &xfs_dax_write_iomap_ops); + xfs_dax_write_iomap_next); else error = iomap_file_unshare(inode, offset, len, - &xfs_buffered_write_iomap_ops, + xfs_buffered_write_iomap_next, &xfs_iomap_write_ops); if (error) goto out; diff --git a/fs/zonefs/file.c b/fs/zonefs/file.c index 5b34849be7a2..319d9502ba56 100644 --- a/fs/zonefs/file.c +++ b/fs/zonefs/file.c @@ -59,10 +59,6 @@ static int zonefs_read_iomap_begin(struct inode *inode, loff_t offset, static DEFINE_IOMAP_ITER_NEXT(zonefs_read_iomap_next, zonefs_read_iomap_begin); -static const struct iomap_ops zonefs_read_iomap_ops = { - .iomap_next = zonefs_read_iomap_next, -}; - static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) @@ -111,19 +107,15 @@ static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset, static DEFINE_IOMAP_ITER_NEXT(zonefs_write_iomap_next, zonefs_write_iomap_begin); -static const struct iomap_ops zonefs_write_iomap_ops = { - .iomap_next = zonefs_write_iomap_next, -}; - static int zonefs_read_folio(struct file *unused, struct folio *folio) { - iomap_bio_read_folio(folio, &zonefs_read_iomap_ops); + iomap_bio_read_folio(folio, zonefs_read_iomap_next); return 0; } static void zonefs_readahead(struct readahead_control *rac) { - iomap_bio_readahead(rac, &zonefs_read_iomap_ops); + iomap_bio_readahead(rac, zonefs_read_iomap_next); } /* @@ -184,7 +176,7 @@ static int zonefs_swap_activate(struct swap_info_struct *sis, } return iomap_swapfile_activate(sis, swap_file, span, - &zonefs_read_iomap_ops); + zonefs_read_iomap_next); } const struct address_space_operations zonefs_file_aops = { @@ -314,7 +306,7 @@ static vm_fault_t zonefs_filemap_page_mkwrite(struct vm_fault *vmf) /* Serialize against truncates */ filemap_invalidate_lock_shared(inode->i_mapping); - ret = iomap_page_mkwrite(vmf, &zonefs_write_iomap_ops, NULL); + ret = iomap_page_mkwrite(vmf, zonefs_write_iomap_next, NULL); filemap_invalidate_unlock_shared(inode->i_mapping); sb_end_pagefault(inode->i_sb); @@ -530,7 +522,7 @@ static ssize_t zonefs_file_dio_write(struct kiocb *iocb, struct iov_iter *from) * page invalidation. Overwrite that error code with EBUSY so that * the user can make sense of the error. */ - ret = iomap_dio_rw(iocb, from, &zonefs_write_iomap_ops, + ret = iomap_dio_rw(iocb, from, zonefs_write_iomap_next, &zonefs_write_dio_ops, 0, NULL, 0); if (ret == -ENOTBLK) ret = -EBUSY; @@ -580,7 +572,7 @@ static ssize_t zonefs_file_buffered_write(struct kiocb *iocb, if (ret <= 0) goto inode_unlock; - ret = iomap_file_buffered_write(iocb, from, &zonefs_write_iomap_ops, + ret = iomap_file_buffered_write(iocb, from, zonefs_write_iomap_next, NULL, NULL); if (ret == -EIO) zonefs_io_error(inode, true); @@ -675,7 +667,7 @@ static ssize_t zonefs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) goto inode_unlock; } file_accessed(iocb->ki_filp); - ret = iomap_dio_rw(iocb, to, &zonefs_read_iomap_ops, + ret = iomap_dio_rw(iocb, to, zonefs_read_iomap_next, &zonefs_read_dio_ops, 0, NULL, 0); } else { ret = generic_file_read_iter(iocb, to); diff --git a/include/linux/dax.h b/include/linux/dax.h index a3af0c63c61b..2e0797e4d408 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -3,6 +3,7 @@ #define _LINUX_DAX_H #include +#include #include #include @@ -10,9 +11,6 @@ typedef unsigned long dax_entry_t; struct dax_device; struct gendisk; -struct iomap_ops; -struct iomap_iter; -struct iomap; enum dax_access_mode { DAX_ACCESS, @@ -200,11 +198,11 @@ static inline void dax_unlock_mapping_entry(struct address_space *mapping, #endif int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len, - const struct iomap_ops *ops); + iomap_iter_next_fn next); int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, - const struct iomap_ops *ops); + iomap_iter_next_fn next); int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, - const struct iomap_ops *ops); + iomap_iter_next_fn next); static inline bool dax_page_is_idle(struct page *page) { @@ -253,10 +251,10 @@ int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len, void dax_flush(struct dax_device *dax_dev, void *addr, size_t size); ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops); + iomap_iter_next_fn next); vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order, unsigned long *pfnp, int *errp, - const struct iomap_ops *ops); + iomap_iter_next_fn next); vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, unsigned int order, unsigned long pfn); int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); @@ -275,11 +273,11 @@ void dax_break_layout_final(struct inode *inode); int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff, struct inode *dest, loff_t destoff, loff_t len, bool *is_same, - const struct iomap_ops *ops); + iomap_iter_next_fn next); int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, loff_t *len, unsigned int remap_flags, - const struct iomap_ops *ops); + iomap_iter_next_fn next); static inline bool dax_mapping(struct address_space *mapping) { return mapping->host && IS_DAX(mapping->host); diff --git a/include/linux/fs.h b/include/linux/fs.h index acea6c40caaa..5b3a100aba77 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -70,7 +70,8 @@ struct fsnotify_mark_connector; struct fs_context; struct fs_parameter_spec; struct file_kattr; -struct iomap_ops; +struct iomap_iter; +struct iomap; struct delegated_inode; extern void __init inode_init(void); @@ -2083,7 +2084,9 @@ int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write); int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, loff_t *len, unsigned int remap_flags, - const struct iomap_ops *dax_read_ops); + int (*dax_read_next)(const struct iomap_iter *iter, + struct iomap *iomap, + struct iomap *srcmap)); int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, loff_t *count, unsigned int remap_flags); diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 66211ddd96cb..78a1b7603a4f 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -239,10 +239,6 @@ typedef int (*iomap_iter_end_fn)(struct inode *inode, loff_t pos, loff_t length, typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter, struct iomap *iomap, struct iomap *srcmap); -struct iomap_ops { - iomap_iter_next_fn iomap_next; -}; - /** * struct iomap_iter - Iterate through a range of a file * @inode: Set at the start of the iteration and should not change. @@ -254,7 +250,7 @@ struct iomap_ops { * incremental iter advance. * @status: Status of the most recent iteration. Zero on success or a negative * errno on error. - * @flags: Zero or more of the iomap_next flags above. + * @flags: Zero or more of the iomap_iter_next flags above. * @iomap: Map describing the I/O iteration * @srcmap: Source map for COW operations */ @@ -271,7 +267,7 @@ struct iomap_iter { void *private; }; -int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops); +int iomap_iter(struct iomap_iter *iter, iomap_iter_next_fn next); int iomap_iter_advance(struct iomap_iter *iter, u64 count); /** @@ -339,7 +335,7 @@ int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, * @begin: callback that produces a mapping for the current position * @end: optional callback that finishes the previous mapping, or NULL * - * Inline helper that implements the common body of an ->iomap_next() + * Inline helper that implements the common body of an iomap_iter_next_fn * callback: it finishes the previous mapping via @end (if present), decides * via iomap_iter_continue() whether to keep going, and obtains the next * mapping via @begin. @@ -393,6 +389,14 @@ int name(const struct iomap_iter *iter, struct iomap *iomap, \ #define DEFINE_IOMAP_ITER_NEXT(name, begin_fn) \ DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, NULL) +/* + * Forward-declare an iomap_iter_next_fn callback generated by + * DEFINE_IOMAP_ITER_NEXT(). + */ +#define DECLARE_IOMAP_ITER_NEXT(name) \ +int name(const struct iomap_iter *iter, struct iomap *iomap, \ + struct iomap *srcmap) + /* * Return the file offset for the first unchanged block after a short write. * @@ -427,15 +431,15 @@ static inline bool iomap_want_unshare_iter(const struct iomap_iter *iter) } ssize_t iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *from, - const struct iomap_ops *ops, + iomap_iter_next_fn next, const struct iomap_write_ops *write_ops, void *private); int iomap_fsverity_write(struct file *file, loff_t pos, size_t length, - const void *buf, const struct iomap_ops *ops, + const void *buf, iomap_iter_next_fn next, const struct iomap_write_ops *write_ops); -void iomap_read_folio(const struct iomap_ops *ops, - struct iomap_read_folio_ctx *ctx, void *private); -void iomap_readahead(const struct iomap_ops *ops, - struct iomap_read_folio_ctx *ctx, void *private); +void iomap_read_folio(iomap_iter_next_fn next, struct iomap_read_folio_ctx *ctx, + void *private); +void iomap_readahead(iomap_iter_next_fn next, struct iomap_read_folio_ctx *ctx, + void *private); bool iomap_is_partially_uptodate(struct folio *, size_t from, size_t count); struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len); bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags); @@ -443,17 +447,17 @@ void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len); bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio); void iomap_folio_mark_uptodate(struct folio *folio); int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, - const struct iomap_ops *ops, + iomap_iter_next_fn next, const struct iomap_write_ops *write_ops); unsigned int iomap_fill_dirty_folios(struct iomap_iter *iter, loff_t *start, loff_t end, unsigned int *iomap_flags); int iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, - bool *did_zero, const struct iomap_ops *ops, + bool *did_zero, iomap_iter_next_fn next, const struct iomap_write_ops *write_ops, void *private); int iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, - const struct iomap_ops *ops, + iomap_iter_next_fn next, const struct iomap_write_ops *write_ops, void *private); -vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops, +vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, iomap_iter_next_fn next, void *private); typedef void (*iomap_punch_t)(struct inode *inode, loff_t offset, loff_t length, struct iomap *iomap); @@ -462,13 +466,13 @@ void iomap_write_delalloc_release(struct inode *inode, loff_t start_byte, iomap_punch_t punch); int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, - u64 start, u64 len, const struct iomap_ops *ops); + u64 start, u64 len, iomap_iter_next_fn next); loff_t iomap_seek_hole(struct inode *inode, loff_t offset, - const struct iomap_ops *ops); + iomap_iter_next_fn next); loff_t iomap_seek_data(struct inode *inode, loff_t offset, - const struct iomap_ops *ops); + iomap_iter_next_fn next); sector_t iomap_bmap(struct address_space *mapping, sector_t bno, - const struct iomap_ops *ops); + iomap_iter_next_fn next); /* * Flags for iomap_ioend->io_flags. @@ -709,10 +713,10 @@ struct iomap_dio_ops { #define IOMAP_DIO_BOUNCE (1 << 4) ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, const struct iomap_dio_ops *dops, + iomap_iter_next_fn next, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before); struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, const struct iomap_dio_ops *dops, + iomap_iter_next_fn next, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before); ssize_t iomap_dio_complete(struct iomap_dio *dio); void iomap_dio_bio_end_io(struct bio *bio); @@ -788,9 +792,9 @@ struct swap_info_struct; int iomap_swapfile_activate(struct swap_info_struct *sis, struct file *swap_file, sector_t *pagespan, - const struct iomap_ops *ops); + iomap_iter_next_fn next); #else -# define iomap_swapfile_activate(sis, swapfile, pagespan, ops) (-EIO) +# define iomap_swapfile_activate(sis, swapfile, pagespan, next) (-EIO) #endif /* CONFIG_SWAP */ extern struct bio_set iomap_ioend_bioset; @@ -804,25 +808,25 @@ void iomap_bio_submit_read_endio(const struct iomap_iter *iter, extern const struct iomap_read_ops iomap_bio_read_ops; static inline void iomap_bio_read_folio(struct folio *folio, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct iomap_read_folio_ctx ctx = { .ops = &iomap_bio_read_ops, .cur_folio = folio, }; - iomap_read_folio(ops, &ctx, NULL); + iomap_read_folio(next, &ctx, NULL); } static inline void iomap_bio_readahead(struct readahead_control *rac, - const struct iomap_ops *ops) + iomap_iter_next_fn next) { struct iomap_read_folio_ctx ctx = { .ops = &iomap_bio_read_ops, .rac = rac, }; - iomap_readahead(ops, &ctx, NULL); + iomap_readahead(next, &ctx, NULL); } #endif /* CONFIG_BLOCK */ From 5612225809c78959c888fa7066bdfdf46b8aee45 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 27 Oct 2026 14:53:26 -0700 Subject: [PATCH 08/28] Documentation: iomap: update docs to reflect iomap_iter_next model Filesystems no longer pass a struct iomap_ops with separate ->iomap_begin() and ->iomap_end() callbacks. Instead, every iomap operation takes a single iomap_iter_next() callback directly. iomap_iter_next() finishes the previous mapping (if any) and produces the next one. Most filesystems build it from begin and end helpers via the iomap_iter_next() helper. Update the iomap documentation to match this change. Reviewed-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20261027215326.2693300-6-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- Documentation/filesystems/iomap/design.rst | 140 +++++++++++++----- .../filesystems/iomap/operations.rst | 68 ++++----- Documentation/filesystems/iomap/porting.rst | 16 +- 3 files changed, 151 insertions(+), 73 deletions(-) diff --git a/Documentation/filesystems/iomap/design.rst b/Documentation/filesystems/iomap/design.rst index 681e0cbb6944..508e63ce205c 100644 --- a/Documentation/filesystems/iomap/design.rst +++ b/Documentation/filesystems/iomap/design.rst @@ -75,7 +75,10 @@ At a high level, an iomap operation `looks like this 1. For each byte in the operation range... - 1. Obtain a space mapping via ``->iomap_begin`` + 1. Obtain the next space mapping via the ``iomap_iter_next_fn`` callback. + From the second iteration onwards this same callback first finishes + the previous mapping (committing or unreserving space as needed) + and then produces the next one. 2. For each sub-unit of work... @@ -86,7 +89,13 @@ At a high level, an iomap operation `looks like this 3. Increment operation cursor - 4. Release the mapping via ``->iomap_end``, if necessary +iomap repeats this until the range is fully consumed. The ``iomap_iter_next_fn`` +callback returns ``1`` while there is more of the range left to process, +``0`` once it is fully consumed, or a negative errno on error. +Filesystems rarely implement this callback by hand. The ``iomap_iter_next`` +helper implements the finish-then-produce sequence in terms of two smaller +callbacks, ``begin`` and ``end``. See `The Mapping Callback`_ below for more +info. Each iomap operation will be covered in more detail below. This library was covered previously by an `LWN article @@ -189,7 +198,7 @@ The fields are as follows: * **IOMAP_DELALLOC**: A promise to allocate space at a later time ("delayed allocation"). If the filesystem returns IOMAP_F_NEW here and the write fails, the - ``->iomap_end`` function must delete the reservation. + ``end`` function must delete the reservation. The ``addr`` field must be set to ``IOMAP_NULL_ADDR``. * **IOMAP_MAPPED**: The file range maps to specific space on the @@ -208,12 +217,11 @@ The fields are as follows: * **IOMAP_INLINE**: The file range maps to the memory buffer specified by ``inline_data``. - For write operation, the ``->iomap_end`` function presumably - handles persisting the data. + For write operation, the ``end`` function must persist the data. The ``addr`` field must be set to ``IOMAP_NULL_ADDR``. * ``flags`` describe the status of the space mapping. - These flags should be set by the filesystem in ``->iomap_begin``: + These flags should be set by the filesystem in ``begin``: * **IOMAP_F_NEW**: The space under the mapping is newly allocated. Areas that will not be written to must be zeroed. @@ -262,15 +270,15 @@ The fields are as follows: update. These flags can be set by iomap itself during file operations. - The filesystem should supply an ``->iomap_end`` function if it needs + The filesystem should supply an ``end`` function if it needs to observe these flags: * **IOMAP_F_SIZE_CHANGED**: The file size has changed as a result of using this mapping. * **IOMAP_F_STALE**: The mapping was found to be stale. - iomap will call ``->iomap_end`` on this mapping and then - ``->iomap_begin`` to obtain a new mapping. + iomap will call ``end`` on this mapping and then ``begin`` to obtain a + new mapping. Currently, these flags are only set by pagecache operations. @@ -289,41 +297,103 @@ The fields are as follows: * ``private`` is a pointer to `filesystem-private information `_. - This value will be passed unchanged to ``->iomap_end``. + This value will be passed unchanged to ``end``. * ``validity_cookie`` is a magic freshness value set by the filesystem that should be used to detect stale mappings. For pagecache operations this is critical for correct operation because page faults can occur, which implies that filesystem locks - should not be held between ``->iomap_begin`` and ``->iomap_end``. + should not be held between ``begin`` and ``end``. Filesystems with completely static mappings need not set this value. Only pagecache operations revalidate mappings; see the section about ``iomap_valid`` for details. -``struct iomap_ops`` +The Mapping Callback -------------------- -Every iomap function requires the filesystem to pass an operations -structure to obtain a mapping and (optionally) to release the mapping: +Every iomap operation takes an ``iomap_iter_next_fn`` callback from the +filesystem. iomap calls it once per iteration of the file range: .. code-block:: c - struct iomap_ops { - int (*iomap_begin)(struct inode *inode, loff_t pos, loff_t length, - unsigned flags, struct iomap *iomap, - struct iomap *srcmap); + typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap); - int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length, - ssize_t written, unsigned flags, - struct iomap *iomap); - }; +``iomap_iter_next_fn`` +~~~~~~~~~~~~~~~~~~~~~~ -``->iomap_begin`` -~~~~~~~~~~~~~~~~~ +This must finish the previous mapping, if any, and then produce the next +mapping for the current iteration position described by ``iter``. +The mapping is returned through ``iomap`` (and through ``srcmap`` for +operations that read from one mapping while writing to another; see +``begin`` below). -iomap operations call ``->iomap_begin`` to obtain one file mapping for -the range of bytes specified by ``pos`` and ``length`` for the file -``inode``. +The callback returns ``1`` to continue iterating, ``0`` once the file +range has been fully consumed, or a negative errno on error. + +Most filesystems are not expected to implement all of these behaviors +in this callback themselves. They should instead call ``iomap_iter_next`` as +described below. + +``iomap_iter_next`` +~~~~~~~~~~~~~~~~~~~ + +Filesystems rarely need a hand-written ``iomap_iter_next_fn`` callback. The +``iomap_iter_next`` helper implements the finish-then-produce sequence in +terms of two smaller callbacks, ``begin`` and ``end``, so most +``iomap_iter_next_fn`` implementations are simply: + +.. code-block:: c + + static int my_iomap_next(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap) + { + return iomap_iter_next(iter, iomap, srcmap, + my_iomap_begin, my_iomap_end); + } + +This boilerplate is normally generated with the ``DEFINE_IOMAP_ITER_NEXT()`` +and ``DEFINE_IOMAP_ITER_NEXT_END()`` macros rather than written by hand. Use +``DEFINE_IOMAP_ITER_NEXT()`` for the common case with no ``end`` callback, and +``DEFINE_IOMAP_ITER_NEXT_END()`` when an ``end`` callback is needed: + +.. code-block:: c + + /* no end() callback */ + static DEFINE_IOMAP_ITER_NEXT(my_iomap_next, my_iomap_begin); + + /* with an end() callback */ + static DEFINE_IOMAP_ITER_NEXT_END(my_iomap_next, my_iomap_begin, + my_iomap_end); + +The macro generates a function with external linkage. Prefix the invocation +with ``static`` for a file-local callback, or leave it non-``static`` and add +a matching declaration to a header when the callback is referenced from +another file. + +``end`` may be ``NULL`` when the filesystem has nothing to finish. +The two callbacks have these prototypes: + +.. code-block:: c + + typedef int (*iomap_begin_fn)(struct inode *inode, loff_t pos, + loff_t length, unsigned flags, + struct iomap *iomap, struct iomap *srcmap); + + typedef int (*iomap_end_fn)(struct inode *inode, loff_t pos, + loff_t length, ssize_t written, + unsigned flags, struct iomap *iomap); + +``iomap_iter_next`` is an inline helper, so when it is called with fixed +``begin`` and ``end`` functions the compiler can inline both into the +filesystem's ``iomap_iter_next_fn`` callback, keeping indirect calls out of the +iteration hot path. The two callbacks are described next. + +``begin`` +~~~~~~~~~ + +The ``begin`` callback obtains one file mapping for the range of bytes +specified by ``pos`` and ``length`` for the file ``inode``. This mapping should be returned through the ``iomap`` pointer. The mapping must cover at least the first byte of the supplied file range, but it does not need to cover the entire requested range. @@ -377,18 +447,18 @@ information via ``srcmap``. Only pagecache and fsdax operations support reading from one mapping and writing to another. -``->iomap_end`` -~~~~~~~~~~~~~~~ +``end`` +~~~~~~~ -After the operation completes, the ``->iomap_end`` function, if present, -is called to signal that iomap is finished with a mapping. +After the operation completes, the ``end`` callback, if present, is called to +signal that iomap is finished with a mapping. Typically, implementations will use this function to tear down any -context that were set up in ``->iomap_begin``. +context that was set up in ``begin``. For example, a write might wish to commit the reservations for the bytes that were operated upon and unreserve any space that was not operated upon. ``written`` might be zero if no bytes were touched. -``flags`` will contain the same value passed to ``->iomap_begin``. +``flags`` will contain the same value passed to ``begin``. iomap ops for reads are not likely to need to supply this function. Both functions should return a negative errno code on error, or zero on @@ -421,8 +491,8 @@ iomap is concerned: accessing the folio until writeback is underway. * The **lower** level primitive is taken by the filesystem in the - ``->iomap_begin`` and ``->iomap_end`` functions to coordinate - access to the file space mapping information. + ``begin`` and ``end`` functions to coordinate access to the file space + mapping information. The fields of the iomap object should be filled out while holding this primitive. The upper level synchronization primitive, if any, remains held diff --git a/Documentation/filesystems/iomap/operations.rst b/Documentation/filesystems/iomap/operations.rst index da982ca7e413..a5784dee0850 100644 --- a/Documentation/filesystems/iomap/operations.rst +++ b/Documentation/filesystems/iomap/operations.rst @@ -17,6 +17,12 @@ Supported File Operations Below are a discussion of the high level file operations that iomap implements. +Each operation takes an ``iomap_iter_next_fn`` callback that supplies the file +mappings, as described in the iomap design document. The per-operation +``flags`` documented below are passed to that callback; references to +``begin`` and ``end`` name the two steps a typical callback is built from +via ``iomap_iter_next``. + Buffered I/O ============ @@ -90,10 +96,10 @@ iomap calls these functions: `_ that was set up by ``->get_folio``. - - ``iomap_valid``: The filesystem may not hold locks between - ``->iomap_begin`` and ``->iomap_end`` because pagecache operations - can take folio locks, fault on userspace pages, initiate writeback - for memory reclamation, or engage in other time-consuming actions. + - ``iomap_valid``: The filesystem may not hold locks between ``begin`` and + ``end`` because pagecache operations can take folio locks, fault on + userspace pages, initiate writeback for memory reclamation, or engage in + other time-consuming actions. If a file's space mapping data are mutable, it is possible that the mapping for a particular pagecache folio can `change in the time it takes @@ -114,12 +120,12 @@ iomap calls these functions: If the mapping is not valid, the mapping will be sampled again. To support making the validity decision, the filesystem's - ``->iomap_begin`` function may set ``struct iomap::validity_cookie`` + ``begin`` function may set ``struct iomap::validity_cookie`` at the same time that it populates the other iomap fields. A simple validation cookie implementation is a sequence counter. If the filesystem bumps the sequence counter every time it modifies the inode's extent map, it can be placed in the ``struct - iomap::validity_cookie`` during ``->iomap_begin``. + iomap::validity_cookie`` during ``begin``. If the value in the cookie is found to be different to the value the filesystem holds when the mapping is passed back to ``->iomap_valid``, then the iomap should considered stale and the @@ -199,7 +205,7 @@ Buffered Readahead and Reads The ``iomap_readahead`` function initiates readahead to the pagecache. The ``iomap_read_folio`` function reads one folio's worth of data into the pagecache. -The ``flags`` argument to ``->iomap_begin`` will be set to zero. +The ``flags`` argument to ``begin`` will be set to zero. The pagecache takes whatever locks it needs before calling the filesystem. @@ -231,7 +237,7 @@ Buffered Writes The ``iomap_file_buffered_write`` function writes an ``iocb`` to the pagecache. ``IOMAP_WRITE`` or ``IOMAP_WRITE`` | ``IOMAP_NOWAIT`` will be passed as -the ``flags`` argument to ``->iomap_begin``. +the ``flags`` argument to ``begin``. Callers commonly take ``i_rwsem`` in either shared or exclusive mode before calling this function. @@ -241,7 +247,7 @@ mmap Write Faults The ``iomap_page_mkwrite`` function handles a write fault to a folio in the pagecache. ``IOMAP_WRITE | IOMAP_FAULT`` will be passed as the ``flags`` argument -to ``->iomap_begin``. +to ``begin``. Callers commonly take the mmap ``invalidate_lock`` in shared or exclusive mode before calling this function. @@ -256,7 +262,7 @@ such `reservations `_ because writeback will not consume the reservation. The ``iomap_write_delalloc_release`` can be called from a -``->iomap_end`` function to find all the clean areas of the folios +``end`` function to find all the clean areas of the folios caching a fresh (``IOMAP_F_NEW``) delalloc mapping. It takes the ``invalidate_lock``. @@ -274,7 +280,7 @@ Filesystems can call ``iomap_zero_range`` to perform zeroing of the pagecache for non-truncation file operations that are not aligned to the fsblock size. ``IOMAP_ZERO`` will be passed as the ``flags`` argument to -``->iomap_begin``. +``begin``. Callers typically hold ``i_rwsem`` and ``invalidate_lock`` in exclusive mode before calling this function. @@ -285,7 +291,7 @@ Filesystems can call ``iomap_file_unshare`` to force a file sharing storage with another file to preemptively copy the shared data to newly allocate storage. ``IOMAP_WRITE | IOMAP_UNSHARE`` will be passed as the ``flags`` argument -to ``->iomap_begin``. +to ``begin``. Callers typically hold ``i_rwsem`` and ``invalidate_lock`` in exclusive mode before calling this function. @@ -298,7 +304,7 @@ operation. ``truncate_setsize`` or ``truncate_pagecache`` will take care of everything after the EOF block. ``IOMAP_ZERO`` will be passed as the ``flags`` argument to -``->iomap_begin``. +``begin``. Callers typically hold ``i_rwsem`` and ``invalidate_lock`` in exclusive mode before calling this function. @@ -341,8 +347,8 @@ The fields are as follows: though it will `reuse mappings `_ for runs of contiguous dirty fsblocks within a folio. - Do not return ``IOMAP_INLINE`` mappings here; the ``->iomap_end`` - function must deal with persisting written data. + Do not return ``IOMAP_INLINE`` mappings here; the ``end`` function must + deal with persisting written data. Do not return ``IOMAP_DELALLOC`` mappings here; iomap currently requires mapping to allocated space. Filesystems can skip a potentially expensive mapping lookup if the @@ -428,7 +434,7 @@ writes for files. .. code-block:: c ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, + iomap_iter_next_fn iomap_next, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before); @@ -511,7 +517,7 @@ Return Values * ``-ENOTBLK``: Fall back to buffered I/O. iomap itself will return this value if it cannot invalidate the page cache before issuing the I/O to storage. - The ``->iomap_begin`` or ``->iomap_end`` functions may also return + The ``begin`` or ``end`` functions may also return this value. * ``-EIOCBQUEUED``: The asynchronous direct I/O request has been @@ -526,7 +532,7 @@ A direct I/O read initiates a read I/O from the storage device to the caller's buffer. Dirty parts of the pagecache are flushed to storage before initiating the read io. -The ``flags`` value for ``->iomap_begin`` will be ``IOMAP_DIRECT`` with +The ``flags`` value for ``begin`` will be ``IOMAP_DIRECT`` with any combination of the following enhancements: * ``IOMAP_NOWAIT``, as defined previously. @@ -542,7 +548,7 @@ caller's buffer. Dirty parts of the pagecache are flushed to storage before initiating the write io. The pagecache is invalidated both before and after the write io. -The ``flags`` value for ``->iomap_begin`` will be ``IOMAP_DIRECT | +The ``flags`` value for ``begin`` will be ``IOMAP_DIRECT | IOMAP_WRITE`` with any combination of the following enhancements: * ``IOMAP_NOWAIT``, as defined previously. @@ -644,7 +650,7 @@ fsdax Reads A fsdax read performs a memcpy from storage device to the caller's buffer. -The ``flags`` value for ``->iomap_begin`` will be ``IOMAP_DAX`` with any +The ``flags`` value for ``begin`` will be ``IOMAP_DAX`` with any combination of the following enhancements: * ``IOMAP_NOWAIT``, as defined previously. @@ -657,7 +663,7 @@ fsdax Writes A fsdax write initiates a memcpy to the storage device from the caller's buffer. -The ``flags`` value for ``->iomap_begin`` will be ``IOMAP_DAX | +The ``flags`` value for ``begin`` will be ``IOMAP_DAX | IOMAP_WRITE`` with any combination of the following enhancements: * ``IOMAP_NOWAIT``, as defined previously. @@ -680,9 +686,9 @@ fsdax mmap Faults The ``dax_iomap_fault`` function handles read and write faults to fsdax storage. For a read fault, ``IOMAP_DAX | IOMAP_FAULT`` will be passed as the -``flags`` argument to ``->iomap_begin``. +``flags`` argument to ``begin``. For a write fault, ``IOMAP_DAX | IOMAP_FAULT | IOMAP_WRITE`` will be -passed as the ``flags`` argument to ``->iomap_begin``. +passed as the ``flags`` argument to ``begin``. Callers commonly hold the same locks as they do to call their iomap pagecache counterparts. @@ -692,8 +698,8 @@ fsdax Truncation, fallocate, and Unsharing For fsdax files, the following functions are provided to replace their iomap pagecache I/O counterparts. -The ``flags`` argument to ``->iomap_begin`` are the same as the -pagecache counterparts, with ``IOMAP_DAX`` added. +The ``flags`` argument to ``begin`` are the same as the pagecache counterparts, +with ``IOMAP_DAX`` added. * ``dax_file_unshare`` * ``dax_zero_range`` @@ -719,8 +725,7 @@ SEEK_DATA The ``iomap_seek_data`` function implements the SEEK_DATA "whence" value for llseek. -``IOMAP_REPORT`` will be passed as the ``flags`` argument to -``->iomap_begin``. +``IOMAP_REPORT`` will be passed as the ``flags`` argument to ``begin``. For unwritten mappings, the pagecache will be searched. Regions of the pagecache with a folio mapped and uptodate fsblocks @@ -734,8 +739,7 @@ SEEK_HOLE The ``iomap_seek_hole`` function implements the SEEK_HOLE "whence" value for llseek. -``IOMAP_REPORT`` will be passed as the ``flags`` argument to -``->iomap_begin``. +``IOMAP_REPORT`` will be passed as the ``flags`` argument to ``begin``. For unwritten mappings, the pagecache will be searched. Regions of the pagecache with no folio mapped, or a !uptodate fsblock @@ -750,8 +754,7 @@ Swap File Activation The ``iomap_swapfile_activate`` function finds all the base-page aligned regions in a file and sets them up as swap space. The file will be ``fsync()``'d before activation. -``IOMAP_REPORT`` will be passed as the ``flags`` argument to -``->iomap_begin``. +``IOMAP_REPORT`` will be passed as the ``flags`` argument to ``begin``. All mappings must be mapped or unwritten; cannot be dirty or shared, and cannot span multiple block devices. Callers must hold ``i_rwsem`` in exclusive mode; this is already @@ -767,8 +770,7 @@ FS_IOC_FIEMAP The ``iomap_fiemap`` function exports file extent mappings to userspace in the format specified by the ``FS_IOC_FIEMAP`` ioctl. -``IOMAP_REPORT`` will be passed as the ``flags`` argument to -``->iomap_begin``. +``IOMAP_REPORT`` will be passed as the ``flags`` argument to ``begin``. Callers commonly hold ``i_rwsem`` in shared mode before calling this function. diff --git a/Documentation/filesystems/iomap/porting.rst b/Documentation/filesystems/iomap/porting.rst index 3d49a32c0fff..b995f426808a 100644 --- a/Documentation/filesystems/iomap/porting.rst +++ b/Documentation/filesystems/iomap/porting.rst @@ -50,8 +50,14 @@ Build the kernel, run fstests with the ``-g all`` option across a wide variety of your filesystem's supported configurations to build a baseline of which tests pass and which ones fail. -The recommended approach is first to implement ``->iomap_begin`` (and -``->iomap_end`` if necessary) to allow iomap to obtain a read-only +Every iomap operation is driven by an ``iomap_iter_next_fn`` callback. +Filesystems normally do not write one by hand: implement ``begin`` +(and ``end`` if necessary) and generate the callback with the +``DEFINE_IOMAP_ITER_NEXT()`` macro, or ``DEFINE_IOMAP_ITER_NEXT_END()`` if an +``end`` callback is needed. + +The recommended approach is first to implement ``begin`` (and +``end`` if necessary) to allow iomap to obtain a read-only mapping of a file range. In most cases, this is a relatively trivial conversion of the existing ``get_block()`` function for read-only mappings. @@ -62,7 +68,7 @@ If FIEMAP is returning the correct information, it's a good sign that other read-only mapping operations will do the right thing. Next, modify the filesystem's ``get_block(create = false)`` -implementation to use the new ``->iomap_begin`` implementation to map +implementation to use the new ``begin`` implementation to map file space for selected read operations. Hide behind a debugging knob the ability to switch on the iomap mapping functions for selected call paths. @@ -82,14 +88,14 @@ I/O path because of bufferheads. The buffered read I/O paths doesn't need to be converted yet, though the direct I/O read path should be converted in this phase. -At this point, you should look over your ``->iomap_begin`` function. +At this point, you should look over your ``begin`` function. If it switches between large blocks of code based on dispatching of the ``flags`` argument, you should consider breaking it up into per-operation iomap ops with smaller, more cohesive functions. XFS is a good example of this. The next thing to do is implement ``get_blocks(create == true)`` -functionality in the ``->iomap_begin``/``->iomap_end`` methods. +functionality in the ``begin``/``end`` methods. It is strongly recommended to create separate mapping functions and iomap ops for write operations. Then convert the direct I/O write path to iomap, and start running fsx From 807c1d7125a2b3ca5705a803aa2e19f9d53d1ea3 Mon Sep 17 00:00:00 2001 From: Abbott Liu Date: Wed, 30 Sep 2026 16:46:32 +0800 Subject: [PATCH 09/28] initrd: flush deferred fput() before mounting root Booting an old-style initrd, i.e. a filesystem image instead of a cpio archive, with "root=/dev/ram rootfstype=squashfs" can fail to mount the root filesystem: RAMDISK: squashfs filesystem found at block 0 RAMDISK: Loading 68449KiB [1 disk] into ram disk... done. No filesystem could mount root, tried: squashfs Kernel panic - not syncing: VFS: Unable to mount root fs on "/dev/ram" or unknown-block(1,0) rd_load_image() copies the image into /dev/ram with kernel_write() and then drops the file with fput(). The data only reaches the ramdisk once the final __fput() releases the block device and bdev_release() calls sync_blockdev(). But that __fput() is deferred. Since commit 343f4c49f243 ("kthread: Don't allocate kthread_struct for init and umh") init is a user mode task, so the __fput() is queued as task work and only runs once init returns to userspace, i.e. long after mount_root(). Before that it ran as delayed fput work at least a jiffy later, which doesn't help either. Whether the mount survives this depends on the block size. Squashfs calls sb_min_blocksize() first and set_blocksize() syncs and drops the page cache when the requested block size differs from the current one. That hides the problem for most configurations. With CONFIG_SQUASHFS_4K_DEVBLK_SIZE=y and 4K pages both block sizes are 4K because set_init_blocksize() gives a page-aligned ramdisk a PAGE_SIZE block size on the first open. So set_blocksize() does nothing and squashfs reads the superblock with bios that bypass the page cache. brd has no page for that range yet and returns zeroes, so the magic check fails and the mount returns -EINVAL. With rootfstype= naming a single filesystem there is nothing left to retry and mount_root_generic() panics. Fixing this in squashfs would be the wrong end. Most block-backed filesystems read their superblock or their file data with bios that bypass the block device page cache, so they are exposed to the same race, just less loudly. So flush the deferred fput() in rd_load_image() right after dropping the ramdisk file, the same way mount_root_generic() already does before it retries with another filesystem type. All deferred closes at this point belong to files the boot itself created. Verified on arm64 QEMU with an openEuler Embedded rootfs image packed as squashfs. Fixes: 93e72b3c612a ("squashfs: migrate from ll_rw_block usage to BIO") Signed-off-by: Abbott Liu Link: https://patch.msgid.link/20260930084632.2965573-1-liuwenliang@huawei.com Signed-off-by: Christian Brauner (Amutable) --- init/do_mounts_rd.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/init/do_mounts_rd.c b/init/do_mounts_rd.c index 48bfab2fc62f..448a734a0f62 100644 --- a/init/do_mounts_rd.c +++ b/init/do_mounts_rd.c @@ -258,6 +258,11 @@ done: fput(in_file); noclose_input: fput(out_file); + /* + * The image only reaches the ramdisk when the deferred __fput() + * syncs the block device. Flush it before mount_root() reads it. + */ + init_flush_fput(); out: kfree(buf); init_unlink("/dev/ram"); From afa735786a89bde4959db52f20e3579de54ec740 Mon Sep 17 00:00:00 2001 From: Chris Mason Date: Sun, 1 Nov 2026 13:34:40 +0100 Subject: [PATCH 10/28] backing-file: pass RWF_NOAPPEND to synchronous writes pwritev2() with RWF_NOAPPEND ignores the offset and appends the data on an overlayfs or FUSE passthrough file that was opened with O_APPEND, and the call still returns success. The flag must make the write go to the given offset even though the file is in append mode. Any user who can write to the file is affected, with the default mount options. overlayfs and FUSE passthrough forward the reads and writes on their files to a backing file on an underlying filesystem, through the helpers in fs/backing-file.c. The flag works for the file that the user opened: kiocb_set_rw_flags() clears IOCB_APPEND in the write's kiocb. It also copies RWF_NOAPPEND into ki_flags, as it does with every RWF_ flag. overlayfs and FUSE passthrough then pass the write on to the backing file. They opened it with the open flags of the user's file, so it is in append mode too. For a synchronous write, do_backing_file_write_iter() calls vfs_iter_write(), which sets up a new kiocb for the backing file. That kiocb starts out with IOCB_APPEND set. vfs_iter_write() is only given the flags of the user's kiocb that are in BACKING_IOCB_MASK, and RWF_NOAPPEND is not one of them. So nothing clears IOCB_APPEND again, and the data goes to the end of the file while the call returns success. An asynchronous write reuses the flags of the user's kiocb and is not affected. Reproduced by calling pwritev2() with RWF_NOAPPEND at offset 0, on an overlayfs file and on a FUSE passthrough file opened with O_APPEND: the data was at the end of the file afterwards. Fix this by adding RWF_NOAPPEND to BACKING_IOCB_MASK. Fixes: 73fa7547c70b ("vfs: add RWF_NOAPPEND flag for pwritev2") Cc: stable@vger.kernel.org Signed-off-by: Chris Mason Signed-off-by: Amir Goldstein Link: https://patch.msgid.link/20261101123440.3184069-1-amir73il@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/backing-file.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/backing-file.c b/fs/backing-file.c index 5780f60662d0..c601b27f0fd3 100644 --- a/fs/backing-file.c +++ b/fs/backing-file.c @@ -115,8 +115,13 @@ struct backing_aio { static struct kmem_cache *backing_aio_cachep; +/* + * RWF_NOAPPEND stays set in ki_flags. Pass it on, because the backing file is + * in append mode as well and a new kiocb for it starts with IOCB_APPEND. + */ #define BACKING_IOCB_MASK \ - (IOCB_NOWAIT | IOCB_HIPRI | IOCB_DSYNC | IOCB_SYNC | IOCB_APPEND) + (IOCB_NOWAIT | IOCB_HIPRI | IOCB_DSYNC | IOCB_SYNC | IOCB_APPEND | \ + (__force int) RWF_NOAPPEND) static rwf_t iocb_to_rw_flags(int flags) { From 94defd41d80f10bbe883c0c8d1b28d1665b6b153 Mon Sep 17 00:00:00 2001 From: Shashank Mohan Jain Date: Sun, 27 Sep 2026 10:47:25 +0530 Subject: [PATCH 11/28] errseq: don't let errseq_check_and_advance() hide later errors errseq_check_and_advance() reads the errseq_t, sets ERRSEQ_SEEN with a cmpxchg() and then advances *since to the value it computed, ignoring whether the cmpxchg() succeeded. When it fails because errseq_set() recorded a different error in the meantime, *since ends up holding a value that was never stored in the errseq_t. errseq_set() only bumps the counter when ERRSEQ_SEEN is set. So as long as nobody has seen the new error, recording the original errno again recreates exactly the old value, and once another subscriber marks it seen, it is equal to the stale *since. The next check through that cursor then reports nothing, although -ENOSPC was recorded after the value that check reported, and -EIO was recorded again after it returned: cursor f writeback cursor g -------- --------- -------- set(-EIO) check_and_advance(&f) old = [c, EIO] set(-ENOSPC) -> [c, ENOSPC] cmpxchg() fails f = [c, EIO, SEEN] returns -EIO set(-EIO) -> [c, EIO] (no bump: unseen) check_and_advance(&g) -> [c, EIO, SEEN] returns -EIO check_and_advance(&f) [c, EIO, SEEN] == f returns 0 <- -ENOSPC and the second -EIO are lost For file->f_wb_err this means that fsync() on one descriptor can return 0 although writeback failed after the previous fsync() on that descriptor returned, when writeback errors race with fsync() on another descriptor of the same file. The same applies to syncfs() through sb->s_wb_err, which every file on the filesystem shares, and to the ext4 and jbd2 cursors on the block device mapping. The kernel-doc promises "Negative errno if one has been stored, or 0 if no new error has occurred". Retry with the value found when the cmpxchg() fails, so that *since is only ever advanced to a value that was actually stored with ERRSEQ_SEEN set. Every later errseq_set() then has to bump the counter, and the error is reported. The retry loop terminates because each iteration needs a concurrent update, like the loop in errseq_set(). A TLA+ model of errseq_set() and errseq_check_and_advance(), checked with the TLC model checker, finds the lost error with the current code; with this change TLC checks that an error recorded after a check returned is always reported by the next check (exhaustively for two subscribers with three checks each, and one writer recording four errors or two writers recording two each). The KUnit race test added in the next patch misses the error in 1-6% of 2 million rounds on 4-CPU UML before this change (depending on host load), and in none after it. Fixes: 84cbadadc6ea ("lib: add errseq_t type and infrastructure for handling it") Cc: stable@vger.kernel.org Signed-off-by: Shashank Mohan Jain Link: https://patch.msgid.link/20260927051726.71337-2-jain.sm@gmail.com Reviewed-by: Jan Kara Reviewed-by: Jeff Layton Signed-off-by: Christian Brauner (Amutable) --- lib/errseq.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/lib/errseq.c b/lib/errseq.c index 13a2581c5a87..ea4ff9cc2166 100644 --- a/lib/errseq.c +++ b/lib/errseq.c @@ -162,7 +162,8 @@ EXPORT_SYMBOL(errseq_check); * points to. If it does, then just return 0. * * If it doesn't, then the value has changed. Set the "seen" flag, and try to - * swap it into place as the new eseq value. Then, set that value as the new + * swap it into place as the new eseq value. If the swap fails because the + * value changed, retry with the new value. Then, set that value as the new * "since" value, and return whatever the error portion is set to. * * Note that no locking is provided here for concurrent updates to the "since" @@ -184,24 +185,30 @@ int errseq_check_and_advance(errseq_t *eseq, errseq_t *since) * to take the lock that protects the "since" value. */ old = READ_ONCE(*eseq); - if (old != *since) { + while (old != *since) { /* * Set the flag and try to swap it into place if it has * changed. * - * We don't care about the outcome of the swap here. If the - * swap doesn't occur, then it has either been updated by a - * writer who is altering the value in some way (updating - * counter or resetting the error), or another reader who is - * just setting the "seen" flag. Either outcome is OK, and we - * can advance "since" and return an error based on what we - * have. + * If the swap fails, a writer or another reader changed the + * value under us, so retry with the new value. "since" must + * only ever be set to a value that was stored with the SEEN + * flag: errseq_set() does not bump the counter while the + * flag is clear, so an unstored value could come back later + * and hide the errors that were recorded in the meantime. */ new = old | ERRSEQ_SEEN; - if (new != old) - cmpxchg(eseq, old, new); + if (new != old) { + errseq_t cur = cmpxchg(eseq, old, new); + + if (cur != old) { + old = cur; + continue; + } + } *since = new; err = -(new & ERRNO_MASK); + break; } return err; } From 36ac3a0dedf784cddb9f5161a64e6a50cf4cd0ad Mon Sep 17 00:00:00 2001 From: Shashank Mohan Jain Date: Sun, 27 Sep 2026 10:47:26 +0530 Subject: [PATCH 12/28] lib/tests: errseq: add a concurrent check_and_advance test Add a test case that races errseq_check_and_advance() on one CPU against errseq_set() of a different error on another CPU, then records the first error again and lets a second cursor consume it. The second -EIO is recorded after the first cursor's check returned, so that cursor's next check must report an error, whichever way the race with -ENOSPC went. Without the previous commit 1-6% of 2 million rounds lose the error on UML with 4 CPUs (seccomp=on ncpus=4); with it none do. The case is skipped on a single CPU and stops after 5 seconds. Signed-off-by: Shashank Mohan Jain Link: https://patch.msgid.link/20260927051726.71337-3-jain.sm@gmail.com Reviewed-by: Jan Kara Reviewed-by: Jeff Layton Signed-off-by: Christian Brauner (Amutable) --- lib/tests/errseq_kunit.c | 110 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 5 deletions(-) diff --git a/lib/tests/errseq_kunit.c b/lib/tests/errseq_kunit.c index 8f39ebc4a248..5ebc411ea749 100644 --- a/lib/tests/errseq_kunit.c +++ b/lib/tests/errseq_kunit.c @@ -2,21 +2,26 @@ /* * KUnit tests for the errseq_t error-tracking infrastructure. * - * These exercise the documented single-threaded semantics of the errseq - * API (see Documentation/core-api/errseq.rst and lib/errseq.c): error + * Most cases exercise the documented single-threaded semantics of the + * errseq API (see Documentation/core-api/errseq.rst and lib/errseq.c): error * recording and overwriting, the "seen" handoff between errseq_sample() * and errseq_check_and_advance(), and the re-reporting of an error that * is recorded again after it has been seen. * - * The lockless properties of errseq_t under concurrent updates are - * outside the scope of these deterministic tests, as is the WARN path - * for invalid error values. + * The WARN path for invalid error values is not tested. The last case + * races errseq_set() against errseq_check_and_advance() on another CPU. */ #include +#include +#include #include #include #include +#include +#include +#include +#include /* * A zeroed errseq_t is the "no error has ever occurred" epoch: it @@ -209,6 +214,100 @@ static void errseq_test_two_subscribers_independent(struct kunit *test) KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_b), 0); } +/* + * errseq_check_and_advance() racing with errseq_set() on another CPU. + * + * Each round, cursor @f checks an unseen -EIO while another CPU records + * -ENOSPC. Afterwards -EIO is recorded again and cursor @g checks. The + * second -EIO is recorded after @f's check returned, so @f's next check + * must report an error, whichever way the race with -ENOSPC went. + */ +struct errseq_race { + errseq_t eseq; + atomic_t go; /* round the writer should run */ + atomic_t done; /* last round the writer completed */ + unsigned int wdelay; +}; + +static void errseq_race_spin(unsigned int n) +{ + while (n--) + cpu_relax(); +} + +static int errseq_race_writer(void *data) +{ + struct errseq_race *r = data; + int seen = 0; + + while (!kthread_should_stop()) { + int round = atomic_read_acquire(&r->go); + + if (round == seen) { + cond_resched(); + continue; + } + seen = round; + errseq_race_spin(READ_ONCE(r->wdelay)); + errseq_set(&r->eseq, -ENOSPC); + atomic_set_release(&r->done, round); + } + return 0; +} + +#define ERRSEQ_RACE_ROUNDS 2000000 +#define ERRSEQ_RACE_SECONDS 5 + +static void errseq_test_check_and_advance_race(struct kunit *test) +{ + struct errseq_race *r; + struct task_struct *writer; + unsigned long deadline; + int round, unreported = 0, missed = 0; + + if (num_online_cpus() < 2) + kunit_skip(test, "needs at least two CPUs"); + + r = kunit_kzalloc(test, sizeof(*r), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r); + + writer = kthread_run(errseq_race_writer, r, "errseq_race"); + KUNIT_ASSERT_FALSE(test, IS_ERR(writer)); + + deadline = jiffies + ERRSEQ_RACE_SECONDS * HZ; + for (round = 1; round <= ERRSEQ_RACE_ROUNDS; round++) { + errseq_t f = 0, g = 0; + + WRITE_ONCE(r->eseq, 0); + errseq_set(&r->eseq, -EIO); + WRITE_ONCE(r->wdelay, get_random_u32_below(64)); + atomic_set_release(&r->go, round); + + errseq_race_spin(get_random_u32_below(64)); + if (!errseq_check_and_advance(&r->eseq, &f)) + unreported++; + + while (atomic_read_acquire(&r->done) != round) + cond_resched(); + + /* Recorded after f's check returned: f must hear about it. */ + errseq_set(&r->eseq, -EIO); + errseq_check_and_advance(&r->eseq, &g); + if (!errseq_check_and_advance(&r->eseq, &f)) + missed++; + + if (time_after(jiffies, deadline)) + break; + if (!(round & 1023)) + cond_resched(); + } + kthread_stop(writer); + + kunit_info(test, "%d rounds, %d missed errors\n", round - 1, missed); + KUNIT_EXPECT_EQ(test, unreported, 0); + KUNIT_EXPECT_EQ(test, missed, 0); +} + static struct kunit_case errseq_test_cases[] = { KUNIT_CASE(errseq_test_zero_epoch_reports_no_error), KUNIT_CASE(errseq_test_set_records_error), @@ -223,6 +322,7 @@ static struct kunit_case errseq_test_cases[] = { KUNIT_CASE(errseq_test_repeat_error_visible_to_all_cursors), KUNIT_CASE(errseq_test_advance_stable_when_unchanged), KUNIT_CASE(errseq_test_two_subscribers_independent), + KUNIT_CASE_SLOW(errseq_test_check_and_advance_race), {} }; From 965e5e451146a19f69523fbcbc73d07e401a3775 Mon Sep 17 00:00:00 2001 From: Andrea Parri Date: Mon, 26 Oct 2026 19:06:18 +0100 Subject: [PATCH 13/28] iov_iter: take an unsigned long segment count in import_iovec() readv(), writev() and their positional variants, process_vm_readv(), process_vm_writev(), vmsplice(), process_madvise(), vectored aio and the btrfs encoded I/O ioctls all take a full-width segment count, and all of them hand it to import_iovec() or __import_iovec(). Those take an unsigned int count, so on 64-bit kernels the upper 32 bits are dropped before iovec_from_user() checks the count against UIO_MAXIOV. A count of 0x100000001 is therefore imported as one segment: writev() on a pipe writes a byte and readv() reads it back, instead of both failing with EINVAL. A count of 0x100000000 returns 0 as an empty vector. import_iovec() was added with an unsigned int count, but the helper it wrapped, rw_copy_check_uvector(), took an unsigned long, as does iovec_from_user() today. Until v4.1, readv(), writev(), preadv(), pwritev(), process_vm_readv(), process_vm_writev() and vectored aio therefore failed these counts with EINVAL, as did vmsplice() until v4.18. Widen nr_segs in both import helpers so the UIO_MAXIOV check sees the count the caller passed. Callers with 32-bit counts are unaffected. AI assistance was used to find the bug, develop the fix, and run the reproducers. Fixes: bc917be81059 ("saner iov_iter initialization primitives") Signed-off-by: Andrea Parri Link: https://patch.msgid.link/20261026180619.4147910-1-parri.andrea@gmail.com Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- include/linux/uio.h | 8 ++++---- lib/iov_iter.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/uio.h b/include/linux/uio.h index 638c1116e912..76e4103b9ec4 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -373,11 +373,11 @@ struct iovec *iovec_from_user(const struct iovec __user *uvector, unsigned long nr_segs, unsigned long fast_segs, struct iovec *fast_iov, bool compat); ssize_t import_iovec(int type, const struct iovec __user *uvec, - unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, - struct iov_iter *i); + unsigned long nr_segs, unsigned int fast_segs, + struct iovec **iovp, struct iov_iter *i); ssize_t __import_iovec(int type, const struct iovec __user *uvec, - unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, - struct iov_iter *i, bool compat); + unsigned long nr_segs, unsigned int fast_segs, + struct iovec **iovp, struct iov_iter *i, bool compat); int import_ubuf(int type, void __user *buf, size_t len, struct iov_iter *i); static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction, diff --git a/lib/iov_iter.c b/lib/iov_iter.c index fa4fe2e92383..af005ef5cb82 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1382,8 +1382,8 @@ static ssize_t __import_iovec_ubuf(int type, const struct iovec __user *uvec, } ssize_t __import_iovec(int type, const struct iovec __user *uvec, - unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, - struct iov_iter *i, bool compat) + unsigned long nr_segs, unsigned int fast_segs, + struct iovec **iovp, struct iov_iter *i, bool compat) { ssize_t total_len = 0; unsigned long seg; @@ -1454,7 +1454,7 @@ ssize_t __import_iovec(int type, const struct iovec __user *uvec, * Return: Negative error code on error, bytes imported on success */ ssize_t import_iovec(int type, const struct iovec __user *uvec, - unsigned nr_segs, unsigned fast_segs, + unsigned long nr_segs, unsigned int fast_segs, struct iovec **iovp, struct iov_iter *i) { return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i, From 55a0c91000646b008261e3b1e15d1e774f3026e4 Mon Sep 17 00:00:00 2001 From: Hungyu Lin Date: Sun, 1 Nov 2026 11:38:15 +0000 Subject: [PATCH 14/28] vfs: preserve subsecond timestamps at filesystem time limits timestamp_truncate() clears tv_nsec whenever the clamped seconds equal either filesystem time limit. This also drops the fractional part of timestamps whose seconds were already within the supported range. For example, setting a JFS mtime to 0.123456789 produces 0.000000000, although the filesystem can store the requested value. Utilities such as touch and cp -p consequently lose valid subsecond timestamps when preserving these endpoint values. Timestamps strictly between the filesystem limits are unaffected. Only clear tv_nsec when clamping actually changes tv_sec. Continue to truncate in-range nanoseconds to the filesystem's timestamp granularity, including when the seconds equal either endpoint. Ben Hutchings suggested comparing clamped and original seconds in the 2019 utimes review. That caller was switched to timestamp_truncate(), but the helper retained its endpoint equality check. Link: https://lore.kernel.org/lkml/12c095e595836a7ff7f2c7b2a32cb5544dd29b55.camel@codethink.co.uk/ Fixes: 50e17c000c46 ("vfs: Add timestamp_truncate() api") Signed-off-by: Hungyu Lin Link: https://patch.msgid.link/20261101113815.52911-1-dennylin0707@gmail.com Reviewed-by: Jeff Layton Signed-off-by: Christian Brauner (Amutable) --- fs/inode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/inode.c b/fs/inode.c index f63535ee3629..9cbcb0de7dfc 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2858,9 +2858,10 @@ struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode) { struct super_block *sb = inode->i_sb; unsigned int gran = sb->s_time_gran; + time64_t seconds = t.tv_sec; t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max); - if (unlikely(t.tv_sec == sb->s_time_max || t.tv_sec == sb->s_time_min)) + if (unlikely(t.tv_sec != seconds)) t.tv_nsec = 0; /* Avoid division in the common cases 1 ns and 1 s. */ From 925d238324c79bcb75ad0add1d1b9d31d5f48115 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 3 Nov 2026 07:04:00 -0500 Subject: [PATCH 15/28] statx: don't leak STATX_ATTR_CHANGE_MONOTONIC in stx_attributes_mask STATX_ATTR_CHANGE_MONOTONIC is kernel-internal. cp_statx() strips it from stx_attributes but copies attributes_mask verbatim, so it leaks to userland on filesystems that advertise it (nfs, ceph), e.g. stx_attributes_mask = 0x8000000000303000 on NFS. Add a new mask constant and apply it to both fields. Fixes: a1175d6b1bda ("vfs: plumb i_version handling into struct kstat") Reviewed-by: Jan Kara Signed-off-by: Jeff Layton Link: https://patch.msgid.link/20261103-statx-v2-1-8b09aa8118ab@kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/stat.c | 6 +++--- include/linux/stat.h | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fs/stat.c b/fs/stat.c index a2e63c6c45b7..34d786c1de3e 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -710,8 +710,7 @@ cp_statx(const struct kstat *stat, struct statx __user *buffer) /* STATX_CHANGE_COOKIE is kernel-only for now */ tmp.stx_mask = stat->result_mask & ~STATX_CHANGE_COOKIE; tmp.stx_blksize = stat->blksize; - /* STATX_ATTR_CHANGE_MONOTONIC is kernel-only for now */ - tmp.stx_attributes = stat->attributes & ~STATX_ATTR_CHANGE_MONOTONIC; + tmp.stx_attributes = stat->attributes & ~STATX_ATTR_INTERNAL_MASK; tmp.stx_nlink = stat->nlink; tmp.stx_uid = from_kuid_munged(current_user_ns(), stat->uid); tmp.stx_gid = from_kgid_munged(current_user_ns(), stat->gid); @@ -719,7 +718,8 @@ cp_statx(const struct kstat *stat, struct statx __user *buffer) tmp.stx_ino = stat->ino; tmp.stx_size = stat->size; tmp.stx_blocks = stat->blocks; - tmp.stx_attributes_mask = stat->attributes_mask; + tmp.stx_attributes_mask = stat->attributes_mask & + ~STATX_ATTR_INTERNAL_MASK; tmp.stx_atime.tv_sec = stat->atime.tv_sec; tmp.stx_atime.tv_nsec = stat->atime.tv_nsec; tmp.stx_btime.tv_sec = stat->btime.tv_sec; diff --git a/include/linux/stat.h b/include/linux/stat.h index 9c5709132862..4f0a4af7a067 100644 --- a/include/linux/stat.h +++ b/include/linux/stat.h @@ -69,4 +69,7 @@ struct kstat { /* file attribute values */ #define STATX_ATTR_CHANGE_MONOTONIC 0x8000000000000000ULL /* version monotonically increases */ +/* mask of STATX_ATTR_* bits that is private to the kernel */ +#define STATX_ATTR_INTERNAL_MASK (STATX_ATTR_CHANGE_MONOTONIC) + #endif From 9f4413e522faed76f4cf610098b2369eb3ba1f1c Mon Sep 17 00:00:00 2001 From: Mani Tofigh Date: Tue, 3 Nov 2026 13:19:01 -0500 Subject: [PATCH 16/28] eventpoll: wake the next waiter when a woken waiter is killed Processes sharing an epoll instance can sleep forever while an event is ready. An event wakes only one waiter, which then wakes the next one in ep_done_scan() if events are still ready. Since commit c257a340ede0 ("fs, epoll: short circuit fetching events if thread has been killed"), ep_send_events() returns before the scan when the waiter has a fatal signal. If the woken waiter is killed before it runs, nobody wakes the next one: Task A Task B Task C epoll_wait(ep, -1) epoll_wait(ep, -1) write(efd) /* wakes A only */ kill(A, SIGKILL) ep_send_events() return -EINTR /* skips ep_done_scan() */ /* still asleep */ B then sleeps until another file on the instance becomes ready, if ever. This can hit eg a pre-fork server whose worker gets OOM killed. Commit 7fab29e35630 ("fs/epoll: restore waking from ep_done_scan()") fixed the same kind of hang when the wakeup in ep_done_scan() was removed. Keep the early return so that a dying task still doesn't take events nobody will see, but wake the next waiter there if events are ready, as ep_done_scan() would have. With the fix, B gets the event. This bug was identified and confirmed with the help of a deterministic hypervisor currently under development. The C reproducer below however hits it within about 2s. Fixes: c257a340ede0 ("fs, epoll: short circuit fetching events if thread has been killed") Cc: stable@vger.kernel.org Signed-off-by: Mani Tofigh Link: https://patch.msgid.link/20261103-epoll-killed-waiter-v1-1-36e08c16b8a2@columbia.edu Signed-off-by: Christian Brauner (Amutable) --- fs/eventpoll.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 7a32c70bf2ad..b959af04772c 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -2144,10 +2144,17 @@ static int ep_send_events(struct eventpoll *ep, /* * Always short-circuit for fatal signals to allow threads to make a * timely exit without the chance of finding more events available and - * fetching repeatedly. + * fetching repeatedly. The wakeup that got us here may have been the + * only one for the events on ->rdllist, so pass it on to the next + * waiter, as ep_done_scan() would have. */ - if (fatal_signal_pending(current)) + if (fatal_signal_pending(current)) { + spin_lock_irq(&ep->lock); + if (!list_empty(&ep->rdllist)) + wake_up(&ep->wq); + spin_unlock_irq(&ep->lock); return -EINTR; + } init_poll_funcptr(&pt, NULL); From d87631c337258a90e10ffa76eb76bb5b5d559a40 Mon Sep 17 00:00:00 2001 From: Avi Weiss Date: Sat, 31 Oct 2026 09:22:09 +0200 Subject: [PATCH 17/28] fs: clamp FIONREAD result to int range FIONREAD returns the difference between the file size and the current position in an int. When the difference is outside the int range, the result wraps, so a file with 4 GiB remaining can be reported as empty. Clamp the result to the signed int range before returning it to userspace. Preserve negative counts for file positions past EOF. Signed-off-by: Avi Weiss Link: https://patch.msgid.link/20261031072208.3096100-2-thnkslprpt@gmail.com Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/ioctl.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/ioctl.c b/fs/ioctl.c index 1c152c2b1b67..ecc2a4cfe610 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -550,8 +551,8 @@ static int do_vfs_ioctl(struct file *filp, unsigned int fd, if (!S_ISREG(inode->i_mode) || IS_ANON_FILE(inode)) return vfs_ioctl(filp, cmd, arg); - return put_user(i_size_read(inode) - filp->f_pos, - (int __user *)argp); + return put_user(clamp_t(loff_t, i_size_read(inode) - filp->f_pos, + INT_MIN, INT_MAX), (int __user *)argp); case FS_IOC_GETFLAGS: return ioctl_getflags(filp, argp); From ee8323fdc827c580425644db605a4be776cace57 Mon Sep 17 00:00:00 2001 From: Chen Linxuan Date: Tue, 27 Oct 2026 16:22:15 +0800 Subject: [PATCH 18/28] pidfd: restrict task access ioctls to the caller's pid namespace pidfds can cross pid namespace boundaries: a task may obtain one for a process outside of its own pid namespace hierarchy through SO_PASSPIDFD or SO_PEERPIDFD, via SCM_RIGHTS, or through file descriptor inheritance. The namespace ioctls gate access with a ptrace check against the target's credentials only. That check does not account for the pid namespace relationship between the caller and the target, so a task inside a pid namespace can use a pidfd obtained across that boundary to acquire namespace file descriptors of a task outside of its hierarchy. procfs exposes the equivalent information only for tasks visible in the reader's pid namespace, and PIDFD_GET_INFO already answers with -EREMOTE for targets outside the caller's hierarchy. Extend that policy to the namespace ioctls: fail with -EREMOTE unless the target is visible in the caller's active pid namespace. Callers in ancestor pid namespaces are unaffected. Unlike PIDFD_GET_INFO, the visibility check is performed after the task lookup here, so reaped targets outside of the caller's pid namespace hierarchy answer -ESRCH instead of -EREMOTE. Link: https://lore.kernel.org/all/CAG48ez1T5-tUHVK_iHsgUtWa-inVdrZxA3E-ZCW0iENJvasKfg@mail.gmail.com Link: https://lore.kernel.org/all/20260925-gewischt-auftrag-tierzucht-c153cb3f9f74@brauner Suggested-by: Jann Horn Suggested-by: Christian Brauner Fixes: 5b08bd408534 ("pidfs: allow retrieval of namespace file descriptors") Cc: stable@vger.kernel.org Signed-off-by: Chen Linxuan Link: https://patch.msgid.link/20261027-pidfd-task-access-v1-1-d46cf0fa4ca1@black-desk.cn Signed-off-by: Christian Brauner (Amutable) --- fs/pidfs.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fs/pidfs.c b/fs/pidfs.c index 114d2713a79e..d1aa0229866a 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -590,6 +590,14 @@ static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) if (arg) return -EINVAL; + /* + * Require the target to be visible in the caller's pid namespace + * for operations that grant access to its resources, mirroring + * procfs and PIDFD_GET_INFO. + */ + if (!pid_in_current_pidns(pidfd_pid(file))) + return -EREMOTE; + /* * We're trying to open a file descriptor to the namespace so perform a * filesystem cred ptrace check. Hold @task's exec_update_lock for the From 7a5783cfac71b3c12e7fb1c2894dd4538d57d513 Mon Sep 17 00:00:00 2001 From: Siddharth Chintamaneni Date: Sat, 31 Oct 2026 23:00:48 -0700 Subject: [PATCH 19/28] fs: reject negative offsets for SEEK_DATA and SEEK_HOLE in default_llseek() In default_llseek(), the range checks for SEEK_DATA and SEEK_HOLE compare the signed loff_t offset directly against inode->i_size: if (offset >= inode->i_size) { retval = -ENXIO; goto out; } When a negative offset is supplied to SEEK_HOLE (e.g. lseek(fd, -10, SEEK_HOLE) on a sysctl file in /proc/sys), the signed comparison evaluates to false and execution proceeds to: offset = inode->i_size; This overwrites the negative offset with a non-negative value before the final `if (offset >= 0 || unsigned_offsets(file))` check, causing default_llseek() to update file->f_pos and return inode->i_size instead of failing with -ENXIO. Similarly, for SEEK_DATA on a file with FOP_UNSIGNED_OFFSET, a negative offset bypasses the range check and succeeds via unsigned_offsets(file). Read the inode size once via i_size_read(inode) after acquiring the inode lock and compare (unsigned long long)offset >= eof for both SEEK_DATA and SEEK_HOLE, matching must_set_pos(). Fixes: 982d816581ee ("fs: add SEEK_HOLE and SEEK_DATA flags") Signed-off-by: Siddharth Chintamaneni Link: https://patch.msgid.link/20261101060049.81441-2-sidchintamaneni@gmail.com Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/read_write.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fs/read_write.c b/fs/read_write.c index 676728310899..86ce8cea10d7 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -333,13 +333,15 @@ loff_t default_llseek(struct file *file, loff_t offset, int whence) { struct inode *inode = file_inode(file); loff_t retval; + loff_t eof; retval = inode_lock_killable(inode); if (retval) return retval; + eof = i_size_read(inode); switch (whence) { case SEEK_END: - offset += i_size_read(inode); + offset += eof; break; case SEEK_CUR: if (offset == 0) { @@ -354,7 +356,7 @@ loff_t default_llseek(struct file *file, loff_t offset, int whence) * long as offset isn't at the end of the file then the * offset is data. */ - if (offset >= inode->i_size) { + if ((unsigned long long)offset >= eof) { retval = -ENXIO; goto out; } @@ -365,11 +367,11 @@ loff_t default_llseek(struct file *file, loff_t offset, int whence) * as long as offset isn't i_size or larger, return * i_size. */ - if (offset >= inode->i_size) { + if ((unsigned long long)offset >= eof) { retval = -ENXIO; goto out; } - offset = inode->i_size; + offset = eof; break; } retval = -EINVAL; From 1284ecd57375f9e34ce738cae4ece834995f275d Mon Sep 17 00:00:00 2001 From: Siddharth Chintamaneni Date: Sat, 31 Oct 2026 23:00:49 -0700 Subject: [PATCH 20/28] fs: use must_set_pos() in default_llseek() default_llseek() open-codes the same whence switch for SEEK_END, SEEK_CUR, SEEK_DATA, and SEEK_HOLE that must_set_pos() already implements for generic_file_llseek_size() and generic_llseek_cookie(). Replace the open-coded switch (whence) block in default_llseek() with a call to must_set_pos(file, &offset, whence, i_size_read(inode)). Signed-off-by: Siddharth Chintamaneni Link: https://patch.msgid.link/20261101060049.81441-3-sidchintamaneni@gmail.com Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/read_write.c | 50 ++++++++++++++----------------------------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/fs/read_write.c b/fs/read_write.c index 86ce8cea10d7..b15522cf96cd 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -333,47 +333,25 @@ loff_t default_llseek(struct file *file, loff_t offset, int whence) { struct inode *inode = file_inode(file); loff_t retval; - loff_t eof; + int ret; retval = inode_lock_killable(inode); if (retval) return retval; - eof = i_size_read(inode); - switch (whence) { - case SEEK_END: - offset += eof; - break; - case SEEK_CUR: - if (offset == 0) { - retval = file->f_pos; - goto out; - } - offset += file->f_pos; - break; - case SEEK_DATA: - /* - * In the generic case the entire file is data, so as - * long as offset isn't at the end of the file then the - * offset is data. - */ - if ((unsigned long long)offset >= eof) { - retval = -ENXIO; - goto out; - } - break; - case SEEK_HOLE: - /* - * There is a virtual hole at the end of the file, so - * as long as offset isn't i_size or larger, return - * i_size. - */ - if ((unsigned long long)offset >= eof) { - retval = -ENXIO; - goto out; - } - offset = eof; - break; + + ret = must_set_pos(file, &offset, whence, i_size_read(inode)); + if (ret < 0) { + retval = ret; + goto out; } + if (ret == 0) { + retval = offset; + goto out; + } + + if (whence == SEEK_CUR) + offset += file->f_pos; + retval = -EINVAL; if (offset >= 0 || unsigned_offsets(file)) { if (offset != file->f_pos) From de48f2a93e4f246a1dbb3c93dc777061e5861bd4 Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Tue, 3 Nov 2026 10:55:08 -0500 Subject: [PATCH 21/28] selftests/statmount: fix format warnings on 32-bit and s390 The f_type field of struct statfs is an int on i386 and armhf and an unsigned int on s390x, and the listmount() results are uint64_t, so printing them with %lx and %zi warns: statmount_test_ns.c:243:56: warning: format '%zi' expects argument of type 'signed size_t', but argument 2 has type 'uint64_t' {aka 'long long unsigned int'} [-Wformat=] Cast f_type to unsigned long, and print the listmount() results as unsigned long long. On i386 and armhf the comparison next to it also sign-extends f_type to the 64-bit sb_magic, so a file system with the top bit set in its magic, such as btrfs, fails the test. Compare against the same cast. Signed-off-by: Sasha Levin Link: https://patch.msgid.link/20261103155508.535010-1-sashal@kernel.org Signed-off-by: Christian Brauner (Amutable) --- .../testing/selftests/filesystems/statmount/statmount_test.c | 4 ++-- .../selftests/filesystems/statmount/statmount_test_ns.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/filesystems/statmount/statmount_test.c b/tools/testing/selftests/filesystems/statmount/statmount_test.c index b237a95ec9fb..8133a5841ac0 100644 --- a/tools/testing/selftests/filesystems/statmount/statmount_test.c +++ b/tools/testing/selftests/filesystems/statmount/statmount_test.c @@ -335,10 +335,10 @@ static void test_statmount_sb_basic(void) return; } - if (sm.sb_magic != sf.f_type) { + if (sm.sb_magic != (unsigned long)sf.f_type) { ksft_test_result_fail("unexpected sb magic: 0x%llx != 0x%lx\n", (unsigned long long) sm.sb_magic, - sf.f_type); + (unsigned long) sf.f_type); return; } diff --git a/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c b/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c index e500905e4c07..c9ef755ffaa7 100644 --- a/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c +++ b/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c @@ -240,8 +240,9 @@ static int validate_external_listmount(pid_t pid, uint64_t child_nr_mounts) } if (nr_mounts != child_nr_mounts) { - ksft_print_msg("listmount results is %zi != %zi\n", nr_mounts, - child_nr_mounts); + ksft_print_msg("listmount results is %llu != %llu\n", + (unsigned long long)nr_mounts, + (unsigned long long)child_nr_mounts); return NSID_FAIL; } From 8fa19db00bf36d334ef67b4a2157ecf3f28456cc Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Tue, 3 Nov 2026 11:39:23 -0500 Subject: [PATCH 22/28] selftests/mount_setattr: build with a 64-bit off_t With glibc on 32-bit architectures, off_t is 32 bits unless _FILE_OFFSET_BITS=64 is defined, so the 2 GB ftruncate() of the ext4 image gets a negative length and the test fails: mount_setattr_test.c:1048:30: warning: implicit conversion from 'long long' to '__off_t' (aka 'long') changes value from 2147483648 to -2147483648 [-Wconstant-conversion] Build the test with -D_FILE_OFFSET_BITS=64. Signed-off-by: Sasha Levin Link: https://patch.msgid.link/20261103163924.641377-1-sashal@kernel.org Signed-off-by: Christian Brauner (Amutable) --- tools/testing/selftests/mount_setattr/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mount_setattr/Makefile b/tools/testing/selftests/mount_setattr/Makefile index 4d4f810cdf2c..2e5b16fcdf10 100644 --- a/tools/testing/selftests/mount_setattr/Makefile +++ b/tools/testing/selftests/mount_setattr/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # Makefile for mount selftests. -CFLAGS = -g $(KHDR_INCLUDES) -Wall -O2 -pthread +CFLAGS = -g $(KHDR_INCLUDES) -Wall -O2 -pthread -D_FILE_OFFSET_BITS=64 LOCAL_HDRS += ../filesystems/wrappers.h From e00d0eb8e3e7cb0ce97f40225a3642d3f8118d02 Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Tue, 3 Nov 2026 09:27:56 -0500 Subject: [PATCH 23/28] selftests: probe for sanitizer support in fchmodat2 and openat2 The fchmodat2 and openat2 tests are always built with -fsanitize=address and -fsanitize=undefined. Toolchains without the sanitizer runtimes, such as clang without the compiler-rt package, cannot link them and the tests fail to build: ld: cannot find /usr/lib/llvm-21/lib/clang/21/lib/x86_64-unknown-linux-gnu/libclang_rt.asan_static.a: No such file or directory The same happens with USERLDFLAGS=-static, which AddressSanitizer does not support. KernelCI's clang builds fail this way on every architecture. Link an empty program with the sanitizer flags first, and only add them when that works. With the runtimes available the tests are built as before. Signed-off-by: Sasha Levin Link: https://patch.msgid.link/20261103142756.132111-1-sashal@kernel.org Signed-off-by: Christian Brauner (Amutable) --- tools/testing/selftests/fchmodat2/Makefile | 18 +++++++++++++----- .../selftests/filesystems/openat2/Makefile | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/fchmodat2/Makefile b/tools/testing/selftests/fchmodat2/Makefile index 4373cea79b79..21f6b0b0cdc8 100644 --- a/tools/testing/selftests/fchmodat2/Makefile +++ b/tools/testing/selftests/fchmodat2/Makefile @@ -1,15 +1,23 @@ # SPDX-License-Identifier: GPL-2.0-or-later -CFLAGS += -Wall -O2 -g -fsanitize=address -fsanitize=undefined $(KHDR_INCLUDES) +CFLAGS += -Wall -O2 -g $(KHDR_INCLUDES) + +TEST_GEN_PROGS := fchmodat2_test + +include ../lib.mk # gcc requires -static-libasan in order to ensure that Address Sanitizer's # library is the first one loaded. However, clang already statically links the # Address Sanitizer if -fsanitize is specified. Therefore, simply omit # -static-libasan for clang builds. +SANITIZER_CFLAGS := -fsanitize=address -fsanitize=undefined ifeq ($(LLVM),) - CFLAGS += -static-libasan + SANITIZER_CFLAGS += -static-libasan endif -TEST_GEN_PROGS := fchmodat2_test - -include ../lib.mk +# Build without the sanitizers when the toolchain cannot link them. +ifeq ($(shell echo 'int main(void) { return 0; }' | $(CC) $(CFLAGS) \ + $(SANITIZER_CFLAGS) $(LDFLAGS) -x c - -o /dev/null >/dev/null 2>&1 && \ + echo y),y) +CFLAGS += $(SANITIZER_CFLAGS) +endif diff --git a/tools/testing/selftests/filesystems/openat2/Makefile b/tools/testing/selftests/filesystems/openat2/Makefile index d848aac96bde..2f712dfcbf75 100644 --- a/tools/testing/selftests/filesystems/openat2/Makefile +++ b/tools/testing/selftests/filesystems/openat2/Makefile @@ -1,17 +1,25 @@ # SPDX-License-Identifier: GPL-2.0-or-later CFLAGS += $(KHDR_INCLUDES) -CFLAGS += -Wall -O2 -g -fsanitize=address -fsanitize=undefined $(TOOLS_INCLUDES) +CFLAGS += -Wall -O2 -g $(TOOLS_INCLUDES) TEST_GEN_PROGS := openat2_test resolve_test rename_attack_test emptypath_test +LOCAL_HDRS += helpers.h + +include ../../lib.mk + # gcc requires -static-libasan in order to ensure that Address Sanitizer's # library is the first one loaded. However, clang already statically links the # Address Sanitizer if -fsanitize is specified. Therefore, simply omit # -static-libasan for clang builds. +SANITIZER_CFLAGS := -fsanitize=address -fsanitize=undefined ifeq ($(LLVM),) - CFLAGS += -static-libasan + SANITIZER_CFLAGS += -static-libasan endif -LOCAL_HDRS += helpers.h - -include ../../lib.mk +# Build without the sanitizers when the toolchain cannot link them. +ifeq ($(shell echo 'int main(void) { return 0; }' | $(CC) $(CFLAGS) \ + $(SANITIZER_CFLAGS) $(LDFLAGS) -x c - -o /dev/null >/dev/null 2>&1 && \ + echo y),y) +CFLAGS += $(SANITIZER_CFLAGS) +endif From add39877a72564523c8eb8b18201fbb0f3ea286f Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Tue, 3 Nov 2026 11:54:14 -0500 Subject: [PATCH 24/28] selftests/fs/mount-notify: fix the indentation in the fsmount test A few lines in the fsmount test are indented with spaces instead of a tab. That puts them deeper than the ASSERT_*() in front of them, which ends in a for loop with an empty body, so clang warns: mount-notify_test.c:338:18: warning: for loop has empty body [-Wempty-body] Indent them with tabs. Signed-off-by: Sasha Levin Link: https://patch.msgid.link/20261103165419.687806-1-sashal@kernel.org Signed-off-by: Christian Brauner (Amutable) --- .../filesystems/mount-notify/mount-notify_test.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c b/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c index 6381af6a40e3..6c95c275b162 100644 --- a/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c +++ b/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c @@ -337,18 +337,18 @@ TEST_F(fanotify, fsmount) fs = fsopen("tmpfs", 0); ASSERT_GE(fs, 0); - ret = fsconfig(fs, FSCONFIG_CMD_CREATE, 0, 0, 0); + ret = fsconfig(fs, FSCONFIG_CMD_CREATE, 0, 0, 0); ASSERT_EQ(ret, 0); - mnt = fsmount(fs, 0, 0); + mnt = fsmount(fs, 0, 0); ASSERT_GE(mnt, 0); - close(fs); + close(fs); ret = move_mount(mnt, "", AT_FDCWD, "/a", MOVE_MOUNT_F_EMPTY_PATH); ASSERT_EQ(ret, 0); - close(mnt); + close(mnt); mnts[1] = expect_notify_mask(_metadata, self, FAN_MNT_ATTACH); ASSERT_NE(mnts[0], mnts[1]); From daac2c45354df894aab749efc11a92ca9efd68f1 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 2 Nov 2026 18:53:22 -0800 Subject: [PATCH 25/28] fs: porting.rst: use literal string to avoid warning Avoid the docutils warning: Documentation/filesystems/porting.rst:1417: WARNING: Inline emphasis start-string without end-string. [docutils] by quoting the const pointer in literal quotation marks. (Adding a space between the '*' and the '.' also works but it looks odd.) Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202609231806.ILZP6Fy0-lkp@intel.com/ Fixes: 025756ee7b47 ("fs: port mnt_idmap() and file_mnt_idmap() to const mnt_idmap") Signed-off-by: Randy Dunlap Link: https://patch.msgid.link/20261103025322.2549225-1-rdunlap@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- Documentation/filesystems/porting.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index c38f5df755bb..e41466904501 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -1418,7 +1418,7 @@ All struct mnt_idmap pointers handed to filesystems are const now. ->create(), ->mkdir(), ->mknod(), ->symlink(), ->rename(), ->setattr(), ->getattr(), ->permission(), ->tmpfile(), ->get_acl(), ->set_acl() and ->fileattr_set() as well as the xattr ->set() handler and the vfs_*() -helpers take a const struct mnt_idmap *. mnt_idmap() and file_mnt_idmap() +helpers take a ``const struct mnt_idmap *``. mnt_idmap() and file_mnt_idmap() return one. The idmapping is immutable so nothing should have modified it anyway. References are taken and dropped via mnt_idmap_get() and mnt_idmap_put() as before, both accept a const pointer. From ac4b449b39e605123c12ea10042cafc9b6f39603 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 30 Oct 2026 17:15:21 +0100 Subject: [PATCH 26/28] aio: don't let exit_aio() return before complete() is done exit_aio() kills all contexts in one go and waits for them with a single on-stack struct ctx_rq_wait. The count starts at table->nr, each killed context drops one count in free_ioctx_reqs() and exit_aio() subtracts the empty slots afterwards. Whoever drops the last count either calls complete() or, if it is exit_aio(), skips the wait. With a full table skipped is zero and atomic_sub_and_test(0, ...) only tells us whether the count already hit zero. free_ioctx_reqs() runs from the percpu_ref RCU callback, so if exit_aio() is off the CPU long enough the last callback can have done its atomic_dec_and_test() but not yet called complete(): free_ioctx_reqs() exit_aio() atomic_dec_and_test() // 1 -> 0 atomic_sub_and_test(0) // true kfree(table) return complete(&wait.comp) complete() then locks and writes a completion on a stack frame that is gone. So hold an extra count for exit_aio() itself. Then the last count is always dropped by exit_aio() after it waited, or by a callback while exit_aio() waits for it. An unprivileged user gets there with four io_setup() calls and an exit without io_destroy(). Reported-by: Kyle Zeng Fixes: dc48e56d7616 ("aio: fix serial draining in exit_aio()") Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20261030-work-aio-exit_aio-v1-1-83942022a4c4@kernel.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/aio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/aio.c b/fs/aio.c index bf002a9d2ad7..8cf02878a3cc 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -961,7 +961,7 @@ void exit_aio(struct mm_struct *mm) if (!table) return; - atomic_set(&wait.count, table->nr); + atomic_set(&wait.count, table->nr + 1); init_completion(&wait.comp); skipped = 0; @@ -985,7 +985,7 @@ void exit_aio(struct mm_struct *mm) kill_ioctx(mm, ctx, &wait); } - if (!atomic_sub_and_test(skipped, &wait.count)) { + if (!atomic_sub_and_test(skipped + 1, &wait.count)) { /* Wait until all IO for the context are done. */ wait_for_completion(&wait.comp); } From 8eaa35c6c29ae669949f4b5b0814c3d34c8ed658 Mon Sep 17 00:00:00 2001 From: Avi Weiss Date: Sat, 31 Oct 2026 11:59:23 +0200 Subject: [PATCH 27/28] splice, net: avoid MSG_MORE when the bvec array fills splice_to_socket() leaves tail at the last included pipe buffer when its 16-entry bvec array fills. If the requested length exceeds the available pipe data, pipe_occupancy() counts that buffer again and sets MSG_MORE even though the pipe is empty. Small TCP transfers can then be delayed until the cork timeout expires. Advance tail before checking the array limit so the occupancy test counts only pipe buffers that have not been included in the send. Fixes: ca2d49f77ce4 ("splice, net: Fix splice_to_socket() to handle pipe bufs larger than a page") Cc: stable@vger.kernel.org Signed-off-by: Avi Weiss Link: https://patch.msgid.link/20261031095922.3577591-2-thnkslprpt@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/splice.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/splice.c b/fs/splice.c index e6d43329ed3f..957e984512ad 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -863,9 +863,9 @@ ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, bvec_set_page(&bvec[bc++], buf->page, seg, buf->offset); remain -= seg; + tail++; if (remain == 0 || bc >= ARRAY_SIZE(bvec)) break; - tail++; } if (!bc) From 1448959fe3d013ffdb8302d17e4a8fdad2d2da95 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 6 Nov 2026 14:14:35 +0100 Subject: [PATCH 28/28] fs: fix the SEEK_CUR f_lock condition Commit da06e3c51794 ("fs: don't needlessly acquire f_lock") wanted to skip f_lock for SEEK_CUR on files that already serialize f_pos through f_pos_lock, i.e. FMODE_ATOMIC_POS files and directories, and keep it for everything else. The condition it added does the opposite: file_seek_cur_needs_f_lock() returns false exactly for the files that never take f_pos_lock and true for the ones that do. So since v6.15 a SEEK_CUR on a shared block device or on a character device that uses fixed_size_llseek() updates f_pos without any lock and two concurrent calls can lose an update, while regular files and directories pay for a spinlock they don't need. Invert the condition. Fixes: da06e3c51794 ("fs: don't needlessly acquire f_lock") Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20261106-work-file-seek_f_lock-v1-1-5b9a6c2928d4@kernel.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/file.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fs/file.c b/fs/file.c index 4cd40dfa6129..fd08d34874ac 100644 --- a/fs/file.c +++ b/fs/file.c @@ -1387,14 +1387,18 @@ static inline bool file_needs_f_pos_lock(struct file *file) bool file_seek_cur_needs_f_lock(struct file *file) { - if (!(file->f_mode & FMODE_ATOMIC_POS) && !file->f_op->iterate_shared) - return false; - /* + * Files that go through f_pos_lock when they are shared, i.e. + * FMODE_ATOMIC_POS files and directories, don't need f_lock for a + * SEEK_CUR. Everything else does, since nothing else serializes + * the f_pos update. + * * Note that we are not guaranteed to be called after fdget_pos() on * this file obj, in which case the caller is expected to provide the * appropriate locking. */ + if ((file->f_mode & FMODE_ATOMIC_POS) || file->f_op->iterate_shared) + return false; return true; }