From 51216825099e36bf9d39de375749d8365556c5fc Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Wed, 19 Aug 2026 17:51:43 +0800 Subject: [PATCH 001/539] mm: use a folio in the softleaf_is_device_private path Use the folio APIs in the device_private migration path of do_swap_page(), replacing four calls to compound_head() with two page_folio() calls. The second one re-fetches the folio from vmf->page after migrate_to_ram(), which might have split the folio. Link: https://lore.kernel.org/20260819095144.45660-1-hongfu.li@linux.dev Signed-off-by: Hongfu Li Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand (Arm) Link: https://lore.kernel.org/all/e20678ed-3fa1-4677-a1d7-e2af481e8302@kernel.org/ Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Anshuman Khandual Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/memory.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 8b0c2c735d3d..9cbce5c90bff 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4926,18 +4926,21 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto unlock; /* - * Get a page reference while we know the page can't be - * freed. + * Get a folio reference while we know the folio can't + * be freed. */ - if (trylock_page(vmf->page)) { + folio = page_folio(vmf->page); + if (folio_trylock(folio)) { struct dev_pagemap *pgmap; - get_page(vmf->page); + folio_get(folio); pte_unmap_unlock(vmf->pte, vmf->ptl); pgmap = page_pgmap(vmf->page); ret = pgmap->ops->migrate_to_ram(vmf); - unlock_page(vmf->page); - put_page(vmf->page); + /* migrate_to_ram() might have split the folio. */ + folio = page_folio(vmf->page); + folio_unlock(folio); + folio_put(folio); } else { pte_unmap(vmf->pte); softleaf_entry_wait_on_locked(entry, vmf->ptl); From dc954a916d53b0adbebd1498684cd656ce7ade62 Mon Sep 17 00:00:00 2001 From: Qi Xi Date: Wed, 19 Aug 2026 16:20:52 +0800 Subject: [PATCH 002/539] mm: drop stale MAX_ORDER references The treewide rename in commit 5e0a760b4441 ("mm, treewide: rename MAX_ORDER to MAX_PAGE_ORDER") left a few spots still using the old name: - two comments in include/net/mana/mana.h and mm/page_alloc.c; - the gdb helper scripts/gdb/linux/mm.py, where self.MAX_ORDER is a local mirror of the kernel's MAX_ORDER define. Rename the leftover instances to MAX_PAGE_ORDER so the tree is consistent. No functional changes. Link: https://lore.kernel.org/20260819082052.3338603-1-xiqi2@huawei.com Signed-off-by: Qi Xi Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Cc: Jan Kiszka Cc: Johannes Weiner Cc: Kefeng Wang Cc: Kieran Bingham Cc: Konstantin Taranov Cc: Long Li Cc: Michal Hocko Cc: Nanyong Sun Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/net/mana/mana.h | 4 ++-- mm/page_alloc.c | 2 +- scripts/gdb/linux/mm.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h index 83b7eff4646e..e8fda092be37 100644 --- a/include/net/mana/mana.h +++ b/include/net/mana/mana.h @@ -47,8 +47,8 @@ enum mana_priv_flag_bits { #define COMP_ENTRY_SIZE 64 /* This Max value for RX buffers is derived from __alloc_page()'s max page - * allocation calculation. It allows maximum 2^(MAX_ORDER -1) pages. RX buffer - * size beyond this value gets rejected by __alloc_page() call. + * allocation calculation. It allows maximum 2^MAX_PAGE_ORDER pages. RX + * buffer size beyond this value gets rejected by __alloc_page() call. */ #define MAX_RX_BUFFERS_PER_QUEUE 8192 #define DEF_RX_BUFFERS_PER_QUEUE 1024 diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 12fac9084c48..ab385bc252cc 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -7978,7 +7978,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order, /* * Watermarks have not been initialized yet. * - * Accepting one MAX_ORDER page to ensure progress. + * Accepting one MAX_PAGE_ORDER page to ensure progress. */ if (!wmark) return try_to_accept_memory_one(zone); diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py index dffadccbb01d..28d33624c38b 100644 --- a/scripts/gdb/linux/mm.py +++ b/scripts/gdb/linux/mm.py @@ -56,7 +56,7 @@ class x86_page_ops(): self.MAX_PHYSMEM_BITS = 46 self.SECTION_SIZE_BITS = 27 - self.MAX_ORDER = 10 + self.MAX_PAGE_ORDER = 10 self.SECTIONS_SHIFT = self.MAX_PHYSMEM_BITS - self.SECTION_SIZE_BITS self.NR_MEM_SECTIONS = 1 << self.SECTIONS_SHIFT @@ -233,11 +233,11 @@ class aarch64_page_ops(): self.SECTIONS_SHIFT = self.MAX_PHYSMEM_BITS - self.SECTION_SIZE_BITS if str(constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER).isdigit(): - self.MAX_ORDER = constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER + self.MAX_PAGE_ORDER = constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER else: - self.MAX_ORDER = 10 + self.MAX_PAGE_ORDER = 10 - self.MAX_ORDER_NR_PAGES = 1 << (self.MAX_ORDER) + self.MAX_ORDER_NR_PAGES = 1 << (self.MAX_PAGE_ORDER) self.PFN_SECTION_SHIFT = self.SECTION_SIZE_BITS - self.PAGE_SHIFT self.NR_MEM_SECTIONS = 1 << self.SECTIONS_SHIFT self.PAGES_PER_SECTION = 1 << self.PFN_SECTION_SHIFT From 46fa67175c2ec7a6bba2274f02ea23102d3ba5e6 Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Wed, 26 Aug 2026 20:44:09 +0800 Subject: [PATCH 003/539] mm/vmscan: drop the combined limit gate in __node_reclaim() __node_reclaim() is called from two paths: node_reclaim() and user_proactive_reclaim(). node_reclaim() already bails out early unless node_pagecache_reclaimable() is over pgdat->min_unmapped_pages or the reclaimable slab is over pgdat->min_slab_pages. The identical check inside __node_reclaim() that guards the shrink_node() loop is therefore redundant for this path. user_proactive_reclaim() is proactive reclaim driven by userspace and should not be gated by the per-node min_unmapped_pages / min_slab_pages limits at all [1]. With the gate in place, a proactive request is silently turned into a no-op whenever the node happens to sit below both thresholds. Drop the gate in __node_reclaim() and always run the shrink_node() loop. The node_reclaim() path is unchanged, since its caller has already applied the same test; the proactive path is no longer wrongly gated. Link: https://lore.kernel.org/20260826124409.35569-1-ridong.chen@linux.dev Link: https://sashiko.dev/#/patchset/20260723045718.2052070-1-ridong.chen@linux.dev [1] Fixes: b980077899ea ("mm: introduce per-node proactive reclaim interface") Acked-by: Johannes Weiner Signed-off-by: Ridong Chen Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Shakeel Butt Acked-by: Davidlohr Bueso Assisted-by: Claude:claude-opus-4-8 Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Roman Gushchin Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index f11491ee9ed5..6dff207ad8c6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7851,16 +7851,9 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, noreclaim_flag = memalloc_noreclaim_save(); set_task_reclaim_state(p, &sc->reclaim_state); - if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || - node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { - /* - * Free memory by calling shrink node with increasing - * priorities until we have enough memory freed. - */ - do { - shrink_node(pgdat, sc); - } while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0); - } + do { + shrink_node(pgdat, sc); + } while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0); set_task_reclaim_state(p, NULL); memalloc_noreclaim_restore(noreclaim_flag); From afd20f513ca414dc56c02828b0982a08f7d864dd Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Tue, 25 Aug 2026 10:10:13 +0800 Subject: [PATCH 004/539] mm/hugetlb: fix resv_huge_pages double decrement in memfd error path alloc_hugetlb_folio_reserve() decrements h->resv_huge_pages when dequeuing a folio, but unlike the use_global_reservation handling in hugetlb_alloc_folio(), it does not set HPageRestoreReserve on the folio. Its sole caller memfd_alloc_folio() pre-allocates a reservation via hugetlb_reserve_pages() before allocating. When hugetlb_add_to_page_cache() fails, folio_put() drops the folio without HPageRestoreReserve set, so free_huge_folio() does not restore the reservation. The subsequent hugetlb_unreserve_pages() on the err_unresv path decrements the counter a second time, leaving resv_huge_pages off by one for every failed allocation. Set HPageRestoreReserve when consuming the reservation in alloc_hugetlb_folio_reserve(). On the error path, free_huge_folio() then restores the reservation before hugetlb_unreserve_pages() releases it. The success path is unaffected, as hugetlb_add_to_page_cache() clears the flag once the folio is added to the page cache. Link: https://lore.kernel.org/20260825021013.25672-1-hongfu.li@linux.dev Fixes: 26a8ea80929c ("mm/hugetlb: fix memfd_pin_folios resv_huge_pages leak") Signed-off-by: Hongfu Li Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Cc: David Hildenbrand Cc: Oscar Salvador Cc: Steven Sistare Cc: Vivek Kasireddy --- mm/hugetlb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index cea25773a6c9..a10734f4de11 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2187,8 +2187,10 @@ struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid, folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, preferred_nid, nmask); - if (folio) + if (folio) { + folio_set_hugetlb_restore_reserve(folio); h->resv_huge_pages--; + } spin_unlock_irq(&hugetlb_lock); return folio; From d3bb6b3e4130dae3c01611cb6720d695d68e3f1c Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Tue, 25 Aug 2026 21:47:15 +0530 Subject: [PATCH 005/539] selftests/mm: remove the local PKEY_UNRESTRICTED fallback pkey-helpers.h defines PKEY_UNRESTRICTED itself when the macro is not already known, a stopgap from when the generic definition was still under review. It has been merged since, commit 6d61527d931b ("mm/pkey: Add PKEY_UNRESTRICTED macro"), so the guard is never taken and the FIXME can be honoured. The definition comes from tools/include/uapi/asm-generic/mman-common.h via TOOLS_INCLUDES, which commit e076eaca5906 ("selftests: break the dependency upon local header files") added so that the mm selftests build without "make headers". It is reached through the that the system includes. Building the pkey tests with KHDR_INCLUDES pointing at an empty directory confirms that; emptying TOOLS_INCLUDES as well is what makes the macro go missing. No functional change intended. Link: https://lore.kernel.org/20260825161715.2807297-1-hemanth.selam@gmail.com Signed-off-by: Hemanth Selam Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: SJ Park Assisted-by: Cursor:claude-opus-5 Cc: Kevin Brodsky Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yury Khrustalev --- tools/testing/selftests/mm/pkey-helpers.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h index 46a8a1878dc1..9b949951743d 100644 --- a/tools/testing/selftests/mm/pkey-helpers.h +++ b/tools/testing/selftests/mm/pkey-helpers.h @@ -114,13 +114,6 @@ void record_pkey_malloc(void *ptr, long size, int prot); #define PKEY_MASK (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE) #endif -/* - * FIXME: Remove once the generic PKEY_UNRESTRICTED definition is merged. - */ -#ifndef PKEY_UNRESTRICTED -#define PKEY_UNRESTRICTED 0x0 -#endif - #ifndef set_pkey_bits static inline u64 set_pkey_bits(u64 reg, int pkey, u64 flags) { From fb90e5c2fddff88dff35aaabd658e95fe77170cb Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Fri, 21 Aug 2026 10:16:06 +0800 Subject: [PATCH 006/539] mm/mglru: preserve inactive placement when enabling MGLRU When the LRU is switched to MGLRU (echo y > /sys/kernel/mm/lru_gen/ enabled), fill_evictable() re-inserts every folio via lru_gen_add_folio(..., false). With reclaiming hardcoded to false, an inactive anonymous folio (no PG_active, not in the swapcache) takes the "gen = MIN_NR_GENS" branch in lru_gen_folio_seq() and is seeded at seq = max_seq - 1, which lru_gen_is_active() treats as active. Its inactive placement is lost and NR_INACTIVE_ANON is folded into NR_ACTIVE_ANON. Pass reclaiming=!active so a folio from an inactive list is seeded into an older generation. Folios from the active list carry PG_active and hit the first branch either way, so they are unchanged. Reclaiming also selects the insertion end in lru_gen_add_folio(): list_add_tail() for inactive folios, list_add() for active ones. Both the legacy LRU and a MGLRU generation keep the hottest folios at the head and the coldest at the tail, and reclaim takes from the tail. To preserve that order the folio must be taken from the end matching the insertion end, so take inactive folios from the head and active folios from the tail; otherwise hot/cold would be reversed within the generation. Tested on x86_64, next-20260812, 2G VM + 1G swap, ~1.5G anon pushed onto the inactive list before enabling MGLRU: Active(anon) Inactive(anon) before switch (legacy) 2952 1548792 kB after `echo y`, unpatched 1552052 0 kB after `echo y`, patched 15144 1536636 kB Inactive file folios stay inactive either way (NR_INACTIVE_FILE is preserved). Link: https://lore.kernel.org/20260821021606.877330-1-ridong.chen@linux.dev Fixes: 354ed5974429 ("mm: multi-gen LRU: kill switch") Signed-off-by: Ridong Chen Signed-off-by: Andrew Morton Suggested-by: Barry Song Acked-by: Barry Song Assisted-by: Claude:claude-opus-4-8 Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Jan Alexander Steffens (heftig) Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oleksandr Natalenko Cc: Shakeel Butt Cc: Steven Barrett Cc: Suleiman Souhlal Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao --- mm/vmscan.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 6dff207ad8c6..fdd13299a04a 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -5307,7 +5307,17 @@ static bool fill_evictable(struct lruvec *lruvec) while (!list_empty(head)) { bool success; - struct folio *folio = lru_to_folio(head); + struct folio *folio; + + /* + * lru_gen_add_folio() uses list_add_tail() rather + * than list_add() when reclaiming is true. Match + * its ordering to avoid cold/hot inversion. + */ + if (active) + folio = lru_to_folio(head); + else + folio = list_first_entry(head, struct folio, lru); VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); @@ -5315,7 +5325,11 @@ static bool fill_evictable(struct lruvec *lruvec) VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); lruvec_del_folio(lruvec, folio); - success = lru_gen_add_folio(lruvec, folio, false); + /* + * Borrow reclaiming=true to place inactive folios in + * the older gens. + */ + success = lru_gen_add_folio(lruvec, folio, !active); VM_WARN_ON_ONCE(!success); if (!--remaining) From d3863210464f69c18615cbe400d48a6e53e770d4 Mon Sep 17 00:00:00 2001 From: Chengfeng Ye Date: Mon, 24 Aug 2026 19:24:33 +0800 Subject: [PATCH 007/539] mm/ksm: mark migration stores with WRITE_ONCE() ksm_get_folio() deliberately samples stable_node->kpfn and folio->mapping without taking the folio lock because the KSM folio may be migrated concurrently. folio_migrate_ksm() updates the same state using plain assignments. The reader can load the old kpfn, then the migrator can store the new kpfn, execute smp_wmb(), and clear the old folio's mapping before the reader checks that mapping. Thus the initial kpfn load can overlap its update and the subsequent mapping load can overlap the clear, with no common lock. This leaves marked READ_ONCE() accesses racing with plain stores. The kernel reported: BUG: KCSAN: data-race in folio_migrate_ksm / ksm_get_folio read (marked) to 0xffff8ce401421330 of 8 bytes by task 48 on cpu 3: ksm_get_folio+0x7f/0x2a0 ksm_scan_thread+0x1635/0x3330 kthread+0x1af/0x1f0 write to 0xffff8ce401421330 of 8 bytes by task 102 on cpu 1: folio_migrate_ksm+0x6a/0xd0 folio_migrate_flags+0x193/0x420 __migrate_folio.isra.0+0x162/0x1a0 migrate_folio+0x4c/0x70 move_to_new_folio+0xd6/0x170 Use WRITE_ONCE() for both stores to pair them with the existing lockless reads. This preserves the existing smp_wmb()/smp_rmb() migration protocol and control flow while preventing compiler transformations of the shared accesses. Link: https://lore.kernel.org/20260824112433.191301-1-nicoyip.dev@gmail.com Fixes: c8d6553b9580 ("ksm: make KSM page migration possible") Signed-off-by: Chengfeng Ye Signed-off-by: Andrew Morton Acked-by: Xu Xin Acked-by: David Hildenbrand (Arm) Cc: Hugh Dickins Cc: --- mm/ksm.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 49d48d1e0998..fe42c17490b8 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1115,7 +1115,8 @@ static inline void folio_set_stable_node(struct folio *folio, struct ksm_stable_node *stable_node) { VM_WARN_ON_FOLIO(folio_test_anon(folio) && PageAnonExclusive(&folio->page), folio); - folio->mapping = (void *)((unsigned long)stable_node | FOLIO_MAPPING_KSM); + WRITE_ONCE(folio->mapping, + (void *)((unsigned long)stable_node | FOLIO_MAPPING_KSM)); } #ifdef CONFIG_SYSFS @@ -3316,7 +3317,7 @@ void folio_migrate_ksm(struct folio *newfolio, struct folio *folio) stable_node = folio_stable_node(folio); if (stable_node) { VM_BUG_ON_FOLIO(stable_node->kpfn != folio_pfn(folio), folio); - stable_node->kpfn = folio_pfn(newfolio); + WRITE_ONCE(stable_node->kpfn, folio_pfn(newfolio)); /* * newfolio->mapping was set in advance; now we need smp_wmb() * to make sure that the new stable_node->kpfn is visible From 40965d4997f4b04cc217b16061eae2cb7d243e6d Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Mon, 24 Aug 2026 23:16:55 +0800 Subject: [PATCH 008/539] mm/hugetlb: use hugetlb_vmemmap_optimizable() in boolean contexts The two boot-time sites in hugetlb_hstate_alloc_pages_onenode() and hugetlb_pages_alloc_boot_node() only need to know whether HVO is applicable, not the exact optimizable size. Switch them from hugetlb_vmemmap_optimizable_size() to hugetlb_vmemmap_optimizable() to make intent explicit. No functional change intended. Link: https://lore.kernel.org/20260824151655.30840-1-kaitao.cheng@linux.dev Signed-off-by: Kaitao Cheng Signed-off-by: Andrew Morton Reviewed-by: Joshua Hahn Reviewed-by: Muchun Song Cc: David Hildenbrand Cc: Oscar Salvador --- mm/hugetlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a10734f4de11..e645d2d25bb6 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3427,7 +3427,7 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid, &node_states[N_MEMORY], NULL); if (!folio && !list_empty(&folio_list) && - hugetlb_vmemmap_optimizable_size(h)) { + hugetlb_vmemmap_optimizable(h)) { prep_and_add_allocated_folios(h, &folio_list); INIT_LIST_HEAD(&folio_list); folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid, @@ -3496,7 +3496,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l for (i = 0; i < num; ++i) { struct folio *folio; - if (hugetlb_vmemmap_optimizable_size(h) && + if (hugetlb_vmemmap_optimizable(h) && (si_mem_available() == 0) && !list_empty(&folio_list)) { prep_and_add_allocated_folios(h, &folio_list); INIT_LIST_HEAD(&folio_list); From 172c05ebab49d4d76c54209211efed2bc56f514f Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Mon, 24 Aug 2026 14:10:09 +0800 Subject: [PATCH 009/539] docs: ksm: fix typos in sysfs knob names Patch series "docs/ksm: fix advisor documentation and comment", v3. This series fixes two problems left in the KSM advisor documentation and code comment: - Patch 1 fixes two typos in sysfs knob names ("adivsor_max_cpu" and "adivsor_max_pages_to_scan") in ksm.rst that don't match the actual knob names. - Patch 2 fixes the description of advisor_min_pages_to_scan: it is described as the lower limit of pages_to_scan, but is actually only used as it's initial value, and the runtime pages_to_scan can drop below advisor_min_pages_to_scan. This patch (of 2): The sysfs knob names in mm/ksm.c are "advisor_max_cpu" and "advisor_max_pages_to_scan", but the ksm.rst documentation spelled both as "adivsor_*", fix the two typos. Link: https://lore.kernel.org/20260824061010.3343959-1-tujinjiang@huawei.com Link: https://lore.kernel.org/20260824061010.3343959-2-tujinjiang@huawei.com Signed-off-by: Jinjiang Tu Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: Randy Dunlap Cc: Chengming Zhou Cc: Jonathan Corbet Cc: Kefeng Wang Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Nanyong Sun Cc: Stefan Roesch Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: xu xin --- Documentation/admin-guide/mm/ksm.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst index ad8e7a41f3b5..c9f533b10f6f 100644 --- a/Documentation/admin-guide/mm/ksm.rst +++ b/Documentation/admin-guide/mm/ksm.rst @@ -174,7 +174,7 @@ advisor_mode The section about ``advisor`` explains in detail how the scan time advisor works. -adivsor_max_cpu +advisor_max_cpu specifies the upper limit of the cpu percent usage of the ksmd background thread. The default is 70. @@ -186,7 +186,7 @@ advisor_min_pages_to_scan specifies the lower limit of the ``pages_to_scan`` parameter of the scan time advisor. The default is 500. -adivsor_max_pages_to_scan +advisor_max_pages_to_scan specifies the upper limit of the ``pages_to_scan`` parameter of the scan time advisor. The default is 30000. From 585bb36e6ab8a2735f3c6bcfee3bab0ef7c7c68a Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Mon, 24 Aug 2026 14:10:10 +0800 Subject: [PATCH 010/539] mm/ksm: fix advisor_min_pages_to_scan description Both Documentation/admin-guide/mm/ksm.rst and the comment next to the variable definition in mm/ksm.c describe advisor_min_pages_to_scan as a lower limit of the pages_to_scan parameter, but that is not how the scan-time advisor actually uses it. commit 4e5fa4f5eff6 ("mm/ksm: add ksm advisor") only uses it to initialize ksm_thread_pages_to_scan when the scan-time advisor is enabled. ksm_thread_pages_to_scan is adjusted by scan_time_advisor() after a full scan finishes. ksm_thread_pages_to_scan could be increased or decreased depend on the real scan time is longer or shorter than the target scan time. The min value of ksm_thread_pages_to_scan is only limited by KSM_ADVISOR_MIN_CPU, so ksm_thread_pages_to_scan could be smaller than ksm_advisor_min_pages_to_scan. The semantics of advisor_min_pages_to_scan was updated in the v2 patchset [1], but the documentation wasn't updated. Update the documentation and comment to match the semantics of advisor_min_pages_to_scan. Link: https://lore.kernel.org/linux-mm/20231028000945.2428830-2-shr@devkernel.io/ [1] Link: https://lore.kernel.org/20260824061010.3343959-3-tujinjiang@huawei.com Signed-off-by: Jinjiang Tu Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Cc: Chengming Zhou Cc: Jonathan Corbet Cc: Kefeng Wang Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Nanyong Sun Cc: Stefan Roesch Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: xu xin Cc: Randy Dunlap --- Documentation/admin-guide/mm/ksm.rst | 4 ++-- mm/ksm.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst index c9f533b10f6f..c329ca747b8c 100644 --- a/Documentation/admin-guide/mm/ksm.rst +++ b/Documentation/admin-guide/mm/ksm.rst @@ -183,8 +183,8 @@ advisor_target_scan_time pages. The default value is 200 seconds. advisor_min_pages_to_scan - specifies the lower limit of the ``pages_to_scan`` parameter of the - scan time advisor. The default is 500. + specifies the initial value of the ``pages_to_scan`` parameter of + the scan time advisor. The default is 500. advisor_max_pages_to_scan specifies the upper limit of the ``pages_to_scan`` parameter of the diff --git a/mm/ksm.c b/mm/ksm.c index fe42c17490b8..624f37975e12 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -348,7 +348,7 @@ static enum ksm_advisor_type ksm_advisor; * Only called through the sysfs control interface: */ -/* At least scan this many pages per batch. */ +/* Initial number of pages to scan per batch. */ static unsigned long ksm_advisor_min_pages_to_scan = 500; static void set_advisor_defaults(void) From 576446e2dfe47c5652e2503c178191478c71bbe6 Mon Sep 17 00:00:00 2001 From: Anshuman Date: Wed, 26 Aug 2026 11:43:00 +0530 Subject: [PATCH 011/539] selftests/mm: fix line buffer leak in mremap_test is_range_mapped() is_range_mapped() uses getline() to read /proc/self/maps line by line, but never frees the buffer it allocates. Every exit path (parse failure, match found, or reaching EOF) returns without calling free(line), leaking the buffer on each call. The function is called multiple times in this test, so the leak accumulates across calls. Free line before returning. Link: https://lore.kernel.org/20260826061300.14038-1-anshumantewari123@gmail.com Signed-off-by: Anshuman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: SJ Park Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/mremap_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 131d9d6db867..779ef2d5f1b9 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -156,6 +156,7 @@ static bool is_range_mapped(FILE *maps_fp, unsigned long start, } } + free(line); return success; } From c20cf1c3c8cdd98618d6fcc794003a0e003ae858 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Thu, 27 Aug 2026 10:54:56 +0800 Subject: [PATCH 012/539] mm/memcontrol: fix data-race on reading jiffies_64 KCSAN reported a data-race between tick_do_update_jiffies64() updating jiffies_64 and mem_cgroup_flush_stats_ratelimited() reading it directly. Unlike jiffies, jiffies_64 is not volatile, so raw reads are plain accesses and can even be torn on 32-bit. Use get_jiffies_64() instead, and fix the same pattern in mem_cgroup_flush_foreign(). Link: https://lore.kernel.org/20260827025457.116191-1-jiayuan.chen@linux.dev Fixes: 508bed884767 ("mm: memcg: change flush_next_time to flush_last_time") Fixes: 97b27821b485 ("writeback, memcg: Implement foreign dirty flushing") Signed-off-by: Jiayuan Chen Signed-off-by: Andrew Morton Reported-by: syzbot+ced4d9a8cadb5ef3adae@syzkaller.appspotmail.com Acked-by: Muchun Song Acked-by: Johannes Weiner Acked-by: Michal Hocko Cc: Chris Li Cc: Jan Kara Cc: Jens Axboe Cc: Roman Gushchin Cc: Shakeel Butt Cc: Tejun Heo Cc: --- mm/memcontrol.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 856a7d07586c..d399710e9799 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -757,7 +757,7 @@ static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force) return; if (mem_cgroup_is_root(memcg)) - WRITE_ONCE(flush_last_time, jiffies_64); + WRITE_ONCE(flush_last_time, get_jiffies_64()); css_rstat_flush(&memcg->css); } @@ -785,7 +785,7 @@ void mem_cgroup_flush_stats(struct mem_cgroup *memcg) void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg) { /* Only flush if the periodic flusher is one full cycle late */ - if (time_after64(jiffies_64, READ_ONCE(flush_last_time) + 2*FLUSH_TIME)) + if (time_after64(get_jiffies_64(), READ_ONCE(flush_last_time) + 2 * FLUSH_TIME)) mem_cgroup_flush_stats(memcg); } @@ -3946,7 +3946,7 @@ void mem_cgroup_flush_foreign(struct bdi_writeback *wb) { struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10); - u64 now = jiffies_64; + u64 now = get_jiffies_64(); int i; for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { From 197654a838c88ce00259757475ebddca39c06996 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Thu, 27 Aug 2026 15:05:46 +0800 Subject: [PATCH 013/539] mm/vmstat: annotate data race for per-cpu pageset fields zoneinfo_show_print() reads pcp->count, pcp->high, pcp->batch, pcp->high_min, pcp->high_max and the per-cpu stat_threshold while holding only zone->lock, which does not synchronize these fields. The writers are the page allocation and free fast paths under pcp->lock, decay_pcp_high() which updates pcp->high without any lock, pageset_update() which writes batch/high_min/high_max with WRITE_ONCE(), and refresh_zone_stat_thresholds() which writes stat_threshold locklessly. The race is benign: the values are only printed to /proc/zoneinfo, they are naturally aligned integers, and pageset_update() already documents that users of batch/high_min/high_max must cope with the fields changing asynchronously. Annotate the reads with data_race(), following commit af1c31acc853 ("mm/vmstat: annotate data race for zone->free_area[order].nr_free"). Found by KCSAN testing on an older kernel; the same race still exists on mainline. No functional change intended. BUG: KCSAN: data-race in zoneinfo_show_print+0x355/0x520 root/klinux/mm/vmstat.c:1774 race at unknown origin, with read to 0xffff8e1835410808 of 4 bytes by task 22653 on cpu 12: zoneinfo_show_print+0x355/0x520 root/klinux/mm/vmstat.c:1774 walk_zones_in_node root/klinux/mm/vmstat.c:1496 [inline] zoneinfo_show+0x41/0x70 root/klinux/mm/vmstat.c:1806 seq_read_iter+0x30c/0x970 root/klinux/fs/seq_file.c:230 proc_reg_read_iter+0x10c/0x170 root/klinux/fs/proc/inode.c:305 copy_splice_read+0x2a1/0x4e0 root/klinux/fs/splice.c:365 do_splice_read root/klinux/fs/splice.c:985 [inline] do_splice_read+0x139/0x1a0 root/klinux/fs/splice.c:959 splice_direct_to_actor+0x16b/0x540 root/klinux/fs/splice.c:1089 do_splice_direct_actor root/klinux/fs/splice.c:1207 [inline] do_splice_direct+0x10a/0x180 root/klinux/fs/splice.c:1233 do_sendfile+0x6ea/0x7e0 root/klinux/fs/read_write.c:1363 __do_sys_sendfile64 root/klinux/fs/read_write.c:1424 [inline] __se_sys_sendfile64 root/klinux/fs/read_write.c:1410 [inline] __x64_sys_sendfile64+0x117/0x130 root/klinux/fs/read_write.c:1410 x64_sys_call+0x1cc7/0x1ee0 root/klinux/./arch/x86/include/generated/asm/syscalls_64.h:41 do_syscall_x64 root/klinux/arch/x86/entry/common.c:46 [inline] do_syscall_64+0x75/0x2c0 root/klinux/arch/x86/entry/common.c:76 entry_SYSCALL_64_after_hwframe+0x76/0xe0 value changed: 0x000001b2 -> 0x000001b1 Reported by Kernel Concurrency Sanitizer on: CPU: 12 PID: 22653 Comm: syz-executor.12 Not tainted 6.6.140+ #672 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 Link: https://lore.kernel.org/20260827070546.1336383-1-hui.zhu@linux.dev Signed-off-by: Hui Zhu Signed-off-by: Andrew Morton Acked-by: Vlastimil Babka (SUSE) Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan --- mm/vmstat.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mm/vmstat.c b/mm/vmstat.c index cb57714539fb..a3e809c57f29 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1837,6 +1837,11 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, struct per_cpu_zonestat __maybe_unused *pzstats; pcp = per_cpu_ptr(zone->per_cpu_pageset, i); + /* + * Access to the per-cpu pageset fields is lockless as they + * are used only for printing purposes. Use data_race to + * avoid KCSAN warning. + */ seq_printf(m, "\n cpu: %i" "\n count: %i" @@ -1845,15 +1850,15 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, "\n high_min: %i" "\n high_max: %i", i, - pcp->count, - pcp->high, - pcp->batch, - pcp->high_min, - pcp->high_max); + data_race(pcp->count), + data_race(pcp->high), + data_race(pcp->batch), + data_race(pcp->high_min), + data_race(pcp->high_max)); #ifdef CONFIG_SMP pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); seq_printf(m, "\n vm stats threshold: %d", - pzstats->stat_threshold); + data_race(pzstats->stat_threshold)); #endif } seq_printf(m, From b482e72da822d7bd8ab2e58971f069b1c7a61c74 Mon Sep 17 00:00:00 2001 From: Hao Li Date: Thu, 27 Aug 2026 15:18:06 +0800 Subject: [PATCH 014/539] mm: remove unused anon_vma_trylock_write() The last user of anon_vma_trylock_write() was removed by cc22b9978509, leaving the helper unused. Remove it. Link: https://lore.kernel.org/20260827071845.17636-1-hao.li@linux.dev Signed-off-by: Hao Li Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Reviewed-by: SJ Park Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/internal.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 38b1165212c9..da833cafcd59 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -295,11 +295,6 @@ static inline void anon_vma_lock_write(struct anon_vma *anon_vma) down_write(&anon_vma->root->rwsem); } -static inline int anon_vma_trylock_write(struct anon_vma *anon_vma) -{ - return down_write_trylock(&anon_vma->root->rwsem); -} - static inline void anon_vma_unlock_write(struct anon_vma *anon_vma) { up_write(&anon_vma->root->rwsem); From d67aa44be6d9b240285fdd82bb932e977f97998f Mon Sep 17 00:00:00 2001 From: Yue Haibing Date: Thu, 27 Aug 2026 16:27:22 +0800 Subject: [PATCH 015/539] mm/swap: remove unused declaration swapcache_clear() Commit c246d236b18b ("mm/shmem: never bypass the swap cache for SWP_SYNCHRONOUS_IO") removed the implementations but leave this. Link: https://lore.kernel.org/20260827082722.1809702-1-yuehaibing@huawei.com Signed-off-by: Yue Haibing Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Acked-by: Kairui Song Acked-by: Nick Huang Reviewed-by: Barry Song Reviewed-by: SJ Park Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham --- mm/swap.h | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/swap.h b/mm/swap.h index 90a551a88df6..fddba7a87500 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -324,7 +324,6 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new); void show_swap_cache_info(void); -void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr); struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, From 2b404a141d9c2d8deeae9d4dd7e271899ecd30f5 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 26 Aug 2026 10:17:53 +0800 Subject: [PATCH 016/539] docs: cgroup: document empty-write behavior of memory limit knobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A maintenance script on a cluster wrote an unset variable into memory.max of a workload cgroup; the variable expanded to an empty string, the write succeeded, and the workload in the cgroup was OOM-killed. Nothing pointed back at the write, so it took quite some time to trace the OOM kills to that script. The memory controller documentation does not say what an empty write does; the cpuset controller documents its empty-value semantics. The actual behavior is that the empty string is accepted as 0. Reproduced on a k8s cluster (v1.29, cgroup v2, two-container pod, 384M limit): # LIMIT= # echo "$LIMIT" > $CG/memory.max # echo $? 0 m6demo 0/2 OOMKilled 0 Memory cgroup out of memory: Killed process 339529 (sleep) ... anon-rss:32kB State it where the interface files are introduced, alongside the existing notes on units and page rounding. Link: https://lore.kernel.org/all/aoVUlFdZYLFn_gvJ@tiehlicka/ Link: https://lore.kernel.org/20260826021753.197871-1-cui.tao@linux.dev Signed-off-by: Tao Cui Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Shakeel Butt Cc: Johannes Weiner Cc: Michal Koutný Cc: Muchun Song Cc: Roman Gushchin Cc: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 86a2a0099178..8d2603751c51 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1321,6 +1321,10 @@ All memory amounts are in bytes. If a value which is not aligned to PAGE_SIZE is written, the value may be rounded up to the closest PAGE_SIZE multiple when read back. +For the limit files described below, an empty or all-whitespace +write is accepted and sets the limit to 0. To disable a limit, +write "max"; to set it to zero explicitly, write "0". + memory.current A read-only single value file which exists on non-root cgroups. From ed2052b119be7ccdb5271654cc218e79bb01ca32 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Tue, 25 Aug 2026 13:20:59 +0200 Subject: [PATCH 017/539] selftests/mm: khugepaged: remove str_dup() usage We don't check str_dup() return value and never free it. While both things are irrelevant in practice, let's just clean it up by working on argv[0] directly and avoiding the str_dup(). Nobody after us needs these parts of the argv[0] string anyway. This patch is inspired by previous work from Anshuman Tewari [1]. Link: https://lore.kernel.org/r/20260821114416.12255-1-anshumantewari123@gmail.com [1] Link: https://lore.kernel.org/20260825-remove_str_dup-v1-1-0ba2121a820c@kernel.org Signed-off-by: David Hildenbrand (Arm) Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Anshuman Tewari Reviewed-by: Zi Yan Reviewed-by: Lance Yang Reviewed-by: Dev Jain Acked-by: Usama Arif Reviewed-by: Baolin Wang Reviewed-by: Barry Song Reviewed-by: Andrew Morton --- tools/testing/selftests/mm/khugepaged.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 1d2d6bd72fd2..83d27d069c41 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -1227,7 +1227,7 @@ static void parse_test_type(int argc, char **argv) return; } - buf = strdup(argv[0]); + buf = argv[0]; token = strsep(&buf, ":"); if (!strcmp(token, "all")) { From c333c6e26ca356702c21be2f065427a556db1b7d Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Tue, 25 Aug 2026 20:01:53 +0800 Subject: [PATCH 018/539] mm/memcontrol: remove unused memcg parameter in calculate_high_delay() The memcg argument of calculate_high_delay() is never referenced in its function body. The delay calculation only depends on nr_pages and max_overage, and both callers have already obtained max_overage from the same memcg. Drop this unused parameter and update the two call sites inside __mem_cgroup_handle_over_high(). Link: https://lore.kernel.org/20260825120153.1405-1-hongfu.li@linux.dev Signed-off-by: Hongfu Li Signed-off-by: Andrew Morton Acked-by: Michal Hocko Reviewed-by: Gregory Price (Meta) Reviewed-by: SJ Park Acked-by: Shakeel Butt --- mm/memcontrol.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index d399710e9799..1709ac96bbde 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2515,8 +2515,7 @@ static u64 swap_find_max_overage(struct mem_cgroup *memcg) * Get the number of jiffies that we should penalise a mischievous cgroup which * is exceeding its memory.high by checking both it and its ancestors. */ -static unsigned long calculate_high_delay(struct mem_cgroup *memcg, - unsigned int nr_pages, +static unsigned long calculate_high_delay(unsigned int nr_pages, u64 max_overage) { unsigned long penalty_jiffies; @@ -2594,10 +2593,10 @@ retry_reclaim: * memory.high is breached and reclaim is unable to keep up. Throttle * allocators proactively to slow down excessive growth. */ - penalty_jiffies = calculate_high_delay(memcg, nr_pages, + penalty_jiffies = calculate_high_delay(nr_pages, mem_find_max_overage(memcg)); - penalty_jiffies += calculate_high_delay(memcg, nr_pages, + penalty_jiffies += calculate_high_delay(nr_pages, swap_find_max_overage(memcg)); /* From eba0e6297edd60a85bc96ff5cfb734d066af62dd Mon Sep 17 00:00:00 2001 From: Qi Xi Date: Tue, 25 Aug 2026 20:05:48 +0800 Subject: [PATCH 019/539] mm/page_isolation: fix UBSAN shift-out-of-bounds warning Patch series "mm/page_isolation: fix UBSAN shift-out-of-bounds in isolate_single_pageblock", v3. Patch 1 fixes a UBSAN shift-out-of-bounds warning in the PageBuddy branch of isolate_single_pageblock() triggered by concurrent buddy allocation. Patch 2 addresses the same class of issue in the PageCompound branch, where racy compound_order() and compound_head() reads could lead to out-of-range shifts or incorrect page skipping. This patch (of 2): A contig-range allocation racing with buddy allocation on the adjacent pageblock can trigger: UBSAN: shift-out-of-bounds in mm/page_isolation.c:393:15 shift exponent -749042176 is negative Call trace: isolate_single_pageblock start_isolate_page_range alloc_contig_frozen_range_noprof alloc_contig_range_noprof isolate_single_pageblock() first calls set_migratetype_isolate() with zone->lock held, which marks the pageblock MIGRATE_ISOLATE and moves any free page straddling the boundary out of the way. Once the lock is dropped, it scans the MAX_ORDER_NR_PAGES-aligned window [start_pfn, boundary_pfn) locklessly, only to skip the free pages already handled above and to detect in-use pages straddling the boundary. Since this scan only reads page state to decide how far to skip and returns -EBUSY on a straddling in-use page, it does not take the lock. The window also covers the adjacent pageblock, whose free pages stay on the normal movable/CMA freelist and can be allocated concurrently. So after the scan observes PageBuddy(page), another CPU can allocate the page, leaving a stale value in page->private that makes "1 << order" shift out of range. Use buddy_order_unsafe() with READ_ONCE to read the order, and validate it is within MAX_PAGE_ORDER before shifting to prevent UBSAN warnings. Since pageblock_isolate_and_move_free_pages() already handles free pages straddling boundary_pfn under zone->lock, bail out with -EBUSY instead of VM_WARN_ON_ONCE() when a PageBuddy page appears to cross the boundary during the lockless scan. Link: https://lore.kernel.org/20260825120549.966271-2-xiqi2@huawei.com Fixes: b2c9e2fbba32 ("mm: make alloc_contig_range work at pageblock granularity") Signed-off-by: Qi Xi Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Cc: Johannes Weiner Cc: Kefeng Wang Cc: Michal Hocko Cc: Nanyong Sun Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Brendan Jackman Cc: --- mm/page_isolation.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mm/page_isolation.c b/mm/page_isolation.c index e5dfc7bf4944..e4ee998c00f3 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -388,13 +388,19 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, } if (PageBuddy(page)) { - int order = buddy_order(page); + unsigned int order = buddy_order_unsafe(page); - /* pageblock_isolate_and_move_free_pages() handled this */ - VM_WARN_ON_ONCE(pfn + (1 << order) > boundary_pfn); - - pfn += 1UL << order; - continue; + /* buddy_order_unsafe() is racy. Validate the order before shifting. */ + if (order <= MAX_PAGE_ORDER && + /* + * pageblock_isolate_and_move_free_pages() splits + * cross-boundary PageBuddy, verify it. + */ + pfn + (1UL << order) <= boundary_pfn) { + pfn += 1UL << order; + continue; + } + goto failed; } /* From 4c767c5dd9bc3521fed11e6fe1d52621fe5abf7b Mon Sep 17 00:00:00 2001 From: Qi Xi Date: Tue, 25 Aug 2026 20:05:49 +0800 Subject: [PATCH 020/539] mm/page_isolation: guard compound_order() against racing The PageCompound branch reads compound_head() without holding a reference. A racing split or free can cause compound_head() to return a stale pointer, and compound_nr() reads the order from that stale head, leading to out-of-range shifts and making the skip distance meaningless. Read the order explicitly with compound_order() and validate it is within MAX_FOLIO_ORDER before shifting. Also verify the derived head_pfn against the legitimate pfn: the head must not be past pfn, must be aligned to nr_pages, and pfn must fall within the compound page. Bail out with -EBUSY if any check fails. Link: https://lore.kernel.org/20260825120549.966271-3-xiqi2@huawei.com Fixes: b2c9e2fbba32 ("mm: make alloc_contig_range work at pageblock granularity") Signed-off-by: Qi Xi Signed-off-by: Andrew Morton Suggested-by: Zi Yan Reviewed-by: Zi Yan Cc: Brendan Jackman Cc: Johannes Weiner Cc: Kefeng Wang Cc: Michal Hocko Cc: Nanyong Sun Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: --- mm/page_isolation.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/mm/page_isolation.c b/mm/page_isolation.c index e4ee998c00f3..5aaad037384e 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -419,10 +419,28 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, if (PageCompound(page)) { struct page *head = compound_head(page); unsigned long head_pfn = page_to_pfn(head); - unsigned long nr_pages = compound_nr(head); + unsigned int order = compound_order(head); + unsigned long nr_pages; + + /* compound_order() is racy. Cap it at MAX_FOLIO_ORDER. */ + if (order > MAX_FOLIO_ORDER) + goto failed; + + nr_pages = 1UL << order; + + /* + * compound_head() is also racy, so the derived head_pfn + * needs additional checks to make sure it is valid. + * Otherwise, just fail the check. pfn comes from + * __first_valid_page() as a legitimate PFN, so use it to + * check head_pfn. + */ + if (head_pfn > pfn || !IS_ALIGNED(head_pfn, nr_pages) || + pfn - head_pfn >= nr_pages) + goto failed; if (head_pfn + nr_pages <= boundary_pfn || - PageHuge(page)) { + PageHuge(head)) { pfn = head_pfn + nr_pages; continue; } From 39da6961b2c95a6a546d20f0fd083fe1029b8781 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Tue, 25 Aug 2026 20:30:23 +0800 Subject: [PATCH 021/539] selftests/mm: fix incorrect skip output in pkey_sighandler_tests When pkeys is not supported, ksft_exit_skip() runs with ksft_plan already set, which takes the "ok N # SKIP" branch intended for skipping a single test case. The result is a TAP plan of 5 but only one result line. $ ./pkey_sighandler_tests TAP version 13 1..5 ok 1 # SKIP pkeys not supported # 1 skipped test(s) detected. Consider enabling relevant config options to improve coverage. # Planned tests != run tests (5 != 1) # Totals: pass:0 fail:0 xfail:0 xpass:0 skip:1 error:0 Move ksft_set_plan() after the skip check so ksft_exit_skip() takes the "1..0 # SKIP" branch, the correct TAP representation for skipping an entire test file. $ ./pkey_sighandler_tests TAP version 13 1..0 # SKIP pkeys not supported Link: https://lore.kernel.org/20260825123023.64418-1-zenghui.yu@linux.dev Signed-off-by: Zenghui Yu (Huawei) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/pkey_sighandler_tests.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index c218d0510a2a..74bf79a5399d 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -543,11 +543,12 @@ static void (*pkey_tests[])(void) = { int main(int argc, char *argv[]) { ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(pkey_tests)); if (!is_pkeys_supported()) ksft_exit_skip("pkeys not supported\n"); + ksft_set_plan(ARRAY_SIZE(pkey_tests)); + for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) { tracing_on(); (*pkey_tests[test_nr])(); From 48070b721a60c29c5a801cee8b9053c729e71eca Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:40 +0800 Subject: [PATCH 022/539] mm/sparse: relax struct mem_section size constraints Patch series "mm: Introduce section-based vmemmap optimization for HugeTLB", v6. This series is split out from the earlier, larger series "mm: Generalize HVO for HugeTLB and device DAX" [1]. While the parent series generalizes vmemmap optimization across HugeTLB and device DAX, this subset addresses a single, self-contained step: making the generic sparse-vmemmap code section-based optimization aware and switching HugeTLB bootmem pages to this path. HugeTLB vmemmap optimization currently has its own early boot setup path. It pre-populates optimized vmemmap mappings before the normal sparse-vmemmap population code runs, and sparsemem carries SPARSEMEM_VMEMMAP_PREINIT only to support that special case. That makes the HugeTLB vmemmap optimization path harder to share with other users of sparse-vmemmap optimization and leaves a fair amount of HugeTLB-specific boot-time state in the generic memory initialization flow. This series introduces section-based vmemmap optimization support in the sparse-vmemmap code and switches HugeTLB bootmem pages over to it. Instead of having HugeTLB pre-populate optimized vmemmap mappings itself, HugeTLB now records the compound page order in the corresponding memory sections. The generic sparse-vmemmap population path can then allocate or reuse shared tail vmemmap pages based on section metadata. The patches are organized as follows: - patches 1-2 prepare sparsemem and vmemmap optimization metadata - patches 3-8 teach the common sparse-vmemmap paths to use that state - patches 9-10 switch HugeTLB bootmem optimization to the section-based path - patches 11-17 clean up sparsemem and HugeTLB bootmem code that is no longer needed after the conversion This is intended to be the second smaller step toward the broader HVO generalization. The device DAX conversion and the wider HVO consolidation are left for follow-up series. This patch (of 17): struct mem_section is currently forced to a power-of-2 size so the section-to-root lookup can use a mask instead of a modulo. That requirement makes future extensions harder than necessary: adding a small field can require configuration-dependent padding or layout checks just to preserve the lookup scheme. Keep the lookup correct for any struct mem_section size by using a plain modulo instead. Do not leave the layout entirely unconstrained, though. Keep struct mem_section double-word aligned so modest size changes, such as adding another word-sized field on 64-bit systems, still keep a compact and efficient layout. If future fields grow the structure beyond that sweet spot, the lookup remains correct; only the exact layout efficiency changes. Link: https://lore.kernel.org/20260910063256.64386-2-songmuchun@bytedance.com Link: https://lore.kernel.org/all/20260513130542.35604-1-songmuchun@bytedance.com/ [1] Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Qi Zheng --- include/linux/mmzone.h | 11 +++-------- mm/sparse.c | 2 -- scripts/gdb/linux/mm.py | 6 ++---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 94f9c3ff5416..0a2428714108 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2027,13 +2027,9 @@ struct mem_section { * section. (see page_ext.h about this.) */ struct page_ext *page_ext; - unsigned long pad; #endif - /* - * WARNING: mem_section must be a power-of-2 in size for the - * calculation and use of SECTION_ROOT_MASK to make sense. - */ -}; +/* Sacrifice minor padding space for efficient lookup. */ +} __aligned(2 * sizeof(unsigned long)); #ifdef CONFIG_SPARSEMEM_EXTREME #define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section)) @@ -2043,7 +2039,6 @@ struct mem_section { #define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT) #define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT) -#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1) #ifdef CONFIG_SPARSEMEM_EXTREME extern struct mem_section **mem_section; @@ -2067,7 +2062,7 @@ static inline struct mem_section *__nr_to_section(unsigned long nr) if (!mem_section || !mem_section[root]) return NULL; #endif - return &mem_section[root][nr & SECTION_ROOT_MASK]; + return &mem_section[root][nr % SECTIONS_PER_ROOT]; } /* diff --git a/mm/sparse.c b/mm/sparse.c index 7c15406e77f5..c84b4c7b8c70 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -322,8 +322,6 @@ void __init sparse_init(void) unsigned long pnum_end, pnum_begin, map_count = 1; int nid_begin; - /* see include/linux/mmzone.h 'struct mem_section' definition */ - BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section))); memblocks_present(); if (compound_info_has_mask()) { diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py index 28d33624c38b..193a88d763ab 100644 --- a/scripts/gdb/linux/mm.py +++ b/scripts/gdb/linux/mm.py @@ -70,7 +70,6 @@ class x86_page_ops(): self.SECTIONS_PER_ROOT = 1 self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT) - self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1 try: self.SECTION_HAS_MEM_MAP = 1 << int(gdb.parse_and_eval('SECTION_HAS_MEM_MAP_BIT')) @@ -100,7 +99,7 @@ class x86_page_ops(): def __nr_to_section(self, nr): root = self.SECTION_NR_TO_ROOT(nr) mem_section = gdb.parse_and_eval("mem_section") - return mem_section[root][nr & self.SECTION_ROOT_MASK] + return mem_section[root][nr % self.SECTIONS_PER_ROOT] def pfn_to_section_nr(self, pfn): return pfn >> self.PFN_SECTION_SHIFT @@ -249,7 +248,6 @@ class aarch64_page_ops(): self.SECTIONS_PER_ROOT = 1 self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT) - self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1 self.SUBSECTION_SHIFT = 21 self.SEBSECTION_SIZE = 1 << self.SUBSECTION_SHIFT self.PFN_SUBSECTION_SHIFT = self.SUBSECTION_SHIFT - self.PAGE_SHIFT @@ -304,7 +302,7 @@ class aarch64_page_ops(): def __nr_to_section(self, nr): root = self.SECTION_NR_TO_ROOT(nr) mem_section = gdb.parse_and_eval("mem_section") - return mem_section[root][nr & self.SECTION_ROOT_MASK] + return mem_section[root][nr % self.SECTIONS_PER_ROOT] def pfn_to_section_nr(self, pfn): return pfn >> self.PFN_SECTION_SHIFT From 5e72deef11ba2d9110e3e4f1d9b2edf9e46495bb Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:41 +0800 Subject: [PATCH 023/539] mm/sparse-vmemmap: rename HVO order macros The macros VMEMMAP_TAIL_MIN_ORDER and NR_VMEMMAP_TAILS describe the order range where HVO can be applied, but their names tie that range to the tail-page cache implementation. Rename them with a VMEMMAP_OPTIMIZATION prefix and use the new names in the HVO paths. This makes the code describe the optimization requirements rather than the tail-page cache implementation detail. No functional change intended. Link: https://lore.kernel.org/20260910063256.64386-3-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/mmzone.h | 17 +++++++++-------- mm/hugetlb.c | 4 ++-- mm/hugetlb_vmemmap.c | 2 +- mm/sparse-vmemmap.c | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 0a2428714108..5fb9b37819d5 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -107,13 +107,14 @@ is_power_of_2(sizeof(struct page)) ? \ MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) -/* - * vmemmap optimization (like HVO) is only possible for page orders that fill - * two or more pages with struct pages. - */ -#define VMEMMAP_TAIL_MIN_ORDER (ilog2(2 * PAGE_SIZE / sizeof(struct page))) -#define __NR_VMEMMAP_TAILS (MAX_FOLIO_ORDER - VMEMMAP_TAIL_MIN_ORDER + 1) -#define NR_VMEMMAP_TAILS (__NR_VMEMMAP_TAILS > 0 ? __NR_VMEMMAP_TAILS : 0) +/* The number of struct pages covered by the retained vmemmap pages with HVO enabled. */ +#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES (PAGE_SIZE / sizeof(struct page)) +#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) + +#define __VMEMMAP_OPTIMIZATION_NR_ORDERS \ + (MAX_FOLIO_ORDER - VMEMMAP_OPTIMIZATION_MIN_ORDER + 1) +#define VMEMMAP_OPTIMIZATION_NR_ORDERS \ + (__VMEMMAP_OPTIMIZATION_NR_ORDERS > 0 ? __VMEMMAP_OPTIMIZATION_NR_ORDERS : 0) enum migratetype { MIGRATE_UNMOVABLE, @@ -1158,7 +1159,7 @@ struct zone { atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; #ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP - struct page *vmemmap_tails[NR_VMEMMAP_TAILS]; + struct page *vmemmap_tails[VMEMMAP_OPTIMIZATION_NR_ORDERS]; #endif } ____cacheline_internodealigned_in_smp; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e645d2d25bb6..b2aa988979e9 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3357,7 +3357,7 @@ void __init hugetlb_bootmem_struct_page_init(void) struct zone *zone; for_each_zone(zone) { - for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { + for (int i = 0; i < VMEMMAP_OPTIMIZATION_NR_ORDERS; i++) { struct page *tail, *p; unsigned int order; @@ -3365,7 +3365,7 @@ void __init hugetlb_bootmem_struct_page_init(void) if (!tail) continue; - order = i + VMEMMAP_TAIL_MIN_ORDER; + order = i + VMEMMAP_OPTIMIZATION_MIN_ORDER; p = page_to_virt(tail); /* * prep_and_add_bootmem_folios() can access pageblock diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 917db0984143..ae8fdaa42118 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -494,7 +494,7 @@ static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio * static struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) { - const unsigned int idx = order - VMEMMAP_TAIL_MIN_ORDER; + const unsigned int idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; struct page *tail, *p; int node = zone_to_nid(zone); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 5a2469fb1838..aa6a4a2fae98 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -329,12 +329,12 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * unsigned int idx; int node = zone_to_nid(zone); - if (WARN_ON_ONCE(order < VMEMMAP_TAIL_MIN_ORDER)) + if (WARN_ON_ONCE(order < VMEMMAP_OPTIMIZATION_MIN_ORDER)) return NULL; if (WARN_ON_ONCE(order > MAX_FOLIO_ORDER)) return NULL; - idx = order - VMEMMAP_TAIL_MIN_ORDER; + idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; tail = zone->vmemmap_tails[idx]; if (tail) return tail; From 89fb8ed75e3c4991dc596037b825041b91b4cfc5 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:42 +0800 Subject: [PATCH 024/539] mm/mm_init: skip initializing shared vmemmap tail pages memmap_init_range() initializes every struct page in the target range. For compound pages with vmemmap optimization, the tail struct pages are backed by a shared vmemmap page. Initializing those tail struct pages would overwrite the shared vmemmap page contents, requiring users such as HugeTLB to restore the metadata afterwards. Track the compound page order for HVO-backed sections and use that metadata to detect struct pages that fall into the shared tail vmemmap range. Skip those shared tail pages in memmap_init_range(), then initialize pageblock migratetypes for the processed range with a helper after the per-page initialization loop. Keep direct mem_section access inside sparse helpers. Expose pfn_to_section_compound_order() for callers that only need the order associated with a PFN. This lets memmap_init_range() skip shared tail vmemmap pages without exposing __pfn_to_section() to !SPARSEMEM builds. This is a preparatory change for consolidating handling across users of vmemmap optimization, and it also avoids redundant initialization of shared tail vmemmap pages during early boot. That early-boot benefit appears only once HugeTLB is switched to this common handling, since HugeTLB is the early-boot user that creates those shared tail vmemmap pages. Link: https://lore.kernel.org/20260910063256.64386-4-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/mmzone.h | 8 ++++++++ mm/mm_init.c | 40 +++++++++++++++++++++++----------------- mm/sparse.h | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 17 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 5fb9b37819d5..738bf3d27256 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2022,6 +2022,14 @@ struct mem_section { unsigned long section_mem_map; struct mem_section_usage *usage; +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + /* + * Normally, sections hold regular (order-0) pages. However, for + * sections with HVO enabled, this tracks the compound page order + * to enable deduplication of redundant vmemmap pages. + */ + unsigned int compound_page_order; +#endif #ifdef CONFIG_PAGE_EXTENSION /* * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use diff --git a/mm/mm_init.c b/mm/mm_init.c index 1533aebafb68..6655fe696e0f 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -677,21 +678,19 @@ static inline void fixup_hashdist(void) static inline void fixup_hashdist(void) {} #endif /* CONFIG_NUMA */ -#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) static __meminit void pageblock_migratetype_init_range(unsigned long pfn, - unsigned long nr_pages, int migratetype, bool atomic) + unsigned long nr_pages, int migratetype, bool isolate, bool atomic) { const unsigned long end = pfn + nr_pages; for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) { enum migratetype mt = kho_scratch_migratetype(pfn, migratetype); - init_pageblock_migratetype(pfn_to_page(pfn), mt, false); - if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION)) + init_pageblock_migratetype(pfn_to_page(pfn), mt, isolate); + if (!atomic && IS_ALIGNED(pfn, PFN_DOWN(SZ_1G))) cond_resched(); } } -#endif #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range(pg_data_t *pgdat) @@ -886,6 +885,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone } } + /* + * Vmemmap-optimizable PFNs are backed by shared tail struct pages, + * which have already been initialized during vmemmap population. + */ + if (vmemmap_optimizable_pfn(pfn)) { + const unsigned int order = pfn_to_section_compound_order(pfn); + + pfn = min(ALIGN(pfn, 1UL << order), end_pfn); + continue; + } + page = pfn_to_page(pfn); __init_single_page(page, pfn, zone, nid); if (context == MEMINIT_HOTPLUG) { @@ -897,19 +907,13 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone __SetPageOffline(page); } - /* - * Usually, we want to mark the pageblock MIGRATE_MOVABLE, - * such that unmovable allocations won't be scattered all - * over the place during system boot. - */ - if (pageblock_aligned(pfn)) { - enum migratetype mt = kho_scratch_migratetype(pfn, migratetype); - - init_pageblock_migratetype(page, mt, isolate_pageblock); + if (pageblock_aligned(pfn)) cond_resched(); - } pfn++; } + + pageblock_migratetype_init_range(start_pfn, pfn - start_pfn, migratetype, + isolate_pageblock, /* atomic */ false); } static void __init memmap_init_zone_range(struct zone *zone, @@ -1112,7 +1116,8 @@ void __ref memmap_init_zone_device(struct zone *zone, compound_nr_pages(pfn, altmap, pgmap)); } - pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false); + pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, + /* isolate */ false, /* atomic */ false); pr_debug("%s initialised %lu pages in %ums\n", __func__, nr_pages, jiffies_to_msecs(jiffies - start)); @@ -1921,7 +1926,8 @@ static void __init deferred_free_pages(unsigned long pfn, if (!nr_pages) return; - pageblock_migratetype_init_range(pfn, nr_pages, MIGRATE_MOVABLE, true); + pageblock_migratetype_init_range(pfn, nr_pages, MIGRATE_MOVABLE, + /* isolate */ false, /* atomic */ true); page = pfn_to_page(pfn); diff --git a/mm/sparse.h b/mm/sparse.h index 3b744667a7e6..da79c83adeae 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -10,6 +10,39 @@ #include +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return section->compound_page_order; +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return section_compound_order(__pfn_to_section(pfn)); +} +#else +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return 0; +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return 0; +} +#endif + +static inline bool vmemmap_optimizable_pfn(unsigned long pfn) +{ + const unsigned int order = pfn_to_section_compound_order(pfn); + const unsigned long nr_pages = 1UL << order; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; +} + /* * mm/sparse.c */ From 057847a7faa345b775d006d38dc42221ebe35907 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:43 +0800 Subject: [PATCH 025/539] mm/sparse-vmemmap: initialize shared tail vmemmap pages on allocation The shared tail vmemmap page allocated in vmemmap_get_tail() used to be left uninitialized, because memmap_init_range() would later overwrite it. That forced users such as HugeTLB to defer the initialization to their own setup paths. Now that memmap_init_range() skips shared tail vmemmap pages, initialize them immediately in vmemmap_get_tail() with init_compound_tail() instead. This adds initialization at the point where the shared tail page is allocated. The existing HugeTLB initialization remains necessary until the compound page order is set and memmap_init_range() starts skipping shared tails. It is removed when HugeTLB switches to the section-based mechanism. Link: https://lore.kernel.org/20260910063256.64386-5-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/sparse-vmemmap.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index aa6a4a2fae98..107215cf8488 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -338,19 +338,11 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * tail = zone->vmemmap_tails[idx]; if (tail) return tail; - - /* - * Only allocate the page, but do not initialize it. - * - * Any initialization done here will be overwritten by memmap_init(). - * - * hugetlb_bootmem_struct_page_init() will take care of initialization - * after memmap_init(). - */ - p = vmemmap_alloc_block_zero(PAGE_SIZE, node); if (!p) return NULL; + for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) + init_compound_tail(p + i, NULL, order, zone); tail = virt_to_page(p); zone->vmemmap_tails[idx] = tail; From ed3a33cd73da485f9de4ecaeada5b5cf51c073ef Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:44 +0800 Subject: [PATCH 026/539] mm/sparse-vmemmap: support section-based vmemmap accounting section_nr_vmemmap_pages() can account ordinary sections and DAX sections, but section-based vmemmap optimization stores the compound page order in struct mem_section and retains a different number of vmemmap pages. Teach section_nr_vmemmap_pages() to recognize section-based optimized sections and calculate their vmemmap page count from the stored compound page order and the HVO retained page count. Link: https://lore.kernel.org/20260910063256.64386-6-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/mmzone.h | 6 ++++-- mm/sparse-vmemmap.c | 10 ++++++---- mm/sparse.h | 16 ++++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 738bf3d27256..2c4f63e379da 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -107,8 +107,10 @@ is_power_of_2(sizeof(struct page)) ? \ MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) -/* The number of struct pages covered by the retained vmemmap pages with HVO enabled. */ -#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES (PAGE_SIZE / sizeof(struct page)) +/* The number of retained vmemmap pages with HVO enabled. */ +#define VMEMMAP_OPTIMIZATION_PAGES 1 +#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \ + (VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page)) #define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) #define __VMEMMAP_OPTIMIZATION_NR_ORDERS \ diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 107215cf8488..dea7179fbc90 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -649,24 +649,26 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn) static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - const unsigned int order = pgmap ? pgmap->vmemmap_shift : 0; + const struct mem_section *ms = __pfn_to_section(pfn); + const int order = pgmap ? pgmap->vmemmap_shift : section_compound_order(ms); + const int vmemmap_pages = pgmap ? VMEMMAP_RESERVE_NR : VMEMMAP_OPTIMIZATION_PAGES; const unsigned long pages_per_compound = 1UL << order; VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION); - if (!vmemmap_can_optimize(altmap, pgmap)) + if (!vmemmap_can_optimize(altmap, pgmap) && !section_vmemmap_optimizable(ms)) return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); if (order < PFN_SECTION_SHIFT) { VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound)); - return VMEMMAP_RESERVE_NR * nr_pages / pages_per_compound; + return vmemmap_pages * nr_pages / pages_per_compound; } VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)); if (IS_ALIGNED(pfn, pages_per_compound)) - return VMEMMAP_RESERVE_NR; + return vmemmap_pages; return 0; } diff --git a/mm/sparse.h b/mm/sparse.h index da79c83adeae..563fc1f4d717 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -43,6 +43,17 @@ static inline bool vmemmap_optimizable_pfn(unsigned long pfn) return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; } +static inline bool vmemmap_optimizable_order(unsigned int order) +{ + if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP)) + return false; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER; +} + /* * mm/sparse.c */ @@ -86,6 +97,11 @@ static inline size_t mem_section_usage_size(void) return struct_size_t(struct mem_section_usage, pageblock_flags, BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS)); } + +static inline bool section_vmemmap_optimizable(const struct mem_section *ms) +{ + return vmemmap_optimizable_order(section_compound_order(ms)); +} #else static inline void sparse_init(void) {} #endif /* CONFIG_SPARSEMEM */ From 485a1d2d53b732ce16d414dd483373ed53ef4b8c Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:45 +0800 Subject: [PATCH 027/539] mm/mm_init: factor out pfn_to_zone() pfn_to_zone() in hugetlb_vmemmap.c duplicates the zone lookup logic in __init_deferred_page(). Move it to mm_init.c, declare it in mm/mm_init.h, and reuse it from __init_deferred_page() and HugeTLB early vmemmap initialization instead of open-coding the zone walk there. Link: https://lore.kernel.org/20260910063256.64386-7-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/hugetlb_vmemmap.c | 17 ++--------------- mm/mm_init.c | 28 ++++++++++++++++++---------- mm/mm_init.h | 1 + 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index ae8fdaa42118..c48fcea076a5 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -19,6 +19,7 @@ #include #include "hugetlb_vmemmap.h" #include "internal.h" +#include "mm_init.h" /** * struct vmemmap_remap_walk - walk vmemmap page table @@ -744,20 +745,6 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m) return true; } -static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) -{ - struct zone *zone; - enum zone_type zone_type; - - for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { - zone = &NODE_DATA(nid)->node_zones[zone_type]; - if (zone_spans_pfn(zone, pfn)) - return zone; - } - - return NULL; -} - /* * Initialize memmap section for a gigantic page, HVO-style. */ @@ -787,7 +774,7 @@ void __init hugetlb_vmemmap_init_early(int nid) map = pfn_to_page(pfn); start = (unsigned long)map; end = start + hugetlb_vmemmap_size(m->hstate); - zone = pfn_to_zone(nid, pfn); + zone = pfn_to_zone(pfn, nid); if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate), zone, HUGETLB_VMEMMAP_RESERVE_SIZE)) diff --git a/mm/mm_init.c b/mm/mm_init.c index 6655fe696e0f..ee3bd7921768 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -692,6 +692,20 @@ static __meminit void pageblock_migratetype_init_range(unsigned long pfn, } } +struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid) +{ + pg_data_t *pgdat = NODE_DATA(nid); + + for (enum zone_type zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { + struct zone *zone = &pgdat->node_zones[zone_type]; + + if (zone_spans_pfn(zone, pfn)) + return zone; + } + + return NULL; +} + #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range(pg_data_t *pgdat) { @@ -750,20 +764,14 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) static void __meminit __init_deferred_page(unsigned long pfn, int nid) { - pg_data_t *pgdat = NODE_DATA(nid); - int zid; + struct zone *zone; if (early_page_initialised(pfn, nid)) return; - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - struct zone *zone = &pgdat->node_zones[zid]; - - if (zone_spans_pfn(zone, pfn)) - break; - } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); - + zone = pfn_to_zone(pfn, nid); + __init_single_page(pfn_to_page(pfn), pfn, + zone ? zone_idx(zone) : MAX_NR_ZONES, nid); if (pageblock_aligned(pfn)) { enum migratetype mt = kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); diff --git a/mm/mm_init.h b/mm/mm_init.h index 39f75df9be1c..c9fc35e7e9f1 100644 --- a/mm/mm_init.h +++ b/mm/mm_init.h @@ -39,6 +39,7 @@ void memmap_init_range(unsigned long size, int nid, unsigned long zone, enum meminit_context context, struct vmem_altmap *altmap, int migratetype, bool isolate_pageblock); +struct zone *pfn_to_zone(unsigned long pfn, int nid); #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ From 38fe068a2df033e3a2f48490744174761fa5f4fe Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:46 +0800 Subject: [PATCH 028/539] mm/sparse-vmemmap: move helpers ahead of future callers Prepare for section-based vmemmap optimization by moving helpers that follow-up changes will use. vmemmap_get_tail() will be called from the PTE population path. section_nr_vmemmap_pages() will be made visible outside the memory hotplug code and called from sparse_init_nid(). Move vmemmap_alloc_block_zero() together with vmemmap_get_tail(), since the tail helper depends on it. Move section_nr_vmemmap_pages() earlier into its own CONFIG_MEMORY_HOTPLUG block. That lets the later patch change its visibility and callers without also moving the function body. No functional change is intended. Link: https://lore.kernel.org/20260910063256.64386-8-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/sparse-vmemmap.c | 134 +++++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 65 deletions(-) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index dea7179fbc90..e2a2a5eab4dc 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -148,6 +148,75 @@ void __meminit vmemmap_verify(pte_t *pte, int node, start, end - 1); } +#ifdef CONFIG_MEMORY_HOTPLUG +static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, + struct vmem_altmap *altmap, struct dev_pagemap *pgmap) +{ + const struct mem_section *ms = __pfn_to_section(pfn); + const int order = pgmap ? pgmap->vmemmap_shift : section_compound_order(ms); + const int vmemmap_pages = pgmap ? VMEMMAP_RESERVE_NR : VMEMMAP_OPTIMIZATION_PAGES; + const unsigned long pages_per_compound = 1UL << order; + + VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); + VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION); + + if (!vmemmap_can_optimize(altmap, pgmap) && !section_vmemmap_optimizable(ms)) + return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); + + if (order < PFN_SECTION_SHIFT) { + VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound)); + return vmemmap_pages * nr_pages / pages_per_compound; + } + + VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)); + + if (IS_ALIGNED(pfn, pages_per_compound)) + return vmemmap_pages; + + return 0; +} +#endif + +static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) +{ + void *p = vmemmap_alloc_block(size, node); + + if (!p) + return NULL; + memset(p, 0, size); + + return p; +} + +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) +{ + struct page *p, *tail; + unsigned int idx; + int node = zone_to_nid(zone); + + if (WARN_ON_ONCE(order < VMEMMAP_OPTIMIZATION_MIN_ORDER)) + return NULL; + if (WARN_ON_ONCE(order > MAX_FOLIO_ORDER)) + return NULL; + + idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; + tail = zone->vmemmap_tails[idx]; + if (tail) + return tail; + p = vmemmap_alloc_block_zero(PAGE_SIZE, node); + if (!p) + return NULL; + for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) + init_compound_tail(p + i, NULL, order, zone); + + tail = virt_to_page(p); + zone->vmemmap_tails[idx] = tail; + + return tail; +} +#endif + static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, struct vmem_altmap *altmap, unsigned long ptpfn, unsigned long flags) @@ -181,17 +250,6 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in return pte; } -static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) -{ - void *p = vmemmap_alloc_block(size, node); - - if (!p) - return NULL; - memset(p, 0, size); - - return p; -} - static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) { pmd_t *pmd = pmd_offset(pud, addr); @@ -323,33 +381,6 @@ void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end, } #ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP -static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) -{ - struct page *p, *tail; - unsigned int idx; - int node = zone_to_nid(zone); - - if (WARN_ON_ONCE(order < VMEMMAP_OPTIMIZATION_MIN_ORDER)) - return NULL; - if (WARN_ON_ONCE(order > MAX_FOLIO_ORDER)) - return NULL; - - idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; - tail = zone->vmemmap_tails[idx]; - if (tail) - return tail; - p = vmemmap_alloc_block_zero(PAGE_SIZE, node); - if (!p) - return NULL; - for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) - init_compound_tail(p + i, NULL, order, zone); - - tail = virt_to_page(p); - zone->vmemmap_tails[idx] = tail; - - return tail; -} - int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end, unsigned int order, struct zone *zone, unsigned long headsize) @@ -646,33 +677,6 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn) } } -static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, - struct vmem_altmap *altmap, struct dev_pagemap *pgmap) -{ - const struct mem_section *ms = __pfn_to_section(pfn); - const int order = pgmap ? pgmap->vmemmap_shift : section_compound_order(ms); - const int vmemmap_pages = pgmap ? VMEMMAP_RESERVE_NR : VMEMMAP_OPTIMIZATION_PAGES; - const unsigned long pages_per_compound = 1UL << order; - - VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); - VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION); - - if (!vmemmap_can_optimize(altmap, pgmap) && !section_vmemmap_optimizable(ms)) - return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); - - if (order < PFN_SECTION_SHIFT) { - VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound)); - return vmemmap_pages * nr_pages / pages_per_compound; - } - - VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)); - - if (IS_ALIGNED(pfn, pages_per_compound)) - return vmemmap_pages; - - return 0; -} - static struct page * __meminit populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) From a64b8c60c6efce2df3772c18c4bf47d1f3548a5c Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:47 +0800 Subject: [PATCH 029/539] mm/sparse-vmemmap: support section-based vmemmap optimization Teach sparse-vmemmap population code to use the compound page order when deciding whether a vmemmap page can be optimized. With this information, the common sparse-vmemmap population path can allocate or reuse shared tail vmemmap pages directly instead of relying on HugeTLB-specific handling. This centralizes vmemmap optimization logic in the sparse-vmemmap code, based on section metadata, and prepares for sharing the same mechanism across different users of vmemmap optimization, including HugeTLB and DAX. Link: https://lore.kernel.org/20260910063256.64386-9-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/sparse-vmemmap.c | 46 +++++++++++++++++++++++++++++++++++++-------- mm/sparse.c | 4 ++-- mm/sparse.h | 7 +++++++ 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index e2a2a5eab4dc..faebf344cdac 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -148,8 +148,7 @@ void __meminit vmemmap_verify(pte_t *pte, int node, start, end - 1); } -#ifdef CONFIG_MEMORY_HOTPLUG -static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, +int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { const struct mem_section *ms = __pfn_to_section(pfn); @@ -175,7 +174,6 @@ static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long n return 0; } -#endif static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) { @@ -215,19 +213,44 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * return tail; } +#else +static inline struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) +{ + return NULL; +} #endif +static __meminit void *vmemmap_alloc_pte(unsigned long pfn, int node, + struct vmem_altmap *altmap) +{ + struct zone *zone; + struct page *page; + const unsigned int order = pfn_to_section_compound_order(pfn); + + if (!vmemmap_optimizable_pfn(pfn)) + return vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); + + zone = pfn_to_zone(pfn, node); + page = vmemmap_get_tail(order, zone); + if (!page) + return NULL; + + return page_address(page); +} + static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, struct vmem_altmap *altmap, unsigned long ptpfn, unsigned long flags) { pte_t *pte = pte_offset_kernel(pmd, addr); + unsigned long pfn = page_to_pfn((struct page *)addr); + if (pte_none(ptep_get(pte))) { pte_t entry; - void *p; if (ptpfn == (unsigned long)-1) { - p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); + void *p = vmemmap_alloc_pte(pfn, node, altmap); + if (!p) return NULL; ptpfn = PHYS_PFN(__pa(p)); @@ -246,7 +269,8 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in } entry = pfn_pte(ptpfn, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); - } + } else if (WARN_ON_ONCE(vmemmap_optimizable_pfn(pfn))) + return NULL; return pte; } @@ -435,6 +459,9 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, pmd_t *pmd; for (addr = start; addr < end; addr = next) { + unsigned long pfn = page_to_pfn((struct page *)addr); + const struct mem_section *ms = __pfn_to_section(pfn); + next = pmd_addr_end(addr, end); pgd = vmemmap_pgd_populate(addr, node); @@ -450,7 +477,7 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, return -ENOMEM; pmd = pmd_offset(pud, addr); - if (pmd_none(pmdp_get(pmd))) { + if (pmd_none(pmdp_get(pmd)) && !section_vmemmap_optimizable(ms)) { void *p; p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap); @@ -468,8 +495,11 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, */ return -ENOMEM; } - } else if (vmemmap_check_pmd(pmd, node, addr, next)) + } else if (vmemmap_check_pmd(pmd, node, addr, next)) { + if (WARN_ON_ONCE(section_vmemmap_optimizable(ms))) + return -EOPNOTSUPP; continue; + } if (vmemmap_populate_basepages(addr, next, node, altmap)) return -ENOMEM; } diff --git a/mm/sparse.c b/mm/sparse.c index c84b4c7b8c70..e6cb67ca9c8d 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -305,8 +305,8 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, nid, NULL, NULL); if (!map) panic("Failed to allocate memmap for section %lu\n", pnum); - memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), - PAGE_SIZE)); + memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION, + NULL, NULL)); sparse_init_early_section(nid, map, pnum, 0); } } diff --git a/mm/sparse.h b/mm/sparse.h index 563fc1f4d717..74dd79c51e75 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -111,8 +111,15 @@ static inline void sparse_init(void) {} */ #ifdef CONFIG_SPARSEMEM_VMEMMAP void sparse_init_subsection_map(void); +int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, + struct vmem_altmap *altmap, struct dev_pagemap *pgmap); #else static inline void sparse_init_subsection_map(void) {} +static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, + struct vmem_altmap *altmap, struct dev_pagemap *pgmap) +{ + return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); +} #endif /* CONFIG_SPARSEMEM_VMEMMAP */ #endif /* __MM_SPARSE_H */ From 74d2a4e89f19bac902d4139427143096ceeec60d Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:48 +0800 Subject: [PATCH 030/539] mm/sparse: initialize memory sections earlier Upcoming HugeTLB bootmem changes need sparsemem section metadata before the HugeTLB bootmem allocation path runs. The memory sections are initialized from sparse_init(), which is called too late for that setup. Move the code that initializes sparsemem section metadata for memblock ranges into mm_core_init_early(), before free_area_init() and the HugeTLB bootmem setup. Rename the helper to sparse_sections_init() so the new caller describes the sparsemem-specific initialization step. This is a preparatory change. Link: https://lore.kernel.org/20260910063256.64386-10-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/mm_init.c | 1 + mm/sparse.c | 9 +-------- mm/sparse.h | 2 ++ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index ee3bd7921768..304c88da5cce 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2642,6 +2642,7 @@ void __init mm_core_init_early(void) { kho_memory_init_early(); + sparse_sections_init(); free_area_init(); hugetlb_cma_reserve(); diff --git a/mm/sparse.c b/mm/sparse.c index e6cb67ca9c8d..428d81838ece 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -191,12 +191,7 @@ static void __init memory_present(int nid, unsigned long start, unsigned long en } } -/* - * Mark all memblocks as present using memory_present(). - * This is a convenience function that is useful to mark all of the systems - * memory as present during initialization. - */ -static void __init memblocks_present(void) +void __init sparse_sections_init(void) { unsigned long start, end; int i, nid; @@ -322,8 +317,6 @@ void __init sparse_init(void) unsigned long pnum_end, pnum_begin, map_count = 1; int nid_begin; - memblocks_present(); - if (compound_info_has_mask()) { VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0), MAX_FOLIO_VMEMMAP_ALIGN)); diff --git a/mm/sparse.h b/mm/sparse.h index 74dd79c51e75..e998347867f4 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -59,6 +59,7 @@ static inline bool vmemmap_optimizable_order(unsigned int order) */ #ifdef CONFIG_SPARSEMEM void sparse_init(void); +void sparse_sections_init(void); int sparse_index_init(unsigned long section_nr, int nid); static inline void sparse_init_one_section(struct mem_section *ms, @@ -104,6 +105,7 @@ static inline bool section_vmemmap_optimizable(const struct mem_section *ms) } #else static inline void sparse_init(void) {} +static inline void sparse_sections_init(void) {} #endif /* CONFIG_SPARSEMEM */ /* From b8abb1c0c40f1f82e8dbd7986a531d5229e62f01 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:49 +0800 Subject: [PATCH 031/539] mm/hugetlb: switch HugeTLB to section-based vmemmap optimization HugeTLB bootmem vmemmap optimization still carries its own early setup path, including pre-populating optimized mappings before the generic sparse-vmemmap code runs. Now that the section-based vmemmap optimization can derive HugeTLB vmemmap deduplication from section metadata, HugeTLB only needs to mark the bootmem huge page range with the appropriate order. The generic sparse-vmemmap population path can then allocate and map the shared tail vmemmap pages without any HugeTLB-specific early population code. Do that by recording the compound page order when a bootmem huge page is allocated and dropping the dedicated pre-HVO helpers and related special-casing. This removes duplicate early setup logic and switches HugeTLB to the section-based vmemmap optimization path. Link: https://lore.kernel.org/20260910063256.64386-11-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/hugetlb.h | 1 - include/linux/mm.h | 3 -- mm/hugetlb.c | 31 +++----------- mm/hugetlb_vmemmap.c | 91 ++++------------------------------------- mm/hugetlb_vmemmap.h | 16 +++----- mm/sparse-vmemmap.c | 31 -------------- mm/sparse.h | 30 ++++++++++++++ 7 files changed, 48 insertions(+), 155 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 16c4c4caa126..fe28f98e1b22 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -171,7 +171,6 @@ struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio); extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; -extern struct list_head huge_boot_pages[MAX_NUMNODES]; void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); diff --git a/include/linux/mm.h b/include/linux/mm.h index dd09c438fa23..441bd39eab73 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5159,9 +5159,6 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); int vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); -int vmemmap_populate_hvo(unsigned long start, unsigned long end, - unsigned int order, struct zone *zone, - unsigned long headsize); void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, unsigned long headsize); void vmemmap_populate_print_last(void); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index b2aa988979e9..52d4f7b4cbd7 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -52,6 +52,7 @@ #include "hugetlb_cma.h" #include "hugetlb_internal.h" #include "mm_init.h" +#include "sparse.h" #include int hugetlb_max_hstate __read_mostly; @@ -59,7 +60,7 @@ unsigned int default_hstate_idx; struct hstate hstates[HUGE_MAX_HSTATE]; __initdata nodemask_t hugetlb_bootmem_nodes; -__initdata struct list_head huge_boot_pages[MAX_NUMNODES]; +static struct list_head huge_boot_pages[MAX_NUMNODES] __initdata; /* * Due to ordering constraints across the init code for various @@ -3150,6 +3151,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) } else { list_add_tail(&m->list, &huge_boot_pages[nid]); m->flags |= HUGE_BOOTMEM_ZONES_VALID; + hugetlb_vmemmap_optimize_bootmem_page(m); /* * Only initialize the head struct page in memmap_init_reserved_pages, * rest of the struct pages will be initialized by the HugeTLB @@ -3310,6 +3312,8 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) * this folio. */ folio_set_hugetlb_vmemmap_optimized(folio); + section_set_compound_order_range(folio_pfn(folio), + folio_nr_pages(folio), 0); if (hugetlb_bootmem_page_earlycma(m)) folio_set_hugetlb_cma(folio); @@ -3353,31 +3357,6 @@ void __init hugetlb_bootmem_struct_page_init(void) .max_threads = num_node_state(N_MEMORY), .numa_aware = true, }; -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP - struct zone *zone; - - for_each_zone(zone) { - for (int i = 0; i < VMEMMAP_OPTIMIZATION_NR_ORDERS; i++) { - struct page *tail, *p; - unsigned int order; - - tail = zone->vmemmap_tails[i]; - if (!tail) - continue; - - order = i + VMEMMAP_OPTIMIZATION_MIN_ORDER; - p = page_to_virt(tail); - /* - * prep_and_add_bootmem_folios() can access pageblock - * flags on bootmem HugeTLB pages, so initialize the - * shared tail struct pages here before bootmem folios - * start using them. - */ - for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) - init_compound_tail(p + j, NULL, order, zone); - } - } -#endif padata_do_multithreaded(&job); } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index c48fcea076a5..d1b031dcd177 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -18,8 +18,7 @@ #include #include "hugetlb_vmemmap.h" -#include "internal.h" -#include "mm_init.h" +#include "sparse.h" /** * struct vmemmap_remap_walk - walk vmemmap page table @@ -706,95 +705,19 @@ void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head __hugetlb_vmemmap_optimize_folios(h, folio_list, true); } -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT - -/* Return true of a bootmem allocated HugeTLB page should be pre-HVO-ed */ -static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m) +void __init hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m) { - unsigned long section_size, psize, pmd_vmemmap_size; - phys_addr_t paddr; - - if (!READ_ONCE(vmemmap_optimize_enabled)) - return false; - - if (!hugetlb_vmemmap_optimizable(m->hstate)) - return false; - - psize = huge_page_size(m->hstate); - paddr = virt_to_phys(m); - - /* - * Pre-HVO only works if the bootmem huge page - * is aligned to the section size. - */ - section_size = (1UL << PA_SECTION_SHIFT); - if (!IS_ALIGNED(paddr, section_size) || - !IS_ALIGNED(psize, section_size)) - return false; - - /* - * The pre-HVO code does not deal with splitting PMDS, - * so the bootmem page must be aligned to the number - * of base pages that can be mapped with one vmemmap PMD. - */ - pmd_vmemmap_size = (PMD_SIZE / (sizeof(struct page))) << PAGE_SHIFT; - if (!IS_ALIGNED(paddr, pmd_vmemmap_size) || - !IS_ALIGNED(psize, pmd_vmemmap_size)) - return false; - - return true; -} - -/* - * Initialize memmap section for a gigantic page, HVO-style. - */ -void __init hugetlb_vmemmap_init_early(int nid) -{ - unsigned long psize, paddr, section_size; - unsigned long ns, i, pnum, pfn, nr_pages; - unsigned long start, end; - struct huge_bootmem_page *m = NULL; - void *map; + struct hstate *h = m->hstate; + unsigned long pfn = PHYS_PFN(__pa(m)); if (!READ_ONCE(vmemmap_optimize_enabled)) return; - section_size = (1UL << PA_SECTION_SHIFT); - - list_for_each_entry(m, &huge_boot_pages[nid], list) { - struct zone *zone; - - if (!vmemmap_should_optimize_bootmem_page(m)) - continue; - - nr_pages = pages_per_huge_page(m->hstate); - psize = nr_pages << PAGE_SHIFT; - paddr = virt_to_phys(m); - pfn = PHYS_PFN(paddr); - map = pfn_to_page(pfn); - start = (unsigned long)map; - end = start + hugetlb_vmemmap_size(m->hstate); - zone = pfn_to_zone(pfn, nid); - - if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate), - zone, HUGETLB_VMEMMAP_RESERVE_SIZE)) - panic("Failed to allocate memmap for HugeTLB page\n"); - memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE)); - - pnum = pfn_to_section_nr(pfn); - ns = psize / section_size; - - for (i = 0; i < ns; i++) { - sparse_init_early_section(nid, map, pnum, - SECTION_IS_VMEMMAP_PREINIT); - map += section_map_size(); - pnum++; - } - + section_set_compound_order_range(pfn, pages_per_huge_page(h), + huge_page_order(h)); + if (vmemmap_optimizable_order(pfn_to_section_compound_order(pfn))) m->flags |= HUGE_BOOTMEM_HVO; - } } -#endif static const struct ctl_table hugetlb_vmemmap_sysctls[] = { { diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 7ac49c52457d..20eb03df542a 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -9,8 +9,7 @@ #ifndef _LINUX_HUGETLB_VMEMMAP_H #define _LINUX_HUGETLB_VMEMMAP_H #include -#include -#include +#include "internal.h" /* * Reserve one vmemmap page, all vmemmap addresses are mapped to it. See @@ -27,10 +26,7 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h, void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio); void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list); void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list); -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -void hugetlb_vmemmap_init_early(int nid); -#endif - +void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m); static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h) { @@ -76,14 +72,14 @@ static inline void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, { } -static inline void hugetlb_vmemmap_init_early(int nid) -{ -} - static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h) { return 0; } + +static inline void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m) +{ +} #endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */ static inline bool hugetlb_vmemmap_optimizable(const struct hstate *h) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index faebf344cdac..8c2b9bed236d 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -32,8 +32,6 @@ #include #include -#include "hugetlb_vmemmap.h" - /* * Flags for vmemmap_populate_range and friends. */ @@ -404,34 +402,6 @@ void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end, } } -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP -int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end, - unsigned int order, struct zone *zone, - unsigned long headsize) -{ - unsigned long maddr; - struct page *tail; - pte_t *pte; - int node = zone_to_nid(zone); - - tail = vmemmap_get_tail(order, zone); - if (!tail) - return -ENOMEM; - - for (maddr = addr; maddr < addr + headsize; maddr += PAGE_SIZE) { - pte = vmemmap_populate_address(maddr, node, NULL, -1, 0); - if (!pte) - return -ENOMEM; - } - - /* - * Reuse the last page struct page mapped above for the rest. - */ - return vmemmap_populate_range(maddr, end, node, NULL, - page_to_pfn(tail), 0); -} -#endif - void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node, unsigned long addr, unsigned long next) { @@ -634,7 +604,6 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn, */ void __init sparse_vmemmap_init_nid_early(int nid) { - hugetlb_vmemmap_init_early(nid); } #endif diff --git a/mm/sparse.h b/mm/sparse.h index e998347867f4..d3a71ef4fad0 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -16,6 +16,26 @@ static inline unsigned int section_compound_order(const struct mem_section *sect return section->compound_page_order; } +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ + VM_WARN_ON(section_compound_order(section) && order && + section_compound_order(section) != order); + section->compound_page_order = order; +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ + unsigned long section_nr = pfn_to_section_nr(pfn); + + if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)) + return; + + for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++) + section_set_compound_order(__nr_to_section(section_nr + i), order); +} + static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) { return section_compound_order(__pfn_to_section(pfn)); @@ -26,6 +46,16 @@ static inline unsigned int section_compound_order(const struct mem_section *sect return 0; } +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ +} + static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) { return 0; From 328649715319855cf16dd7b6aaecb8c44eef4240 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:50 +0800 Subject: [PATCH 032/539] mm/sparse-vmemmap: remove SPARSEMEM_VMEMMAP_PREINIT support SPARSEMEM_VMEMMAP_PREINIT existed only to support HugeTLB's early vmemmap optimization setup. Now that HugeTLB bootmem vmemmap optimization uses the common section-based sparse-vmemmap path, the sparse initialization code no longer needs a separate pre-initialization mechanism for vmemmap population. Remove the related Kconfig symbols, section flag, and empty early hook, so present sections always go through the normal sparse setup path. Link: https://lore.kernel.org/20260910063256.64386-12-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Acked-by: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- arch/x86/Kconfig | 1 - fs/Kconfig | 1 - include/linux/mmzone.h | 25 ------------------------- mm/Kconfig | 5 ----- mm/sparse-vmemmap.c | 13 ------------- mm/sparse.c | 23 ++++++++--------------- 6 files changed, 8 insertions(+), 60 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 15fd9ec5ecac..7aa74bcc72f9 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -150,7 +150,6 @@ config X86 select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP if X86_64 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP if X86_64 - select ARCH_WANT_HUGETLB_VMEMMAP_PREINIT if X86_64 select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH select ARCH_WANT_IRQS_OFF_ACTIVATE_MM diff --git a/fs/Kconfig b/fs/Kconfig index e05917adcd60..d1c210c6508f 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -278,7 +278,6 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP def_bool HUGETLB_PAGE depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP depends on SPARSEMEM_VMEMMAP - select SPARSEMEM_VMEMMAP_PREINIT if ARCH_WANT_HUGETLB_VMEMMAP_PREINIT config HUGETLB_PMD_PAGE_TABLE_SHARING def_bool HUGETLB_PAGE diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 2c4f63e379da..f2d39a888eac 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2095,9 +2095,6 @@ enum { SECTION_IS_EARLY_BIT, #ifdef CONFIG_ZONE_DEVICE SECTION_TAINT_ZONE_DEVICE_BIT, -#endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT - SECTION_IS_VMEMMAP_PREINIT_BIT, #endif SECTION_MAP_LAST_BIT, }; @@ -2109,9 +2106,6 @@ enum { #ifdef CONFIG_ZONE_DEVICE #define SECTION_TAINT_ZONE_DEVICE BIT(SECTION_TAINT_ZONE_DEVICE_BIT) #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -#define SECTION_IS_VMEMMAP_PREINIT BIT(SECTION_IS_VMEMMAP_PREINIT_BIT) -#endif #define SECTION_MAP_MASK (~(BIT(SECTION_MAP_LAST_BIT) - 1)) #define SECTION_NID_SHIFT SECTION_MAP_LAST_BIT @@ -2166,24 +2160,6 @@ static inline int online_device_section(const struct mem_section *section) } #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -static inline int preinited_vmemmap_section(const struct mem_section *section) -{ - return (section && - (section->section_mem_map & SECTION_IS_VMEMMAP_PREINIT)); -} - -void sparse_vmemmap_init_nid_early(int nid); -#else -static inline int preinited_vmemmap_section(const struct mem_section *section) -{ - return 0; -} -static inline void sparse_vmemmap_init_nid_early(int nid) -{ -} -#endif - static inline int online_section_nr(unsigned long nr) { return online_section(__nr_to_section(nr)); @@ -2385,7 +2361,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #endif #else -#define sparse_vmemmap_init_nid_early(_nid) do {} while (0) #define pfn_in_present_section pfn_valid #endif /* CONFIG_SPARSEMEM */ diff --git a/mm/Kconfig b/mm/Kconfig index 604c58199acb..2c385f8b2944 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -461,8 +461,6 @@ config SPARSEMEM_VMEMMAP pfn_to_page and page_to_pfn operations. This is the most efficient option when sufficient kernel resources are available. -config SPARSEMEM_VMEMMAP_PREINIT - bool # # Select this config option from the architecture Kconfig, if it is preferred # to enable the feature of HugeTLB/dev_dax vmemmap optimization. @@ -473,9 +471,6 @@ config ARCH_WANT_OPTIMIZE_DAX_VMEMMAP config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP bool -config ARCH_WANT_HUGETLB_VMEMMAP_PREINIT - bool - config HAVE_MEMBLOCK_PHYS_MAP bool diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 8c2b9bed236d..f22d815d7af0 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -594,19 +594,6 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn, return pfn_to_page(pfn); } -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -/* - * This is called just before initializing sections for a NUMA node. - * Any special initialization that needs to be done before the - * generic initialization can be done from here. Sections that - * are initialized in hooks called from here will be skipped by - * the generic initialization. - */ -void __init sparse_vmemmap_init_nid_early(int nid) -{ -} -#endif - static void subsection_mask_set(unsigned long *map, unsigned long pfn, unsigned long nr_pages) { diff --git a/mm/sparse.c b/mm/sparse.c index 428d81838ece..f4f393a033a4 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -283,27 +283,20 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, if (sparse_usage_init(nid, map_count)) panic("Failed to allocate usemap for node %d\n", nid); - sparse_vmemmap_init_nid_early(nid); - for_each_present_section_nr(pnum_begin, pnum) { - struct mem_section *ms; unsigned long pfn = section_nr_to_pfn(pnum); + struct page *map; if (pnum >= pnum_end) break; - ms = __nr_to_section(pnum); - if (!preinited_vmemmap_section(ms)) { - struct page *map; - - map = __populate_section_memmap(pfn, PAGES_PER_SECTION, - nid, NULL, NULL); - if (!map) - panic("Failed to allocate memmap for section %lu\n", pnum); - memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION, - NULL, NULL)); - sparse_init_early_section(nid, map, pnum, 0); - } + map = __populate_section_memmap(pfn, PAGES_PER_SECTION, + nid, NULL, NULL); + if (!map) + panic("Failed to allocate memmap for section %lu\n", pnum); + memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION, + NULL, NULL)); + sparse_init_early_section(nid, map, pnum, 0); } sparse_usage_fini(); } From 1269621f9a870d283739f5fd929404e7cd9a7f0f Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:51 +0800 Subject: [PATCH 033/539] mm/sparse: inline usemap allocation into sparse_init_nid() After removing SPARSEMEM_VMEMMAP_PREINIT, sparse_init_nid() no longer needs the transient sparse_usagebuf state and its helper wrappers. Allocate the usemap buffer directly in sparse_init_nid(), pass it to sparse_init_one_section(), and drop sparse_usage_init(), sparse_usage_fini(), and sparse_init_early_section(). Link: https://lore.kernel.org/20260910063256.64386-13-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Qi Zheng Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/mmzone.h | 3 --- mm/sparse.c | 46 +++++++----------------------------------- 2 files changed, 7 insertions(+), 42 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index f2d39a888eac..6acc14b169bb 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2223,9 +2223,6 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long } #endif -void sparse_init_early_section(int nid, struct page *map, unsigned long pnum, - unsigned long flags); - #ifndef CONFIG_HAVE_ARCH_PFN_VALID /** * pfn_valid - check if there is a valid memory map entry for a PFN diff --git a/mm/sparse.c b/mm/sparse.c index f4f393a033a4..d19c173b40fe 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -234,42 +234,6 @@ void __weak __meminit vmemmap_populate_print_last(void) { } -static void *sparse_usagebuf __initdata; -static void *sparse_usagebuf_end __initdata; - -/* - * Helper function that is used for generic section initialization, and - * can also be used by any hooks added above. - */ -void __init sparse_init_early_section(int nid, struct page *map, - unsigned long pnum, unsigned long flags) -{ - BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end); - sparse_init_one_section(__nr_to_section(pnum), pnum, map, - sparse_usagebuf, SECTION_IS_EARLY | flags); - sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size(); -} - -static int __init sparse_usage_init(int nid, unsigned long map_count) -{ - unsigned long size; - - size = mem_section_usage_size() * map_count; - sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid); - if (!sparse_usagebuf) { - sparse_usagebuf_end = NULL; - return -ENOMEM; - } - - sparse_usagebuf_end = sparse_usagebuf + size; - return 0; -} - -static void __init sparse_usage_fini(void) -{ - sparse_usagebuf = sparse_usagebuf_end = NULL; -} - /* * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) * And number of present sections in this node is map_count. @@ -279,8 +243,11 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, unsigned long map_count) { unsigned long pnum; + struct mem_section_usage *usage; - if (sparse_usage_init(nid, map_count)) + usage = memblock_alloc_node(map_count * mem_section_usage_size(), + SMP_CACHE_BYTES, nid); + if (!usage) panic("Failed to allocate usemap for node %d\n", nid); for_each_present_section_nr(pnum_begin, pnum) { @@ -296,9 +263,10 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, panic("Failed to allocate memmap for section %lu\n", pnum); memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION, NULL, NULL)); - sparse_init_early_section(nid, map, pnum, 0); + sparse_init_one_section(__nr_to_section(pnum), pnum, map, usage, + SECTION_IS_EARLY); + usage = (void *)usage + mem_section_usage_size(); } - sparse_usage_fini(); } /* From 7d55d0a1187dc37f6e37d925efe79b02d3a27e8f Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:52 +0800 Subject: [PATCH 034/539] mm/sparse: remove section_map_size() section_map_size() no longer provides any shared logic. After the sparse-vmemmap changes, its only remaining user is the !CONFIG_SPARSEMEM_VMEMMAP path in __populate_section_memmap(), which can compute the size inline with PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION). Remove section_map_size() and inline the remaining calculation. Link: https://lore.kernel.org/20260910063256.64386-14-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Qi Zheng Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/mm.h | 1 - mm/sparse.c | 15 ++------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 441bd39eab73..b19711b6dbc6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5140,7 +5140,6 @@ static inline void print_vma_addr(char *prefix, unsigned long rip) } #endif -unsigned long section_map_size(void); struct page * __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); diff --git a/mm/sparse.c b/mm/sparse.c index d19c173b40fe..cc28bb41fdb1 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -208,23 +208,12 @@ void __init sparse_sections_init(void) memory_present(nid, start, end); } -#ifdef CONFIG_SPARSEMEM_VMEMMAP -unsigned long __init section_map_size(void) -{ - return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE); -} - -#else -unsigned long __init section_map_size(void) -{ - return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); -} - +#ifndef CONFIG_SPARSEMEM_VMEMMAP struct page __init *__populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - unsigned long size = section_map_size(); + const unsigned long size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false); } From 4d012c702c6896e7f7a6dbd2bdc678f1a2cfeb1d Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:53 +0800 Subject: [PATCH 035/539] mm/hugetlb: remove HUGE_BOOTMEM_HVO The HUGE_BOOTMEM_HVO flag tracked whether a bootmem huge page had already gone through the old early vmemmap optimization path. Now that HugeTLB uses section-based vmemmap optimization, that state is already reflected in the compound page order stored in section metadata. Remove HUGE_BOOTMEM_HVO and its helper, and use the section state directly when deciding whether to mark a folio as vmemmap-optimized. Link: https://lore.kernel.org/20260910063256.64386-15-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/hugetlb.h | 5 ++--- mm/hugetlb.c | 16 +++------------- mm/hugetlb_vmemmap.c | 2 -- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index fe28f98e1b22..3559041a5a57 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -675,9 +675,8 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -#define HUGE_BOOTMEM_HVO 0x0001 -#define HUGE_BOOTMEM_ZONES_VALID 0x0002 -#define HUGE_BOOTMEM_CMA 0x0004 +#define HUGE_BOOTMEM_ZONES_VALID BIT(0) +#define HUGE_BOOTMEM_CMA BIT(1) int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 52d4f7b4cbd7..1fab9e7c01b5 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3209,11 +3209,6 @@ static void __init hugetlb_folio_init_vmemmap(struct folio *folio, prep_compound_head(&folio->page, huge_page_order(h)); } -static bool __init hugetlb_bootmem_page_prehvo(struct huge_bootmem_page *m) -{ - return m->flags & HUGE_BOOTMEM_HVO; -} - static bool __init hugetlb_bootmem_page_earlycma(struct huge_bootmem_page *m) { return m->flags & HUGE_BOOTMEM_CMA; @@ -3288,6 +3283,7 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) { struct page *page = virt_to_page(m); struct folio *folio = (void *)page; + const unsigned long pfn = folio_pfn(folio); h = m->hstate; /* @@ -3305,15 +3301,9 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) HUGETLB_VMEMMAP_RESERVE_PAGES); init_new_hugetlb_folio(folio); - if (hugetlb_bootmem_page_prehvo(m)) - /* - * If pre-HVO was done, just set the - * flag, the HVO code will then skip - * this folio. - */ + if (vmemmap_optimizable_order(pfn_to_section_compound_order(pfn))) folio_set_hugetlb_vmemmap_optimized(folio); - section_set_compound_order_range(folio_pfn(folio), - folio_nr_pages(folio), 0); + section_set_compound_order_range(pfn, folio_nr_pages(folio), 0); if (hugetlb_bootmem_page_earlycma(m)) folio_set_hugetlb_cma(folio); diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index d1b031dcd177..fba0c5a7d46f 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -715,8 +715,6 @@ void __init hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m) section_set_compound_order_range(pfn, pages_per_huge_page(h), huge_page_order(h)); - if (vmemmap_optimizable_order(pfn_to_section_compound_order(pfn))) - m->flags |= HUGE_BOOTMEM_HVO; } static const struct ctl_table hugetlb_vmemmap_sysctls[] = { From eedb6c38351d37bb225849f4edb73e4d7a3592a0 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:54 +0800 Subject: [PATCH 036/539] mm/hugetlb: remove HUGE_BOOTMEM_CMA Track early CMA hugetlb pages from the hstate instead of storing a redundant bootmem flag. This removes the unused helper and keeps the bootmem metadata minimal. Link: https://lore.kernel.org/20260910063256.64386-16-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/hugetlb.h | 1 - mm/hugetlb.c | 14 ++++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 3559041a5a57..255a258f11d1 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -676,7 +676,6 @@ struct hstate { }; #define HUGE_BOOTMEM_ZONES_VALID BIT(0) -#define HUGE_BOOTMEM_CMA BIT(1) int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 1fab9e7c01b5..83b89764d70f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3133,7 +3133,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) */ INIT_LIST_HEAD(&m->list); m->hstate = h; - m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0; + m->flags = 0; /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */ if (!hugetlb_early_cma(h) && @@ -3209,11 +3209,6 @@ static void __init hugetlb_folio_init_vmemmap(struct folio *folio, prep_compound_head(&folio->page, huge_page_order(h)); } -static bool __init hugetlb_bootmem_page_earlycma(struct huge_bootmem_page *m) -{ - return m->flags & HUGE_BOOTMEM_CMA; -} - /* * memblock-allocated pageblocks might not have the migrate type set * if marked with the 'noinit' flag. Set it to the default (MIGRATE_MOVABLE) @@ -3305,9 +3300,6 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) folio_set_hugetlb_vmemmap_optimized(folio); section_set_compound_order_range(pfn, folio_nr_pages(folio), 0); - if (hugetlb_bootmem_page_earlycma(m)) - folio_set_hugetlb_cma(folio); - list_add(&folio->lru, &folio_list); /* @@ -3318,7 +3310,9 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) * For CMA pages, this is done in init_cma_pageblock * (via hugetlb_bootmem_init_migratetype), so skip it here. */ - if (!folio_test_hugetlb_cma(folio)) + if (hugetlb_early_cma(h)) + folio_set_hugetlb_cma(folio); + else adjust_managed_page_count(page, pages_per_huge_page(h)); cond_resched(); } From a28e0348838fe03b5e3de82c1c0f74d7d37d866c Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:55 +0800 Subject: [PATCH 037/539] mm/hugetlb: localize struct huge_bootmem_page struct huge_bootmem_page is only used by hugetlb boot-time allocation code, but its definition currently lives in mm/internal.h because hugetlb_vmemmap_optimize_bootmem_page() takes it as an argument. This exposes a hugetlb-specific internal type more broadly than needed. Change hugetlb_vmemmap_optimize_bootmem_page() to take the information it actually needs. With that interface, mm/hugetlb_vmemmap.h no longer needs to include mm/internal.h, and struct huge_bootmem_page can move into mm/hugetlb.c. Link: https://lore.kernel.org/20260910063256.64386-17-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/hugetlb.c | 8 +++++++- mm/hugetlb_vmemmap.c | 9 +++------ mm/hugetlb_vmemmap.h | 5 ++--- mm/internal.h | 7 ------- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 83b89764d70f..a0fe9106374d 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -55,6 +55,12 @@ #include "sparse.h" #include +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; +}; + int hugetlb_max_hstate __read_mostly; unsigned int default_hstate_idx; struct hstate hstates[HUGE_MAX_HSTATE]; @@ -3151,7 +3157,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) } else { list_add_tail(&m->list, &huge_boot_pages[nid]); m->flags |= HUGE_BOOTMEM_ZONES_VALID; - hugetlb_vmemmap_optimize_bootmem_page(m); + hugetlb_vmemmap_optimize_bootmem_page(pfn, huge_page_order(h)); /* * Only initialize the head struct page in memmap_init_reserved_pages, * rest of the struct pages will be initialized by the HugeTLB diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index fba0c5a7d46f..f977d0a7e002 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -19,6 +19,7 @@ #include #include "hugetlb_vmemmap.h" #include "sparse.h" +#include "internal.h" /** * struct vmemmap_remap_walk - walk vmemmap page table @@ -705,16 +706,12 @@ void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head __hugetlb_vmemmap_optimize_folios(h, folio_list, true); } -void __init hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m) +void __init hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order) { - struct hstate *h = m->hstate; - unsigned long pfn = PHYS_PFN(__pa(m)); - if (!READ_ONCE(vmemmap_optimize_enabled)) return; - section_set_compound_order_range(pfn, pages_per_huge_page(h), - huge_page_order(h)); + section_set_compound_order_range(pfn, 1UL << order, order); } static const struct ctl_table hugetlb_vmemmap_sysctls[] = { diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 20eb03df542a..464192e32dec 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -9,7 +9,6 @@ #ifndef _LINUX_HUGETLB_VMEMMAP_H #define _LINUX_HUGETLB_VMEMMAP_H #include -#include "internal.h" /* * Reserve one vmemmap page, all vmemmap addresses are mapped to it. See @@ -26,7 +25,7 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h, void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio); void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list); void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list); -void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m); +void hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order); static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h) { @@ -77,7 +76,7 @@ static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate return 0; } -static inline void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m) +static inline void hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order) { } #endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */ diff --git a/mm/internal.h b/mm/internal.h index da833cafcd59..5cc220db9076 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,13 +23,6 @@ #include "vma.h" struct folio_batch; -struct hstate; - -struct huge_bootmem_page { - struct list_head list; - struct hstate *hstate; - unsigned long flags; -}; /* mm/workingset.c */ bool workingset_test_recent(void *shadow, bool file, bool *workingset, From 3de632e082537d613c94db82a24899d3b7288d3c Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 10 Sep 2026 14:32:56 +0800 Subject: [PATCH 038/539] mm/hugetlb: localize HUGE_BOOTMEM_ZONES_VALID HUGE_BOOTMEM_ZONES_VALID is only used by the huge_bootmem_page flag handling in mm/hugetlb.c. Keep the definition next to that private data structure instead of exposing it through the public hugetlb header. No functional change is intended. Link: https://lore.kernel.org/20260910063256.64386-18-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Qi Zheng Cc: David Hildenbrand (Arm) Cc: David Laight Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/hugetlb.h | 2 -- mm/hugetlb.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 255a258f11d1..900c95e346b2 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -675,8 +675,6 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -#define HUGE_BOOTMEM_ZONES_VALID BIT(0) - int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a0fe9106374d..1b9b9985dc03 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -55,6 +55,8 @@ #include "sparse.h" #include +#define HUGE_BOOTMEM_ZONES_VALID BIT(0) + struct huge_bootmem_page { struct list_head list; struct hstate *hstate; From 36b0ee4aac60e8bf394c359a0a988319289c9f66 Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 25 Aug 2026 16:57:54 +0800 Subject: [PATCH 039/539] selftests/mm: emit TAP header in uffd-wp-mremap Patch series "selftests/mm: TAP output and global-state fixes", v4. uffd-wp-mremap and mremap_test never print the TAP header (and mremap_test skips with a bare exit(KSFT_SKIP) rather than a KTAP skip), so their output is not valid KTAP; hugetlb-soft-offline toggles enable_soft_offline during the run and leaves it disabled afterwards. This patch (of 3): uffd-wp-mremap calls ksft_set_plan() without ksft_print_header(), so its output is not valid KTAP. Add the header, like the sibling uffd tests (uffd-stress, uffd-unit-tests). Link: https://lore.kernel.org/20260825085756.63030-1-husong@kylinos.cn Link: https://lore.kernel.org/20260825085756.63030-2-husong@kylinos.cn Signed-off-by: Song Hu Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Sarthak Sharma Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: Muhammad Usama Anjum Tested-by: Muhammad Usama Anjum Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Peter Xu Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/uffd-wp-mremap.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/mm/uffd-wp-mremap.c b/tools/testing/selftests/mm/uffd-wp-mremap.c index c973d6722720..572c2516e874 100644 --- a/tools/testing/selftests/mm/uffd-wp-mremap.c +++ b/tools/testing/selftests/mm/uffd-wp-mremap.c @@ -347,6 +347,8 @@ int main(int argc, char **argv) struct thp_settings settings; int i, j, plan = 0; + ksft_print_header(); + hugepage_save_settings(true, true); check_uffd_wp_feature_supported(); From fb2f274a79c7e6ba91c111ed6d92fe128f6b5772 Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 25 Aug 2026 16:57:55 +0800 Subject: [PATCH 040/539] selftests/mm: emit TAP header and use TAP skip in mremap_test mremap_test calls ksft_set_plan() without ksft_print_header(), and its get_mmap_min_addr() skip path uses a bare exit(KSFT_SKIP) that prints no TAP line, so its output is not valid KTAP. Add the header and switch the skip to ksft_exit_skip(). Also fix two more KTAP compliance issues spotted in review: - get_mmap_min_addr() calls strerror(errno) after fclose(), which may clobber errno; save errno before fclose() instead. - Some ksft_*() messages embed "\n\t", so the text after each embedded newline is printed without the "# " prefix. Split those into separate messages. And cache mmap_min_addr in main() before ksft_set_plan(), so that the skip paths in get_mmap_min_addr() are taken before the plan is set; a skip after the plan leaves the run with fewer tests than planned. Link: https://lore.kernel.org/20260825085756.63030-3-husong@kylinos.cn Signed-off-by: Song Hu Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Sarthak Sharma Reviewed-by: Muhammad Usama Anjum Tested-by: Muhammad Usama Anjum Acked-by: Lorenzo Stoakes (ARM) Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Michal Hocko Cc: Peter Xu Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/mremap_test.c | 43 ++++++++++++++---------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 779ef2d5f1b9..97abf4713cc5 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -111,18 +111,17 @@ static unsigned long long get_mmap_min_addr(void) return addr; fp = fopen("/proc/sys/vm/mmap_min_addr", "r"); - if (fp == NULL) { - ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", - strerror(errno)); - exit(KSFT_SKIP); - } + if (!fp) + ksft_exit_skip("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", + strerror(errno)); n_matched = fscanf(fp, "%llu", &addr); if (n_matched != 1) { - ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", - strerror(errno)); + int err = errno; + fclose(fp); - exit(KSFT_SKIP); + ksft_exit_skip("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", + strerror(err)); } fclose(fp); @@ -1165,10 +1164,11 @@ static void run_mremap_test_case(struct test test_case, int *failures, rand_addr); if (remap_time < 0) { - if (test_case.expect_failure) - ksft_test_result_xfail("%s\n\tExpected mremap failure\n", - test_case.name); - else { + if (test_case.expect_failure) { + ksft_print_msg("%s: expected mremap failure\n", + test_case.name); + ksft_test_result_xfail("%s\n", test_case.name); + } else { ksft_test_result_fail("%s\n", test_case.name); *failures += 1; } @@ -1178,11 +1178,13 @@ static void run_mremap_test_case(struct test test_case, int *failures, * was faulted in. */ if (threshold_mb == VALIDATION_NO_THRESHOLD || - test_case.config.region_size <= threshold_mb * _1MB) - ksft_test_result_pass("%s\n\tmremap time: %12lldns\n", - test_case.name, remap_time); - else + test_case.config.region_size <= threshold_mb * _1MB) { + ksft_print_msg("%s: mremap time: %12lldns\n", + test_case.name, remap_time); ksft_test_result_pass("%s\n", test_case.name); + } else { + ksft_test_result_pass("%s\n", test_case.name); + } } } @@ -1251,13 +1253,18 @@ int main(int argc, char **argv) time_t t; FILE *maps_fp; + ksft_print_header(); + + get_mmap_min_addr(); + pattern_seed = (unsigned int) time(&t); if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0) exit(EXIT_FAILURE); - ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", - threshold_mb, pattern_seed); + ksft_print_msg("Test configs:\n"); + ksft_print_msg("threshold_mb=%u\n", threshold_mb); + ksft_print_msg("pattern_seed=%u\n", pattern_seed); /* * set preallocated random array according to test configs; see the From c0f106b5183e9614bd44bca3ffaea1b428532994 Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 25 Aug 2026 16:57:56 +0800 Subject: [PATCH 041/539] selftests/mm: restore enable_soft_offline in hugetlb-soft-offline hugetlb-soft-offline toggles /proc/sys/vm/enable_soft_offline between 1 and 0 (test_soft_offline_common(1) then (0)) and leaves it at 0 when it finishes, silently disabling soft offlining for the whole system after the run. Save the original value before the test and restore it from an atexit() handler, as hugepage_restore_settings_atexit() in hugepage_settings.c already does. Use read_num()/write_num() from vm_util instead of hand-rolled popen()/fopen() helpers. The restore handler must not call write_num(): on failure it re-enters exit() through ksft_exit_fail_msg(), which is undefined behavior from inside an atexit handler. A non-root run hits it directly - the restore write fails the same way the write that triggered the exit did. Restore with plain open()/write(), best effort. Link: https://lore.kernel.org/20260825085756.63030-4-husong@kylinos.cn Signed-off-by: Song Hu Signed-off-by: Andrew Morton Reviewed-by: Muhammad Usama Anjum Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Peter Xu Cc: Sarthak Sharma Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- .../selftests/mm/hugetlb-soft-offline.c | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/tools/testing/selftests/mm/hugetlb-soft-offline.c b/tools/testing/selftests/mm/hugetlb-soft-offline.c index bc202e4ed2bd..4af9d3db7b5b 100644 --- a/tools/testing/selftests/mm/hugetlb-soft-offline.c +++ b/tools/testing/selftests/mm/hugetlb-soft-offline.c @@ -11,6 +11,7 @@ #define _GNU_SOURCE #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include "kselftest.h" +#include "vm_util.h" #include "hugepage_settings.h" #ifndef MADV_SOFT_OFFLINE @@ -31,6 +33,8 @@ #define EPREFIX " !!! " +#define ENABLE_SOFT_OFFLINE_PATH "/proc/sys/vm/enable_soft_offline" + static int do_soft_offline(int fd, size_t len, int expect_errno) { char *filemap = NULL; @@ -77,26 +81,29 @@ untruncate: return ret; } -static int set_enable_soft_offline(int value) +static unsigned long orig_enable_soft_offline = -1UL; + +/* + * Runs from an atexit handler, so it must not call anything that + * exits on failure: write_num() would re-enter exit() through + * ksft_exit_fail_msg(). + */ +static void restore_enable_soft_offline(void) { - char cmd[256] = {0}; - FILE *cmdfile = NULL; + char buf[24]; + int fd, len; - if (value != 0 && value != 1) - return -EINVAL; + if (orig_enable_soft_offline == -1UL) + return; - sprintf(cmd, "echo %d > /proc/sys/vm/enable_soft_offline", value); - cmdfile = popen(cmd, "r"); - - if (cmdfile) - ksft_print_msg("enable_soft_offline => %d\n", value); - else { - ksft_perror(EPREFIX "failed to set enable_soft_offline"); - return errno; - } - - pclose(cmdfile); - return 0; + len = snprintf(buf, sizeof(buf), "%lu", orig_enable_soft_offline); + fd = open(ENABLE_SOFT_OFFLINE_PATH, O_WRONLY); + if (fd < 0) + return; + if (write(fd, buf, len) != len) + ksft_print_msg("failed to restore enable_soft_offline: %s\n", + strerror(errno)); + close(fd); } static int create_hugetlbfs_file(struct statfs *file_stat) @@ -145,10 +152,7 @@ static void test_soft_offline_common(int enable_soft_offline) hugepagesize_kb = file_stat.f_bsize / 1024; ksft_print_msg("Hugepagesize is %ldkB\n", hugepagesize_kb); - if (set_enable_soft_offline(enable_soft_offline) != 0) { - close(fd); - ksft_exit_fail_msg("Failed to set enable_soft_offline\n"); - } + write_num(ENABLE_SOFT_OFFLINE_PATH, enable_soft_offline); nr_hugepages_before = hugetlb_nr_default_pages(); @@ -192,6 +196,9 @@ int main(int argc, char **argv) ksft_set_plan(2); + orig_enable_soft_offline = read_num(ENABLE_SOFT_OFFLINE_PATH); + atexit(restore_enable_soft_offline); + test_soft_offline_common(1); test_soft_offline_common(0); From a9ce28ef2a10bd1cbe6998713a3cf45d3f17ffd9 Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Sun, 23 Aug 2026 12:40:52 +0800 Subject: [PATCH 042/539] mm/hugetlb: warn instead of silently bailing gigantic pages without runtime support remove_hugetlb_folio() and __update_and_free_hugetlb_folio() silently return for gigantic hstates that lack runtime freeing support. All callers should already filter such hstates upstream, so turn the silent bail into a VM_WARN_ON_ONCE to catch caller regressions instead of masking them. Link: https://lore.kernel.org/20260823044118.1097121-3-xialonglong2025@163.com Signed-off-by: Longlong Xia Signed-off-by: Andrew Morton Acked-by: Muchun Song Assisted-by: Codex:gpt-5.6-sol Cc: David Hildenbrand Cc: Miaohe Lin Cc: Michal Hocko Cc: Oscar Salvador --- mm/hugetlb.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 1b9b9985dc03..76857cdbac1b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1396,8 +1396,11 @@ void remove_hugetlb_folio(struct hstate *h, struct folio *folio, VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio); lockdep_assert_held(&hugetlb_lock); - if (hstate_is_gigantic_no_runtime(h)) + if (hstate_is_gigantic_no_runtime(h)) { + /* Callers must filter gigantic_no_runtime upstream. */ + VM_WARN_ON_ONCE(1); return; + } list_del(&folio->lru); @@ -1458,8 +1461,11 @@ static void __update_and_free_hugetlb_folio(struct hstate *h, { bool clear_flag = folio_test_hugetlb_vmemmap_optimized(folio); - if (hstate_is_gigantic_no_runtime(h)) + if (hstate_is_gigantic_no_runtime(h)) { + /* Callers must filter gigantic_no_runtime upstream. */ + VM_WARN_ON_ONCE(1); return; + } /* * If we don't know which subpages are hwpoisoned, we can't free From be7dacb9c1d48280aeae0830a045f4046c6ef3e2 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 12:28:27 +0300 Subject: [PATCH 043/539] set_memory: add number of pages parameter to set_direct_map APIs Patch series "arch, mm/execmem: resolve confusion about set_direct_map_valid_noflush()", v3. Recent discussion about implementation of execmem's ROX caches on arm64 revealed a confusion about how set_direct_map_valid_noflush() implemented on different architectures. On arm64 it sets or clears the PTE_VALID bit marking a PTE as present or not present. On other architectures it's a range version of set_direct_map_invalid_noflush() and set_direct_map_default_noflush() Unlike arm64::set_direct_map_valid_noflush(), set_direct_map_default_noflush() not only marks PTE as present, but also sets its default protection mode. Other than that, initial design of execmem ROX caches didn't rely on restoration of large mappings that's now available on x86, but completely removed the memory allocated for the ROX cache from the direct map to ensure that large mappings are not split. This precluded usage of VM_FLUSH_RESET_PERMS for the ROX cache allocations and required execmem to implement manipulation of the direct map alias. Current implementation of ROX caches does not remove the direct map alias but simply calls set_memory_rox() that updates the permissions in both vmalloc address space and the direct map and relies on collapse_large_pages() in x86 CPA to keep large mappings. This allow using VM_FLUSH_RESET_PERMS for execmem ROX cache allocations with small adjustments to set_direct_map APIs and vmalloc::reset_perms() behaviour: adding number of pages parameter to set_direct_map APIs and making resetting of the direct map permissions in vmalloc VMAP_HUGE friendly. Implement these adjustments, make execmem always use VM_FLUSH_RESET_PERMS and revert set_direct_map_valid_noflush() changes. This patch (of 6): When set_direct_map APIs were introduced by the commit d253ca0c3865 ("x86/mm/cpa: Add set_direct_map_*() functions") the single page parameter made sense because the initial callers (vmalloc and hibernation) had sets of unsorted struct pages that required changes of their mappings in the direct map. Since there is an increasing demand for direct map manipulation and it is also desirable to be able to update larger physically contiguous mappings, for example an entire large folio, extend set_direct_map APIs to receive number of pages parameter. As there is still only a handful of callers, change the existing functions directly and update all the call sites rather than adding wrappers for single page case. Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-0-949b64a9f755@kernel.org Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-1-949b64a9f755@kernel.org Link: https://lore.kernel.org/all/20260611130144.1385343-4-abarnas@google.com [1] Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Kevin Brodsky Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Huacai Chen Cc: Ingo Molnar Cc: Len Brown Cc: Palmer Dabbelt Cc: Peter Zijlstra Cc: "Rafael J. Wysocki" Cc: Ryan Roberts Cc: Sven Schnelle Cc: "Uladzislau Rezki (Sony)" Cc: Vasily Gorbik Cc: WANG Xuerui Cc: Will Deacon Cc: Dev Jain --- arch/arm64/include/asm/set_memory.h | 4 ++-- arch/arm64/mm/pageattr.c | 8 ++++---- arch/loongarch/include/asm/set_memory.h | 4 ++-- arch/loongarch/mm/pageattr.c | 8 ++++---- arch/riscv/include/asm/set_memory.h | 4 ++-- arch/riscv/mm/pageattr.c | 8 ++++---- arch/s390/include/asm/set_memory.h | 4 ++-- arch/s390/mm/pageattr.c | 8 ++++---- arch/x86/include/asm/set_memory.h | 4 ++-- arch/x86/mm/pat/set_memory.c | 8 ++++---- include/linux/set_memory.h | 6 ++++-- kernel/power/snapshot.c | 4 ++-- mm/secretmem.c | 6 +++--- mm/vmalloc.c | 5 +++-- 14 files changed, 42 insertions(+), 39 deletions(-) diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h index 90f61b17275e..b07fd4e026ea 100644 --- a/arch/arm64/include/asm/set_memory.h +++ b/arch/arm64/include/asm/set_memory.h @@ -11,8 +11,8 @@ bool can_set_direct_map(void); int set_memory_valid(unsigned long addr, int numpages, int enable); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages); +int set_direct_map_default_noflush(struct page *page, unsigned int numpages); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index bbe98ac9ad8c..db8d60a84d14 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -251,7 +251,7 @@ int set_memory_valid(unsigned long addr, int numpages, int enable) __pgprot(PTE_PRESENT_VALID_KERNEL)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages) { pgprot_t clear_mask = __pgprot(PTE_PRESENT_VALID_KERNEL); pgprot_t set_mask = __pgprot(PTE_PRESENT_INVALID); @@ -260,10 +260,10 @@ int set_direct_map_invalid_noflush(struct page *page) return 0; return update_range_prot((unsigned long)page_address(page), - PAGE_SIZE, set_mask, clear_mask); + PAGE_SIZE * numpages, set_mask, clear_mask); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int numpages) { pgprot_t set_mask = __pgprot(PTE_PRESENT_VALID_KERNEL | PTE_WRITE); pgprot_t clear_mask = __pgprot(PTE_PRESENT_INVALID | PTE_RDONLY); @@ -272,7 +272,7 @@ int set_direct_map_default_noflush(struct page *page) return 0; return update_range_prot((unsigned long)page_address(page), - PAGE_SIZE, set_mask, clear_mask); + PAGE_SIZE * numpages, set_mask, clear_mask); } static int __set_memory_enc_dec(unsigned long addr, diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h index 55dfaefd02c8..563aab92896e 100644 --- a/arch/loongarch/include/asm/set_memory.h +++ b/arch/loongarch/include/asm/set_memory.h @@ -15,8 +15,8 @@ int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); bool kernel_page_present(struct page *page); -int set_direct_map_default_noflush(struct page *page); -int set_direct_map_invalid_noflush(struct page *page); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); #endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index 614ccc7afccb..43ad2a104f19 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -198,24 +198,24 @@ bool kernel_page_present(struct page *page) return pte_present(ptep_get(pte)); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { unsigned long addr = (unsigned long)page_address(page); if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0)); + return __set_memory(addr, nr, PAGE_KERNEL, __pgprot(0)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { unsigned long addr = (unsigned long)page_address(page); if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); + return __set_memory(addr, nr, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index ef59e1716a2c..db1d0ed82b69 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -40,8 +40,8 @@ static inline int set_kernel_memory(char *startp, char *endp, } #endif -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 3f76db3d2769..20ef95b1d0c3 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -374,15 +374,15 @@ int set_memory_nx(unsigned long addr, int numpages) return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_address(page), 1, + return __set_memory((unsigned long)page_address(page), nr, __pgprot(0), __pgprot(_PAGE_PRESENT)); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_address(page), 1, + return __set_memory((unsigned long)page_address(page), nr, PAGE_KERNEL, __pgprot(_PAGE_EXEC)); } diff --git a/arch/s390/include/asm/set_memory.h b/arch/s390/include/asm/set_memory.h index 94092f4ae764..6b0aa9147ed8 100644 --- a/arch/s390/include/asm/set_memory.h +++ b/arch/s390/include/asm/set_memory.h @@ -60,8 +60,8 @@ __SET_MEMORY_FUNC(set_memory_rox, SET_MEMORY_RO | SET_MEMORY_X) __SET_MEMORY_FUNC(set_memory_rwnx, SET_MEMORY_RW | SET_MEMORY_NX) __SET_MEMORY_FUNC(set_memory_4k, SET_MEMORY_4K) -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 1e202e3d08e7..7549543d6241 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -382,14 +382,14 @@ out: return rc; } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_INV); + return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_INV); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_DEF); + return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_DEF); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 4362c26aa992..0c4235d159f4 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -86,8 +86,8 @@ int set_pages_wb(struct page *page, int numpages); int set_pages_ro(struct page *page, int numpages); int set_pages_rw(struct page *page, int numpages); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 4652487b5572..d36a58ae94db 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -2685,14 +2685,14 @@ static int __set_pages_np(struct page *page, int numpages, unsigned int cpa_flag return __change_page_attr_set_clr(&cpa, 1); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_pages_np(page, 1, 0); + return __set_pages_np(page, nr, 0); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_pages_p(page, 1, 0); + return __set_pages_p(page, nr, 0); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 3030d9245f5a..0b77f1d7d8b9 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -25,11 +25,13 @@ static inline int set_memory_rox(unsigned long addr, int numpages) #endif #ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP -static inline int set_direct_map_invalid_noflush(struct page *page) +static inline int set_direct_map_invalid_noflush(struct page *page, + unsigned int nr) { return 0; } -static inline int set_direct_map_default_noflush(struct page *page) +static inline int set_direct_map_default_noflush(struct page *page, + unsigned int nr) { return 0; } diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index b209712cb2c3..d5dba0e50b2e 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -88,7 +88,7 @@ static inline int hibernate_restore_unprotect_page(void *page_address) {return 0 static inline void hibernate_map_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { - int ret = set_direct_map_default_noflush(page); + int ret = set_direct_map_default_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); @@ -101,7 +101,7 @@ static inline void hibernate_unmap_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { unsigned long addr = (unsigned long)page_address(page); - int ret = set_direct_map_invalid_noflush(page); + int ret = set_direct_map_invalid_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); diff --git a/mm/secretmem.c b/mm/secretmem.c index 384f5cfc457f..6cbb8efc994a 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -139,7 +139,7 @@ retry: goto out; } - err = set_direct_map_invalid_noflush(folio_page(folio, 0)); + err = set_direct_map_invalid_noflush(folio_page(folio, 0), 1); if (err) { secretmem_unaccount_folio(state, folio); folio_put(folio); @@ -156,7 +156,7 @@ retry: * already happened when we marked the page invalid * which guarantees that this call won't fail */ - set_direct_map_default_noflush(folio_page(folio, 0)); + set_direct_map_default_noflush(folio_page(folio, 0), 1); folio_put(folio); if (err == -EEXIST) goto retry; @@ -228,7 +228,7 @@ static int secretmem_migrate_folio(struct address_space *mapping, static void secretmem_free_folio(struct folio *folio) { - set_direct_map_default_noflush(folio_page(folio, 0)); + set_direct_map_default_noflush(folio_page(folio, 0), 1); folio_zero_segment(folio, 0, folio_size(folio)); } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index bea9f76ed7e7..22566e0b6e38 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3358,14 +3358,15 @@ struct vm_struct *remove_vm_area(const void *addr) } static inline void set_area_direct_map(const struct vm_struct *area, - int (*set_direct_map)(struct page *page)) + int (*set_direct_map)(struct page *page, + unsigned int nr)) { unsigned long i; /* HUGE_VMALLOC passes small pages to set_direct_map */ for (i = 0; i < area->nr_pages; i++) if (page_address(area->pages[i])) - set_direct_map(area->pages[i]); + set_direct_map(area->pages[i], 1); } /* From 40702f168a9edf8eeb47c63adf7bc2eaede6bfbe Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 12:28:28 +0300 Subject: [PATCH 044/539] mm/vmalloc: set area's page_order after allocation succeeds __vmalloc_area_node() calls set_vm_area_page_order() to set area's page_order before actually allocating pages to populate the area. If allocation of large pages in HUGE_VMAP case fails midway, this leaves the area with elevated page_order throughout the cleanup path. There is no actual issue with this because the only place that currently relies on area->page_order on the cleanup path is the loop calculating the direct map alias range in vm_reset_perms() and it anyway skips unpopulated pages. But having set_vm_area_page_order() in the middle of __vmalloc_area_node() makes things very obscure, hard to reason about and error prone against future changes of the cleanup path. Move the call to set_vm_area_page_order() just before the successful return from __vmalloc_area_node() where page order is guaranteed. While on it, initialize local page_order variable with its declaration. Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-2-949b64a9f755@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Reviewed-by: Uladzislau Rezki (Sony) Reviewed-by: Dev Jain Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: David Hildenbrand Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Huacai Chen Cc: Ingo Molnar Cc: Len Brown Cc: Palmer Dabbelt Cc: Peter Zijlstra Cc: "Rafael J. Wysocki" Cc: Ryan Roberts Cc: Sven Schnelle Cc: Vasily Gorbik Cc: WANG Xuerui Cc: Will Deacon Cc: Kevin Brodsky --- mm/vmalloc.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 22566e0b6e38..9bd94e2f2895 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3873,7 +3873,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, unsigned long size = get_vm_area_size(area); unsigned long array_size; unsigned long nr_small_pages = size >> PAGE_SHIFT; - unsigned int page_order; + unsigned int page_order = page_shift - PAGE_SHIFT; unsigned int flags; int ret; @@ -3901,9 +3901,6 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, goto fail; } - set_vm_area_page_order(area, page_shift - PAGE_SHIFT); - page_order = vm_area_page_order(area); - /* * High-order nofail allocations are really expensive and * potentially dangerous (pre-mature OOM, disruptive reclaim @@ -3958,6 +3955,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, goto fail; } + set_vm_area_page_order(area, page_order); return area->addr; fail: From 1f73dfdeb9c2ae04daa82adf68c22f9d32989210 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 12:28:29 +0300 Subject: [PATCH 045/539] mm/vmalloc: constify vm parameter of get_vm_area_page_order() get_vm_area_page_order() and vm_area_page_order() do not need to modify struct vm_struct passed to them. Constify the parameter. Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-3-949b64a9f755@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Reviewed-by: Uladzislau Rezki (Sony) Reviewed-by: Dev Jain Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Kevin Brodsky Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Huacai Chen Cc: Ingo Molnar Cc: Len Brown Cc: Palmer Dabbelt Cc: Peter Zijlstra Cc: "Rafael J. Wysocki" Cc: Ryan Roberts Cc: Sven Schnelle Cc: Vasily Gorbik Cc: WANG Xuerui Cc: Will Deacon --- mm/vmalloc.c | 4 ++-- mm/vmalloc.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 9bd94e2f2895..a3e2e4244c0f 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3124,7 +3124,7 @@ EXPORT_SYMBOL(vm_map_ram); static struct vm_struct *vmlist __initdata; -static inline unsigned int vm_area_page_order(struct vm_struct *vm) +static inline unsigned int vm_area_page_order(const struct vm_struct *vm) { #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC return vm->page_order; @@ -3133,7 +3133,7 @@ static inline unsigned int vm_area_page_order(struct vm_struct *vm) #endif } -unsigned int get_vm_area_page_order(struct vm_struct *vm) +unsigned int get_vm_area_page_order(const struct vm_struct *vm) { return vm_area_page_order(vm); } diff --git a/mm/vmalloc.h b/mm/vmalloc.h index 8866ddcff668..211869f36509 100644 --- a/mm/vmalloc.h +++ b/mm/vmalloc.h @@ -12,7 +12,7 @@ void __init vmalloc_init(void); int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask); -unsigned int get_vm_area_page_order(struct vm_struct *vm); +unsigned int get_vm_area_page_order(const struct vm_struct *vm); #else static inline void vmalloc_init(void) {} From 86dc0280b807098d0cf15348719a01426aea0ea1 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 12:28:30 +0300 Subject: [PATCH 046/539] mm/vmalloc: make set_area_direct_map HUGE_VMAP friendly set_area_direct_map() always updates direct map alias permissions in single page increments. For HUGE_VMAP areas it's suboptimal. Not only the loop in set_area_direct_map() needlessly has more iterations (e.g times 512 on x86), but it also causes fragmentation of the direct map that could be avoided for the HUGE_VMAP areas populated with large pages. All pages in an area are always of the same order: either same-order large pages when VM_ALLOW_HUGE_VMAP is set and all huge pages were successfully allocated, or order-0 page when VM_ALLOW_HUGE_VMAP is cleared or when huge pages allocation fails and fallback path is taken. Instead of updating the direct map permissions for every order-0 page in an area, use the area's page_order as the loop increment and update the large pages in one call to set_direct_map_{invalid,default}_noflush(). Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-4-949b64a9f755@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Reviewed-by: Dev Jain Reviewed-by: Kevin Brodsky Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: David Hildenbrand Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Huacai Chen Cc: Ingo Molnar Cc: Len Brown Cc: Palmer Dabbelt Cc: Peter Zijlstra Cc: "Rafael J. Wysocki" Cc: Ryan Roberts Cc: Sven Schnelle Cc: "Uladzislau Rezki (Sony)" Cc: Vasily Gorbik Cc: WANG Xuerui Cc: Will Deacon --- mm/vmalloc.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index a3e2e4244c0f..395358b363fc 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3361,12 +3361,15 @@ static inline void set_area_direct_map(const struct vm_struct *area, int (*set_direct_map)(struct page *page, unsigned int nr)) { - unsigned long i; + unsigned int nr = (1U << vm_area_page_order(area)); - /* HUGE_VMALLOC passes small pages to set_direct_map */ - for (i = 0; i < area->nr_pages; i++) - if (page_address(area->pages[i])) - set_direct_map(area->pages[i], 1); + for (unsigned long i = 0; i < area->nr_pages; i += nr) { + if (page_address(area->pages[i])) { + int err = set_direct_map(area->pages[i], nr); + + WARN_ON_ONCE(err); + } + } } /* From 4c4b2ed5b783b98b04d4376e7a2176e4a9f64748 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 12:28:31 +0300 Subject: [PATCH 047/539] mm/execmem: use VM_FLUSH_RESET_PERMS for ROX cache allocations Initially execmem completely removed direct map alias for the memory allocated for the ROX cache in PMD_SIZE chunks. When that memory was freed, its direct map was restored also in PMD_SIZE chunks to avoid fragmentation of the direct map caused by vmalloc::vm_reset_perms(). This required execmem to implement the wrappers for set_direct_map APIs for proper sequencing of removal and restoration of the direct map aliases. Since then x86's CPA gained support for collapsing the direct map page tables for ROX pages and execmem switched from removing ROX caches from the direct map to making them ROX there, so execmem only needs to update direct map alias permissions when freeing the ROX cache memory. vmalloc already handles those updates for areas with VM_FLUSH_RESET_PERMS set and vmalloc::vm_reset_perms() does not force split of the direct map for PMD_SIZE chunks. Set the area permissions with set_vm_flush_reset_perms() when populating the execmem cache just before flipping the area to ROX. This way freeing an allocated area on an error path won't incur two updates of the direct map alias of that area and TLB flushing in vm_reset_perms(). Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-5-949b64a9f755@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Reviewed-by: Kevin Brodsky Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: David Hildenbrand Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Huacai Chen Cc: Ingo Molnar Cc: Len Brown Cc: Palmer Dabbelt Cc: Peter Zijlstra Cc: "Rafael J. Wysocki" Cc: Ryan Roberts Cc: Sven Schnelle Cc: "Uladzislau Rezki (Sony)" Cc: Vasily Gorbik Cc: WANG Xuerui Cc: Will Deacon Cc: Dev Jain --- mm/execmem.c | 40 +++++++--------------------------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/mm/execmem.c b/mm/execmem.c index 74a178a87e75..ad07cae9ed58 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -113,28 +113,6 @@ static inline unsigned long mas_range_len(struct ma_state *mas) return mas->last - mas->index + 1; } -static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid) -{ - unsigned int nr = (1 << get_vm_area_page_order(vm)); - unsigned int updated = 0; - int err = 0; - - for (int i = 0; i < vm->nr_pages; i += nr) { - err = set_direct_map_valid_noflush(vm->pages[i], nr, valid); - if (err) - goto err_restore; - updated += nr; - } - - return 0; - -err_restore: - for (int i = 0; i < updated; i += nr) - set_direct_map_valid_noflush(vm->pages[i], nr, !valid); - - return err; -} - static int execmem_force_rw(void *ptr, size_t size) { unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT; @@ -169,9 +147,6 @@ static void execmem_cache_clean(struct work_struct *work) if (IS_ALIGNED(size, PMD_SIZE) && IS_ALIGNED(mas.index, PMD_SIZE)) { - struct vm_struct *vm = find_vm_area(area); - - execmem_set_direct_map_valid(vm, true); mas_store_gfp(&mas, NULL, GFP_KERNEL); vfree(area); } @@ -301,6 +276,8 @@ static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t si /* fill memory with instructions that will trap */ execmem_fill_trapping_insns(p, alloc_size); + set_vm_flush_reset_perms(p); + err = set_memory_rox((unsigned long)p, vm->nr_pages); if (err) goto err_free_mem; @@ -312,18 +289,15 @@ static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t si */ mutex_lock(mutex); err = execmem_cache_add_locked(p, alloc_size, GFP_KERNEL); - if (err) - goto err_reset_direct_map; - - p = execmem_cache_alloc_locked(range, size); - + if (!err) + p = execmem_cache_alloc_locked(range, size); mutex_unlock(mutex); + if (err) + goto err_free_mem; + return p; -err_reset_direct_map: - mutex_unlock(mutex); - execmem_set_direct_map_valid(vm, true); err_free_mem: vfree(p); return NULL; From 0e774fdb2ed62c9b3031bc16dcde68d05f24b2fd Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 12:28:32 +0300 Subject: [PATCH 048/539] Revert "arch: introduce set_direct_map_valid_noflush()" Commit 0c6378a71574 ("arch: introduce set_direct_map_valid_noflush()") added set_direct_map_valid_noflush() to allow updating the direct map for a physically contiguous range in execmem. As Brendan recently pointed out [1], this API is confusing because on arm64 it means that is sets VALID bit in ptes, while on other architectures it is an analog of set_direct_map_default_noflush(). The only user of set_direct_map_valid_noflush() was execmem's ROX cache freeing path and it was switched to utilize VM_FLUSH_RESET_PERMS for resetting permissions of the direct map alias. With the last user gone and with set_direct_map_{invalid,default}_noflush() accepting number of pages as a parameter, set_direct_map_valid_noflush() become a copy of set_memory_valid() on arm64 and a duplicate of set_direct_map_{invalid,default}_noflush() on other architecture, it is safe to remove set_direct_map_valid_noflush(). Also drop a stale comment in arm64::__kernel_map_pages() that Linus bothered to add when merging changes containing set_direct_map_valid_noflush() to his tree. This reverts commit 0c6378a71574daa6cd1534ad42a956e3262756c7. Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-6-949b64a9f755@kernel.org Link: https://lore.kernel.org/all/DJ69RCVRBO0Y.3JCYSW50IC4RC@linux.dev [1] Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Reviewed-by: Brendan Jackman Acked-by: David Hildenbrand (Arm) Reviewed-by: Kevin Brodsky Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Huacai Chen Cc: Ingo Molnar Cc: Len Brown Cc: Palmer Dabbelt Cc: Peter Zijlstra Cc: "Rafael J. Wysocki" Cc: Ryan Roberts Cc: Sven Schnelle Cc: "Uladzislau Rezki (Sony)" Cc: Vasily Gorbik Cc: WANG Xuerui Cc: Will Deacon Cc: Dev Jain --- arch/arm64/include/asm/set_memory.h | 1 - arch/arm64/mm/pageattr.c | 16 ---------------- arch/loongarch/include/asm/set_memory.h | 1 - arch/loongarch/mm/pageattr.c | 19 ------------------- arch/riscv/include/asm/set_memory.h | 1 - arch/riscv/mm/pageattr.c | 15 --------------- arch/s390/include/asm/set_memory.h | 1 - arch/s390/mm/pageattr.c | 12 ------------ arch/x86/include/asm/set_memory.h | 1 - arch/x86/mm/pat/set_memory.c | 8 -------- include/linux/set_memory.h | 6 ------ 11 files changed, 81 deletions(-) diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h index b07fd4e026ea..0091ba12200e 100644 --- a/arch/arm64/include/asm/set_memory.h +++ b/arch/arm64/include/asm/set_memory.h @@ -13,7 +13,6 @@ int set_memory_valid(unsigned long addr, int numpages, int enable); int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages); int set_direct_map_default_noflush(struct page *page, unsigned int numpages); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); int set_memory_encrypted(unsigned long addr, int numpages); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index db8d60a84d14..132938b32eb1 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -355,23 +355,7 @@ int realm_register_memory_enc_ops(void) return arm64_mem_crypt_ops_register(&realm_crypt_ops); } -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - unsigned long addr = (unsigned long)page_address(page); - - if (!can_set_direct_map()) - return 0; - - return set_memory_valid(addr, nr, valid); -} - #ifdef CONFIG_DEBUG_PAGEALLOC -/* - * This is - apart from the return value - doing the same - * thing as the new set_direct_map_valid_noflush() function. - * - * Unify? Explain the conceptual differences? - */ void __kernel_map_pages(struct page *page, int numpages, int enable) { if (!can_set_direct_map()) diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h index 563aab92896e..4bb01172fbc2 100644 --- a/arch/loongarch/include/asm/set_memory.h +++ b/arch/loongarch/include/asm/set_memory.h @@ -17,6 +17,5 @@ int set_memory_rw(unsigned long addr, int numpages); bool kernel_page_present(struct page *page); int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); #endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index 43ad2a104f19..a7dcff40f759 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -217,22 +217,3 @@ int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) return __set_memory(addr, nr, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); } - -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - unsigned long addr = (unsigned long)page_address(page); - pgprot_t set, clear; - - if (addr < vm_map_base) - return 0; - - if (valid) { - set = PAGE_KERNEL; - clear = __pgprot(0); - } else { - set = __pgprot(0); - clear = __pgprot(_PAGE_PRESENT | _PAGE_VALID); - } - - return __set_memory(addr, nr, set, clear); -} diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index db1d0ed82b69..e9f9960c1947 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -42,7 +42,6 @@ static inline int set_kernel_memory(char *startp, char *endp, int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); int set_direct_map_default_noflush(struct page *page, unsigned int nr); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); #endif /* __ASSEMBLER__ */ diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 20ef95b1d0c3..5b3cf326455d 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -386,21 +386,6 @@ int set_direct_map_default_noflush(struct page *page, unsigned int nr) PAGE_KERNEL, __pgprot(_PAGE_EXEC)); } -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - pgprot_t set, clear; - - if (valid) { - set = PAGE_KERNEL; - clear = __pgprot(_PAGE_EXEC); - } else { - set = __pgprot(0); - clear = __pgprot(_PAGE_PRESENT); - } - - return __set_memory((unsigned long)page_address(page), nr, set, clear); -} - #ifdef CONFIG_DEBUG_PAGEALLOC static int debug_pagealloc_set_page(pte_t *pte, unsigned long addr, void *data) { diff --git a/arch/s390/include/asm/set_memory.h b/arch/s390/include/asm/set_memory.h index 6b0aa9147ed8..e3562bf0c1aa 100644 --- a/arch/s390/include/asm/set_memory.h +++ b/arch/s390/include/asm/set_memory.h @@ -62,7 +62,6 @@ __SET_MEMORY_FUNC(set_memory_4k, SET_MEMORY_4K) int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); int set_direct_map_default_noflush(struct page *page, unsigned int nr); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); #endif diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 7549543d6241..80e834e8b8e1 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -392,18 +392,6 @@ int set_direct_map_default_noflush(struct page *page, unsigned int nr) return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_DEF); } -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - unsigned long flags; - - if (valid) - flags = SET_MEMORY_DEF; - else - flags = SET_MEMORY_INV; - - return __set_memory((unsigned long)page_to_virt(page), nr, flags); -} - bool kernel_page_present(struct page *page) { unsigned long addr; diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 0c4235d159f4..39271a5ea925 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -88,7 +88,6 @@ int set_pages_rw(struct page *page, int numpages); int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); int set_direct_map_default_noflush(struct page *page, unsigned int nr); -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); extern int kernel_set_to_readonly; diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index d36a58ae94db..7b6d983088ea 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -2695,14 +2695,6 @@ int set_direct_map_default_noflush(struct page *page, unsigned int nr) return __set_pages_p(page, nr, 0); } -int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) -{ - if (valid) - return __set_pages_p(page, nr, 0); - - return __set_pages_np(page, nr, 0); -} - #ifdef CONFIG_DEBUG_PAGEALLOC void __kernel_map_pages(struct page *page, int numpages, int enable) { diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 0b77f1d7d8b9..3fe293cfed8c 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -36,12 +36,6 @@ static inline int set_direct_map_default_noflush(struct page *page, return 0; } -static inline int set_direct_map_valid_noflush(struct page *page, - unsigned nr, bool valid) -{ - return 0; -} - static inline bool kernel_page_present(struct page *page) { return true; From 59042a88efe1d5ddc457f192f4c6368909693133 Mon Sep 17 00:00:00 2001 From: Hao Jia Date: Fri, 28 Aug 2026 16:31:49 +0800 Subject: [PATCH 049/539] zram: fix idle age_sec underflow in idle_store() After commit 2e8ff2f51dde ("zram: use u32 for entry ac_time tracking"), idle_store() computes the idle cutoff as: cutoff = ktime_sub((u32)ktime_get_boottime_seconds(), age_sec); Because the left operand is cast to u32, when age_sec exceeds the current uptime the subtraction wraps modulo 2^32 and the huge result is zero-extended into the s64 cutoff. mark_idle() then marks every entry as idle instead of matching nothing. For instance, running echo 86400 > /sys/block/zramX/idle on a machine up for only two minutes marks all newly written pages idle and hands them to idle writeback and recompression. No slot can have been accessed before the system booted, so an age_sec that reaches back past uptime cannot match any slot. Return early in that case, without walking the table or taking any slot locks. Track the cutoff as time64_t rather than ktime_t. Both cutoff and ac_time are boot-time values in seconds, so a plain arithmetic comparison against ac_time in mark_idle() is correct and no ktime helpers are needed. Link: https://lore.kernel.org/20260828083149.45760-1-jiahao.kernel@gmail.com Fixes: 2e8ff2f51dde ("zram: use u32 for entry ac_time tracking") Signed-off-by: Hao Jia Signed-off-by: Andrew Morton Suggested-by: Sergey Senozhatsky Cc: Brian Geffon Cc: Jens Axboe Cc: Minchan Kim Cc: --- drivers/block/zram/zram_drv.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index a9b3bb1d3bef..4ba0f77b2abd 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -415,7 +415,7 @@ static ssize_t mem_used_max_store(struct device *dev, * Mark all pages which are older than or equal to cutoff as IDLE. * Callers should hold the zram init lock in read mode */ -static void mark_idle(struct zram *zram, ktime_t cutoff) +static void mark_idle(struct zram *zram, time64_t cutoff) { int is_idle = 1; unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; @@ -439,7 +439,7 @@ static void mark_idle(struct zram *zram, ktime_t cutoff) #ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME is_idle = !cutoff || - ktime_after(cutoff, zram->table[index].attr.ac_time); + cutoff > zram->table[index].attr.ac_time; #endif if (is_idle) set_slot_flag(zram, index, ZRAM_IDLE); @@ -453,21 +453,26 @@ static ssize_t idle_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); - ktime_t cutoff = 0; + time64_t cutoff = 0; if (!sysfs_streq(buf, "all")) { /* * If it did not parse as 'all' try to treat it as an integer * when we have memory tracking enabled. */ + time64_t uptime; u32 age_sec; - if (IS_ENABLED(CONFIG_ZRAM_TRACK_ENTRY_ACTIME) && - !kstrtouint(buf, 0, &age_sec)) - cutoff = ktime_sub((u32)ktime_get_boottime_seconds(), - age_sec); - else + if (!IS_ENABLED(CONFIG_ZRAM_TRACK_ENTRY_ACTIME) || + kstrtouint(buf, 0, &age_sec)) return -EINVAL; + + /* No slot can be older than the system uptime */ + uptime = ktime_get_boottime_seconds(); + if (age_sec >= uptime) + return len; + + cutoff = uptime - age_sec; } guard(rwsem_read)(&zram->dev_lock); From 942c8ec27aa4ffc587511650718349964e1696e6 Mon Sep 17 00:00:00 2001 From: Vernon Yang Date: Wed, 9 Sep 2026 10:58:01 +0800 Subject: [PATCH 050/539] mm: khugepaged: fix swap entry value to folio_pfn() Patch series "mm: khugepaged: fix tracepoint UAF", v5. The khugepaged tracepoints take a folio pointer and call folio_pfn(), but by then the folio may no longer be valid: freed after folio_put(), folio_unlock() or pte_unmap_unlock(), or not a folio at all but an xarray-encoded swap entry. On classic SPARSEMEM, dereferencing it oopses khugepaged as soon as the trace event is enabled; on other memory models it merely prints a bogus pfn. Pass the pfn to the tracepoints directly, captured while the folio is still pinned, closing the use-after-free windows in mm_khugepaged_scan_file(), mm_khugepaged_scan_pmd() and mm_khugepaged_collapse_file(). This patch (of 3): When the swap entries found exceed max_ptes_swap, the loop is left via break with folio still holding the xarray value that encodes the swap entry, not valid folio pointer. That value is passed to trace_mm_khugepaged_scan_file(), which feeds it to folio_pfn(). On FLATMEM and SPARSEMEM_VMEMMAP, the page_to_pfn() is plain pointer arithmetic, so the trace event merely prints bogus scan_pfn. On classic SPARSEMEM, the page_to_pfn() reads page->flags, dereferencing the tiny encoded integer and oopsing khugepaged whenever the trace event is enabled. So when folio is the swap entry value, simply set pfn to -1, just like exhausted scan naturally. And the folio_put() has maybe dropped the last reference of folio. The trace_mm_khugepaged_scan_file() is left with a dangling folio pointer. so using the folio_pfn() before dropping the reference, closing use-after-free window. Link: https://lore.kernel.org/20260909025804.3233645-1-vernon2gm@gmail.com Link: https://lore.kernel.org/20260909025804.3233645-2-vernon2gm@gmail.com Fixes: d41fd2016ed0 ("mm/khugepaged: add tracepoint to hpage_collapse_scan_file()") Signed-off-by: Vernon Yang Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Barry Song Cc: Dev Jain Cc: Lance Yang Cc: Lorenzo Stoakes Cc: Ryan Roberts Cc: Zach O'Keefe Cc: --- include/trace/events/huge_memory.h | 6 +++--- mm/khugepaged.c | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index 5a48c5406cce..7b526528f85b 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -178,10 +178,10 @@ TRACE_EVENT(mm_collapse_huge_page_swapin, TRACE_EVENT(mm_khugepaged_scan_file, - TP_PROTO(struct mm_struct *mm, struct folio *folio, struct file *file, + TP_PROTO(struct mm_struct *mm, unsigned long pfn, struct file *file, int present, int swap, int result), - TP_ARGS(mm, folio, file, present, swap, result), + TP_ARGS(mm, pfn, file, present, swap, result), TP_STRUCT__entry( __field(struct mm_struct *, mm) @@ -194,7 +194,7 @@ TRACE_EVENT(mm_khugepaged_scan_file, TP_fast_assign( __entry->mm = mm; - __entry->pfn = folio ? folio_pfn(folio) : -1; + __entry->pfn = pfn; __assign_str(filename); __entry->present = present; __entry->swap = swap; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 75639298efc2..6380a12b8eee 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2683,6 +2683,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, int present, swap; int node = NUMA_NO_NODE; enum scan_result result = SCAN_SUCCEED; + unsigned long failed_pfn = -1; present = 0; swap = 0; @@ -2715,6 +2716,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, if (is_pmd_order(folio_order(folio))) { result = SCAN_PTE_MAPPED_HUGEPAGE; + failed_pfn = folio_pfn(folio); /* * PMD-sized THP implies that we can only try * retracting the PTE table. @@ -2726,6 +2728,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, node = folio_nid(folio); if (collapse_scan_abort(node, cc)) { result = SCAN_SCAN_ABORT; + failed_pfn = folio_pfn(folio); folio_put(folio); break; } @@ -2733,12 +2736,14 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, if (!folio_test_lru(folio)) { result = SCAN_PAGE_LRU; + failed_pfn = folio_pfn(folio); folio_put(folio); break; } if (folio_expected_ref_count(folio) + 1 != folio_ref_count(folio)) { result = SCAN_PAGE_COUNT; + failed_pfn = folio_pfn(folio); folio_put(folio); break; } @@ -2773,7 +2778,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, } } - trace_mm_khugepaged_scan_file(mm, folio, file, present, swap, result); + trace_mm_khugepaged_scan_file(mm, failed_pfn, file, present, swap, result); return result; } From ba2f1f92ddfabd340c9248fd2380104f717ee8b1 Mon Sep 17 00:00:00 2001 From: Vernon Yang Date: Wed, 9 Sep 2026 10:58:02 +0800 Subject: [PATCH 051/539] mm: khugepaged: fix folio is used after pte_unmap_unlock() After the page table lock has dropped, the folio can be freed concurrently. The trace_mm_khugepaged_scan_pmd() is left with a dangling folio pointer. So using the folio_pfn() before dropping the page table lock, closing use-after-free window. And other pre-existing bug, When the `for (i = 0; i < HPAGE_PMD_NR; i++)` iteration to terminate and the folio operation preceding is normal, but pfn will be incorrect. so we really only trace the PFN if it really was problematic. Link: https://lore.kernel.org/20260909025804.3233645-3-vernon2gm@gmail.com Fixes: 7d2eba0557c1 ("mm: add tracepoint for scanning pages") Signed-off-by: Vernon Yang Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes (ARM) Cc: Barry Song Cc: Dev Jain Cc: Lance Yang Cc: Ryan Roberts Cc: Zach O'Keefe Cc: --- include/trace/events/huge_memory.h | 6 +++--- mm/khugepaged.c | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index 7b526528f85b..fa828967e1fb 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -55,10 +55,10 @@ SCAN_STATUS TRACE_EVENT(mm_khugepaged_scan_pmd, - TP_PROTO(struct mm_struct *mm, struct folio *folio, + TP_PROTO(struct mm_struct *mm, unsigned long pfn, int referenced, int none_or_zero, int status, int unmapped), - TP_ARGS(mm, folio, referenced, none_or_zero, status, unmapped), + TP_ARGS(mm, pfn, referenced, none_or_zero, status, unmapped), TP_STRUCT__entry( __field(struct mm_struct *, mm) @@ -71,7 +71,7 @@ TRACE_EVENT(mm_khugepaged_scan_pmd, TP_fast_assign( __entry->mm = mm; - __entry->pfn = folio ? folio_pfn(folio) : -1; + __entry->pfn = pfn; __entry->referenced = referenced; __entry->none_or_zero = none_or_zero; __entry->status = status; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 6380a12b8eee..730829947b4c 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1612,6 +1612,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, enum scan_result result = SCAN_FAIL; struct page *page = NULL; struct folio *folio = NULL; + unsigned long failed_pfn = -1; unsigned long addr; unsigned long enabled_orders; spinlock_t *ptl; @@ -1706,11 +1707,13 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, if (cc->is_khugepaged && !(vma->vm_flags & VM_DROPPABLE) && folio_test_lazyfree(folio) && !pte_dirty(pteval)) { result = SCAN_PAGE_LAZYFREE; + failed_pfn = folio_pfn(folio); goto out_unmap; } if (!folio_test_anon(folio)) { result = SCAN_PAGE_ANON; + failed_pfn = folio_pfn(folio); goto out_unmap; } @@ -1721,6 +1724,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, if (folio_maybe_mapped_shared(folio)) { if (++shared > max_ptes_shared) { result = SCAN_EXCEED_SHARED_PTE; + failed_pfn = folio_pfn(folio); count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_SHARED_PTE, MTHP_STAT_COLLAPSE_EXCEED_SHARED); goto out_unmap; @@ -1738,15 +1742,18 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, node = folio_nid(folio); if (collapse_scan_abort(node, cc)) { result = SCAN_SCAN_ABORT; + failed_pfn = folio_pfn(folio); goto out_unmap; } cc->node_load[node]++; if (!folio_test_lru(folio)) { result = SCAN_PAGE_LRU; + failed_pfn = folio_pfn(folio); goto out_unmap; } if (folio_test_locked(folio)) { result = SCAN_PAGE_LOCK; + failed_pfn = folio_pfn(folio); goto out_unmap; } @@ -1759,6 +1766,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, */ if (folio_expected_ref_count(folio) != folio_ref_count(folio)) { result = SCAN_PAGE_COUNT; + failed_pfn = folio_pfn(folio); goto out_unmap; } @@ -1784,7 +1792,7 @@ out_unmap: *lock_dropped = true; } out: - trace_mm_khugepaged_scan_pmd(mm, folio, referenced, + trace_mm_khugepaged_scan_pmd(mm, failed_pfn, referenced, none_or_zero, result, unmapped); return result; } From 9010383e71e6ce75fcd3c8f87133c8ce6c0e6ab4 Mon Sep 17 00:00:00 2001 From: Vernon Yang Date: Wed, 9 Sep 2026 10:58:03 +0800 Subject: [PATCH 052/539] mm: khugepaged: fix folio is used after folio_put/unlock() On the rollback path, folio_put() has already dropped the last reference of new_folio. On the success path, new_folio is already unlocked and can be freed concurrently. The trace_mm_khugepaged_collapse_file() is left with a dangling folio pointer. So using the folio_pfn() before dropping the reference, closing use-after-free window. Link: https://lore.kernel.org/20260909025804.3233645-4-vernon2gm@gmail.com Fixes: 4c9473e87e75 ("mm/khugepaged: add tracepoint to collapse_file()") Signed-off-by: Vernon Yang Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes (ARM) Cc: Barry Song Cc: Dev Jain Cc: Lance Yang Cc: Ryan Roberts Cc: Zach O'Keefe Cc: --- include/trace/events/huge_memory.h | 6 +++--- mm/khugepaged.c | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index fa828967e1fb..5fb4d92cfd84 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -211,10 +211,10 @@ TRACE_EVENT(mm_khugepaged_scan_file, ); TRACE_EVENT(mm_khugepaged_collapse_file, - TP_PROTO(struct mm_struct *mm, struct folio *new_folio, pgoff_t index, + TP_PROTO(struct mm_struct *mm, unsigned long new_pfn, pgoff_t index, unsigned long addr, bool is_shmem, struct file *file, int nr, int result), - TP_ARGS(mm, new_folio, index, addr, is_shmem, file, nr, result), + TP_ARGS(mm, new_pfn, index, addr, is_shmem, file, nr, result), TP_STRUCT__entry( __field(struct mm_struct *, mm) __field(unsigned long, hpfn) @@ -228,7 +228,7 @@ TRACE_EVENT(mm_khugepaged_collapse_file, TP_fast_assign( __entry->mm = mm; - __entry->hpfn = new_folio ? folio_pfn(new_folio) : -1; + __entry->hpfn = new_pfn; __entry->index = index; __entry->addr = addr; __entry->is_shmem = is_shmem; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 730829947b4c..792166950bba 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2253,6 +2253,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, struct address_space *mapping = file->f_mapping; struct page *dst; struct folio *folio, *tmp, *new_folio; + unsigned long new_pfn = -1; pgoff_t index = 0, end = start + HPAGE_PMD_NR; LIST_HEAD(pagelist); XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); @@ -2272,6 +2273,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, result = alloc_charge_folio(&new_folio, mm, cc, HPAGE_PMD_ORDER); if (result != SCAN_SUCCEED) goto out; + new_pfn = folio_pfn(new_folio); mapping_set_update(&xas, mapping); @@ -2675,7 +2677,7 @@ rollback: folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); - trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result); + trace_mm_khugepaged_collapse_file(mm, new_pfn, index, addr, is_shmem, file, HPAGE_PMD_NR, result); return result; } From 877c1d0059ca82eb1e88d1f6c488dec317f7d724 Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Mon, 17 Aug 2026 17:03:26 +0800 Subject: [PATCH 053/539] mm: memcontrol: make obj_cgroup_memcg() handle NULL objcg Patch series "make unused huge shrinker memcg aware", v4. The shmem unused huge shrinker maintains a per-superblock list of inodes whose tail huge folio extends beyond i_size. Because this list is not memcg aware, reclaim triggered by memcg A can scan inodes across the entire superblock and split huge folios charged to unrelated memcg B, causing unexpected impact on it. In the worst case, memcg A has no reclaimable shmem at all, making the reclaim entirely useless and incurring unnecessary latency. We observed this in production, where page lock contention during split caused multi-hundred-millisecond stalls: tid 11340 comm scanner locked a page for 182264 us! kstack: unlock_page+1 split_huge_page_to_list+3135 shmem_unused_huge_shrink+767 super_cache_scan+329 do_shrink_slab+291 shrink_slab+533 shrink_node+400 do_try_to_free_pages+206 try_to_free_mem_cgroup_pages+262 try_charge_memcg+591 mem_cgroup_charge+136 __handle_mm_fault+2431 handle_mm_fault+194 do_user_addr_fault+462 __do_page_fault+176 do_page_fault+48 page_fault+62 Usama's recent patch [1] prevents the shmem unused shrinker from being invoked during memcg-level reclaim altogether, but this is overly conservative: we can do better by reclaiming only the shmem charged to the reclaiming memcg. This series converts the shrinker list to a memcg-aware list_lru, so that non-root memcg reclaim walks only candidates charged to the reclaiming memcg. Global reclaim, root memcg reclaim and shmem quota reclaim retain their existing global semantics. To avoid pinning a dying memcg through a long-lived CSS reference, each inode stores an obj_cgroup reference instead of a mem_cgroup reference. The list_lru add/delete paths resolve the current memcg from the objcg under RCU, staying consistent with list_lru's own memcg migration on offline. This patch (of 3): obj_cgroup_memcg() currently requires a non-NULL objcg, so callers that may hold a NULL objcg must guard the call with an explicit NULL check. This pattern is duplicated in folio_memcg(), folio_memcg_check(), mm/page_owner.c, and mm/zswap.c. Teach obj_cgroup_memcg() to accept NULL and return NULL in that case, then remove the redundant NULL checks at the call sites. Also remove the mem_cgroup_from_entry() wrapper in zswap, which existed solely to provide this NULL-safe behaviour, and replace its two callers with direct obj_cgroup_memcg() calls. No functional change intended. Link: https://lore.kernel.org/cover.1786955972.git.zhengqi.arch@bytedance.com Link: https://lore.kernel.org/09bcf74312246a6e4146be8a0cb9787f8beddb28.1786955972.git.zhengqi.arch@bytedance.com Signed-off-by: Qi Zheng Signed-off-by: Andrew Morton Acked-by: Shakeel Butt Cc: Baolin Wang Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/memcontrol.h | 11 ++++++++--- mm/page_owner.c | 2 +- mm/zswap.c | 17 ++--------------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7d1c0ce189a8..da625d2edb3b 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -380,7 +380,7 @@ enum objext_flags { static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) { lockdep_assert_once(rcu_read_lock_held() || lockdep_is_held(&cgroup_mutex)); - return READ_ONCE(objcg->memcg); + return objcg ? READ_ONCE(objcg->memcg) : NULL; } /* @@ -433,7 +433,7 @@ static inline struct mem_cgroup *folio_memcg(struct folio *folio) { struct obj_cgroup *objcg = folio_objcg(folio); - return objcg ? obj_cgroup_memcg(objcg) : NULL; + return obj_cgroup_memcg(objcg); } /* @@ -476,7 +476,7 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); - return objcg ? obj_cgroup_memcg(objcg) : NULL; + return obj_cgroup_memcg(objcg); } static inline struct mem_cgroup *page_memcg_check(struct page *page) @@ -1050,6 +1050,11 @@ void mem_cgroup_flush_workqueue(void); extern int mem_cgroup_init(void); #else /* CONFIG_MEMCG */ +static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) +{ + return NULL; +} + #define MEM_CGROUP_ID_SHIFT 0 #define root_mem_cgroup (NULL) diff --git a/mm/page_owner.c b/mm/page_owner.c index fbbda7ba914b..3fc37d9b908e 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -575,7 +575,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, } objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); - memcg = objcg ? obj_cgroup_memcg(objcg) : NULL; + memcg = obj_cgroup_memcg(objcg); if (!memcg) goto out_unlock; diff --git a/mm/zswap.c b/mm/zswap.c index 37f34e406c8e..c1dc60926bad 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -647,19 +647,6 @@ static int zswap_enabled_param_set(const char *val, * lru functions **********************************/ -/* should be called under RCU */ -#ifdef CONFIG_MEMCG -static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry) -{ - return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL; -} -#else -static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry) -{ - return NULL; -} -#endif - static inline int entry_to_nid(struct zswap_entry *entry) { return page_to_nid(virt_to_page(entry)); @@ -682,7 +669,7 @@ static void zswap_lru_add(struct zswap_entry *entry) * Similar reasoning holds for list_lru_del(). */ rcu_read_lock(); - memcg = mem_cgroup_from_entry(entry); + memcg = obj_cgroup_memcg(entry->objcg); /* will always succeed */ list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg); rcu_read_unlock(); @@ -694,7 +681,7 @@ static void zswap_lru_del(struct zswap_entry *entry) struct mem_cgroup *memcg; rcu_read_lock(); - memcg = mem_cgroup_from_entry(entry); + memcg = obj_cgroup_memcg(entry->objcg); /* will always succeed */ list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg); rcu_read_unlock(); From 948715f5866cc5b91c096a0308dd5a93b1a7d79b Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Mon, 17 Aug 2026 17:03:27 +0800 Subject: [PATCH 054/539] mm: shmem: move unused huge shrinklist queuing past the truncation check The shmem_get_folio_gfp() adds the inode to the unused huge shrinker list at the alloced label, but a subsequent truncation check may still fail and remove the folio, leaving the inode on the list with a stale folio. The original code works because the shrinker re-looks-up the folio and drops stale entries, but it is cleaner to queue the inode only after all checks that might remove the folio have passed. So just make the pure structural move with no functional change, and it serves as preparation for the memcg-aware shrinker conversion. Link: https://lore.kernel.org/17fcf64faec0dfbbe6cf8a97924e3f39cd3c51a4.1786955972.git.zhengqi.arch@bytedance.com Signed-off-by: Qi Zheng Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt --- mm/shmem.c | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index 848316eaa7f4..9f2585d6dbfb 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2539,27 +2539,6 @@ repeat: alloced: alloced = true; - if (folio_test_large(folio) && - DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < - folio_next_index(folio)) { - struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); - struct shmem_inode_info *info = SHMEM_I(inode); - /* - * Part of the large folio is beyond i_size: subject - * to shrink under memory pressure. - */ - spin_lock(&sbinfo->shrinklist_lock); - /* - * _careful to defend against unlocked access to - * ->shrink_list in shmem_unused_huge_shrink() - */ - if (list_empty_careful(&info->shrinklist)) { - list_add_tail(&info->shrinklist, - &sbinfo->shrinklist); - sbinfo->shrinklist_len++; - } - spin_unlock(&sbinfo->shrinklist_lock); - } if (sgp == SGP_WRITE) folio_set_referenced(folio); @@ -2589,6 +2568,33 @@ clear: error = -EINVAL; goto unlock; } + + /* + * Queue the inode on the shrink list only after all checks that might + * remove the folio have passed. Otherwise the inode could be left on + * the shrinker list with a stale folio. + */ + if (alloced && folio_test_large(folio) && + DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < folio_next_index(folio)) { + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + struct shmem_inode_info *info = SHMEM_I(inode); + /* + * Part of the large folio is beyond i_size: subject + * to shrink under memory pressure. + */ + spin_lock(&sbinfo->shrinklist_lock); + /* + * _careful to defend against unlocked access to + * ->shrink_list in shmem_unused_huge_shrink() + */ + if (list_empty_careful(&info->shrinklist)) { + list_add_tail(&info->shrinklist, + &sbinfo->shrinklist); + sbinfo->shrinklist_len++; + } + spin_unlock(&sbinfo->shrinklist_lock); + } + out: *foliop = folio; return 0; From 743e7d025d2f6f6c801e8bdea714eae64bf8fbb2 Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Mon, 17 Aug 2026 17:03:28 +0800 Subject: [PATCH 055/539] mm: shmem: make unused huge shrinker memcg aware The shmem unused huge shrinker keeps a per-superblock list of inodes whose tail huge folio extends beyond i_size. Since that list is not memcg aware, reclaim triggered by one memcg can scan inodes from the whole superblock and split shmem huge folios charged to unrelated memcgs. Convert the shrink list to a memcg-aware list_lru. Queue each inode on the list_lru sublist matching the memcg and node of the current tail huge folio, so non-root memcg reclaim only walks candidates charged to the reclaiming memcg. Global reclaim, root memcg reclaim and shmem quota reclaim keep global semantics. Rather than pinning a struct mem_cgroup reference in shmem_inode_info, store a struct obj_cgroup reference instead. The list_lru add and delete paths resolve the current memcg from the objcg under RCU, so that memcg offline and list_lru entry migration remain consistent: list_lru migrates entries to the parent memcg sublist on offline, and obj_cgroup_memcg() follows the same reparenting, ensuring the correct sublist is always found at delete time. This avoids pinning a dying memcg through a long-lived CSS reference. The list_lru still tracks inodes while the actual split target is the current tail huge folio, so validate the folio memcg/node during scan. If the folio no longer matches the reclaim context or splitting cannot proceed, requeue the inode according to the current tail folio; if the inode is no longer shrinkable, drop the scan entry. This can be tested with the shrinker debugfs interface by allocating 32 tmpfs tail THPs in each of two memcgs, then scanning the sb-tmpfs shrinker with memcg A's cgroup id: before A scan after A scan base A=64M, B=64M A=64M, B=64M (per-memcg count is skipped) patched A=64M, B=64M A=0, B=64M Link: https://lore.kernel.org/94cc7fe1fd645254ae90effc1c0687678438d940.1786955972.git.zhengqi.arch@bytedance.com Signed-off-by: Qi Zheng Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Qinyun Tan --- include/linux/shmem_fs.h | 12 +- mm/shmem.c | 362 ++++++++++++++++++++++++++++++--------- 2 files changed, 289 insertions(+), 85 deletions(-) diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 5663dff53186..a7c7a96a7cbf 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -11,6 +11,7 @@ #include #include #include +#include /* inode in-kernel data */ @@ -54,6 +55,11 @@ struct shmem_inode_info { struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif struct inode vfs_inode; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct obj_cgroup *shrinklist_objcg; + int shrinklist_nid; +#endif }; #define SHMEM_FL_USER_VISIBLE (FS_FL_USER_VISIBLE | FS_CASEFOLD_FL) @@ -83,9 +89,9 @@ struct shmem_sb_info { ino_t next_ino; /* The next per-sb inode number to use */ ino_t __percpu *ino_batch; /* The next per-cpu inode number to use */ struct mempolicy *mpol; /* default memory policy for mappings */ - spinlock_t shrinklist_lock; /* Protects shrinklist */ - struct list_head shrinklist; /* List of shinkable inodes */ - unsigned long shrinklist_len; /* Length of shrinklist */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct list_lru shrinklist; /* List of shrinkable inodes */ +#endif struct shmem_quota_limits qlimits; /* Default quota limits */ struct simple_xattr_cache xa_cache; }; diff --git a/mm/shmem.c b/mm/shmem.c index 9f2585d6dbfb..ae39966fc730 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -725,51 +725,258 @@ static const char *shmem_format_huge(int huge) } #endif +static bool is_shmem_unused_huge_isolated(struct shmem_inode_info *info) +{ + + return info->shrinklist_nid == -1; +} + +static void set_shmem_unused_huge_isolated(struct shmem_inode_info *info) +{ + info->shrinklist_nid = -1; +} + +static struct obj_cgroup *shmem_get_and_clear_objcg(struct shmem_inode_info *info) +{ + struct obj_cgroup *objcg = info->shrinklist_objcg; + + info->shrinklist_objcg = NULL; + + return objcg; +} + +#ifdef CONFIG_MEMCG +static struct obj_cgroup * +shmem_unused_huge_alloc_lru(struct shmem_sb_info *sbinfo, struct folio *folio, + gfp_t gfp) +{ + int ret; + + ret = folio_memcg_list_lru_alloc(folio, &sbinfo->shrinklist, gfp); + if (ret) + return ERR_PTR(ret); + + return get_obj_cgroup_from_folio(folio); +} +#else +static struct obj_cgroup * +shmem_unused_huge_alloc_lru(struct shmem_sb_info *sbinfo, struct folio *folio, + gfp_t gfp) +{ + return NULL; +} +#endif + +static void shmem_unused_huge_lru_add(struct shmem_sb_info *sbinfo, + struct list_head *item, int nid, + struct obj_cgroup *objcg) +{ + struct mem_cgroup *memcg; + + rcu_read_lock(); + memcg = obj_cgroup_memcg(objcg); + list_lru_add(&sbinfo->shrinklist, item, nid, memcg); + rcu_read_unlock(); +} + +static void shmem_unused_huge_lru_del(struct shmem_sb_info *sbinfo, + struct list_head *item, int nid, + struct obj_cgroup *objcg) +{ + struct mem_cgroup *memcg; + + rcu_read_lock(); + memcg = obj_cgroup_memcg(objcg); + list_lru_del(&sbinfo->shrinklist, item, nid, memcg); + rcu_read_unlock(); +} + +static void shmem_unused_huge_add(struct inode *inode, struct folio *folio, + gfp_t gfp) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + int nid = folio_nid(folio); + struct obj_cgroup *objcg = NULL, *old_objcg = NULL; + + objcg = shmem_unused_huge_alloc_lru(sbinfo, folio, gfp); + if (IS_ERR(objcg)) + return; + + spin_lock(&info->lock); + if (!list_empty(&info->shrinklist)) { + /* isolated on scan list, let shrink handle it */ + if (is_shmem_unused_huge_isolated(info)) + goto unlock; + + if (info->shrinklist_nid == nid && + info->shrinklist_objcg == objcg) + goto unlock; + + shmem_unused_huge_lru_del(sbinfo, &info->shrinklist, + info->shrinklist_nid, + info->shrinklist_objcg); + old_objcg = shmem_get_and_clear_objcg(info); + } + + info->shrinklist_objcg = objcg; + info->shrinklist_nid = nid; + shmem_unused_huge_lru_add(sbinfo, &info->shrinklist, nid, objcg); + objcg = NULL; +unlock: + spin_unlock(&info->lock); + obj_cgroup_put(old_objcg); + obj_cgroup_put(objcg); +} + +static void shmem_unused_huge_del(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + struct obj_cgroup *objcg = NULL; + + spin_lock(&info->lock); + if (!list_empty(&info->shrinklist)) { + shmem_unused_huge_lru_del(sbinfo, &info->shrinklist, + info->shrinklist_nid, + info->shrinklist_objcg); + objcg = shmem_get_and_clear_objcg(info); + } + spin_unlock(&info->lock); + + obj_cgroup_put(objcg); +} + +struct shmem_unused_huge_scan { + struct list_head list; + struct shrink_control *sc; +}; + +static enum lru_status shmem_unused_huge_isolate(struct list_head *item, + struct list_lru_one *lru, + void *arg) +{ + struct shmem_unused_huge_scan *scan = arg; + struct shmem_inode_info *info; + struct inode *inode; + struct obj_cgroup *objcg = NULL; + + info = list_entry(item, struct shmem_inode_info, shrinklist); + + /* + * Use trylock to avoid ABBA deadlock: add/del path takes info->lock + * before the list_lru bucket lock, while here the order is reversed. + */ + if (!spin_trylock(&info->lock)) + return LRU_SKIP; + + /* pin the inode */ + inode = igrab(&info->vfs_inode); + /* inode is about to be evicted */ + if (!inode) { + list_lru_isolate(lru, item); + objcg = shmem_get_and_clear_objcg(info); + spin_unlock(&info->lock); + obj_cgroup_put(objcg); + return LRU_REMOVED; + } + + list_lru_isolate(lru, item); + objcg = shmem_get_and_clear_objcg(info); + set_shmem_unused_huge_isolated(info); + list_add_tail(&info->shrinklist, &scan->list); + spin_unlock(&info->lock); + obj_cgroup_put(objcg); + + return LRU_REMOVED; +} + +static bool is_shmem_unused_huge_match(struct folio *folio, + struct shrink_control *sc) +{ + struct mem_cgroup *memcg = NULL; + bool match; + + /* shmem quota reclaim has no NUMA node or memcg restriction */ + if (!sc) + return true; + + if (folio_nid(folio) != sc->nid) + return false; + + /* + * Only non-root memcg reclaim needs to match the folio charge against + * sc->memcg. Skip the folio memcg check for global shrinker reclaim and + * root memcg reclaim. + */ + if (!sc->memcg || mem_cgroup_is_root(sc->memcg)) + return true; + + memcg = get_mem_cgroup_from_folio(folio); + match = memcg == sc->memcg; + mem_cgroup_put(memcg); + + return match; +} + +static void shmem_unused_huge_drop(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + + spin_lock(&info->lock); + list_del_init(&info->shrinklist); + spin_unlock(&info->lock); +} + +static void shmem_unused_huge_requeue(struct inode *inode, struct folio *folio) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + struct obj_cgroup *objcg; + int nid = folio_nid(folio); + + objcg = shmem_unused_huge_alloc_lru(sbinfo, folio, GFP_NOWAIT); + if (IS_ERR(objcg)) { + shmem_unused_huge_drop(inode); + return; + } + + spin_lock(&info->lock); + /* Requeue the inode to shrinklist */ + list_del_init(&info->shrinklist); + shmem_unused_huge_lru_add(sbinfo, &info->shrinklist, nid, objcg); + info->shrinklist_objcg = objcg; + info->shrinklist_nid = nid; + spin_unlock(&info->lock); +} + static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, struct shrink_control *sc, unsigned long nr_to_free) { - LIST_HEAD(list), *pos, *next; + struct shmem_unused_huge_scan scan; struct inode *inode; struct shmem_inode_info *info; struct folio *folio; - unsigned long batch = sc ? sc->nr_to_scan : 128; + struct list_head *pos, *next; unsigned long split = 0, freed = 0; - if (list_empty(&sbinfo->shrinklist)) - return SHRINK_STOP; + INIT_LIST_HEAD(&scan.list); + scan.sc = sc; + if (sc) + list_lru_shrink_walk(&sbinfo->shrinklist, sc, + shmem_unused_huge_isolate, &scan); + else + list_lru_walk(&sbinfo->shrinklist, shmem_unused_huge_isolate, + &scan, 128); - spin_lock(&sbinfo->shrinklist_lock); - list_for_each_safe(pos, next, &sbinfo->shrinklist) { - info = list_entry(pos, struct shmem_inode_info, shrinklist); - - /* pin the inode */ - inode = igrab(&info->vfs_inode); - - /* inode is about to be evicted */ - if (!inode) { - list_del_init(&info->shrinklist); - goto next; - } - - list_move(&info->shrinklist, &list); -next: - sbinfo->shrinklist_len--; - if (!--batch) - break; - } - spin_unlock(&sbinfo->shrinklist_lock); - - list_for_each_safe(pos, next, &list) { - pgoff_t next, end; + list_for_each_safe(pos, next, &scan.list) { + pgoff_t folio_end, end; loff_t i_size; int ret; info = list_entry(pos, struct shmem_inode_info, shrinklist); inode = &info->vfs_inode; - if (nr_to_free && freed >= nr_to_free) - goto move_back; - i_size = i_size_read(inode); folio = filemap_get_entry(inode->i_mapping, i_size / PAGE_SIZE); if (!folio || xa_is_value(folio)) @@ -782,13 +989,19 @@ next: } /* Check if there is anything to gain from splitting */ - next = folio_next_index(folio); + folio_end = folio_next_index(folio); end = shmem_fallocend(inode, DIV_ROUND_UP(i_size, PAGE_SIZE)); - if (end <= folio->index || end >= next) { + if (end <= folio->index || end >= folio_end) { folio_put(folio); goto drop; } + if (!is_shmem_unused_huge_match(folio, scan.sc)) + goto move_back; + + if (nr_to_free && freed >= nr_to_free) + goto move_back; + /* * Move the inode on the list back to shrinklist if we failed * to lock the page at this time. @@ -796,35 +1009,30 @@ next: * Waiting for the lock may lead to deadlock in the * reclaim path. */ - if (!folio_trylock(folio)) { - folio_put(folio); + if (!folio_trylock(folio)) + goto move_back; + + if (!is_shmem_unused_huge_match(folio, scan.sc)) { + folio_unlock(folio); goto move_back; } ret = split_folio(folio); folio_unlock(folio); - folio_put(folio); /* If split failed move the inode on the list back to shrinklist */ if (ret) goto move_back; - freed += next - end; + freed += folio_end - end; split++; + folio_put(folio); drop: - list_del_init(&info->shrinklist); + shmem_unused_huge_drop(inode); goto put; move_back: - /* - * Make sure the inode is either on the global list or deleted - * from any local list before iput() since it could be deleted - * in another thread once we put the inode (then the local list - * is corrupted). - */ - spin_lock(&sbinfo->shrinklist_lock); - list_move(&info->shrinklist, &sbinfo->shrinklist); - sbinfo->shrinklist_len++; - spin_unlock(&sbinfo->shrinklist_lock); + shmem_unused_huge_requeue(inode, folio); + folio_put(folio); put: iput(inode); } @@ -837,7 +1045,7 @@ static long shmem_unused_huge_scan(struct super_block *sb, { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); - if (!READ_ONCE(sbinfo->shrinklist_len)) + if (!list_lru_shrink_count(&sbinfo->shrinklist, sc)) return SHRINK_STOP; return shmem_unused_huge_shrink(sbinfo, sc, 0); @@ -848,21 +1056,21 @@ static long shmem_unused_huge_count(struct super_block *sb, { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); - /* - * The per-superblock shrinklist is filesystem-global and does not - * honour sc->memcg, so it is only meaningful on the global (kswapd or - * root direct reclaim) shrink path. Skip the per-memcg iterations of - * shrink_slab_memcg() to avoid queueing duplicate global work. - */ - if (!mem_cgroup_shrink_is_root(sc)) - return 0; - - return READ_ONCE(sbinfo->shrinklist_len); + return list_lru_shrink_count(&sbinfo->shrinklist, sc); } #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ #define shmem_huge SHMEM_HUGE_DENY +static void shmem_unused_huge_add(struct inode *inode, struct folio *folio, + gfp_t gfp) +{ +} + +static void shmem_unused_huge_del(struct inode *inode) +{ +} + static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, struct shrink_control *sc, unsigned long nr_to_free) { @@ -1418,14 +1626,7 @@ static void shmem_evict_inode(struct inode *inode) inode->i_size = 0; mapping_set_exiting(inode->i_mapping); shmem_truncate_range(inode, 0, (loff_t)-1); - if (!list_empty(&info->shrinklist)) { - spin_lock(&sbinfo->shrinklist_lock); - if (!list_empty(&info->shrinklist)) { - list_del_init(&info->shrinklist); - sbinfo->shrinklist_len--; - } - spin_unlock(&sbinfo->shrinklist_lock); - } + shmem_unused_huge_del(inode); while (!list_empty(&info->swaplist)) { /* Wait while shmem_unuse() is scanning this inode... */ wait_var_event(&info->stop_eviction, @@ -2576,25 +2777,12 @@ clear: */ if (alloced && folio_test_large(folio) && DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < folio_next_index(folio)) { - struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); - struct shmem_inode_info *info = SHMEM_I(inode); /* * Part of the large folio is beyond i_size: subject * to shrink under memory pressure. */ - spin_lock(&sbinfo->shrinklist_lock); - /* - * _careful to defend against unlocked access to - * ->shrink_list in shmem_unused_huge_shrink() - */ - if (list_empty_careful(&info->shrinklist)) { - list_add_tail(&info->shrinklist, - &sbinfo->shrinklist); - sbinfo->shrinklist_len++; - } - spin_unlock(&sbinfo->shrinklist_lock); + shmem_unused_huge_add(inode, folio, gfp); } - out: *foliop = folio; return 0; @@ -3071,6 +3259,10 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, if (info->fsflags) shmem_set_inode_flags(inode, info->fsflags, NULL); INIT_LIST_HEAD(&info->shrinklist); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + info->shrinklist_objcg = NULL; + info->shrinklist_nid = -1; +#endif INIT_LIST_HEAD(&info->swaplist); cache_no_acl(inode); if (sbinfo->noswap) @@ -4945,6 +5137,9 @@ static void shmem_put_super(struct super_block *sb) #endif free_percpu(sbinfo->ino_batch); percpu_counter_destroy(&sbinfo->used_blocks); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + list_lru_destroy(&sbinfo->shrinklist); +#endif mpol_put(sbinfo->mpol); #ifdef CONFIG_TMPFS_XATTR simple_xattr_cache_cleanup(&sbinfo->xa_cache); @@ -5038,8 +5233,11 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) raw_spin_lock_init(&sbinfo->stat_lock); if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) goto failed; - spin_lock_init(&sbinfo->shrinklist_lock); - INIT_LIST_HEAD(&sbinfo->shrinklist); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (list_lru_init_memcg(&sbinfo->shrinklist, sb->s_shrink)) + goto failed; +#endif sb->s_maxbytes = MAX_LFS_FILESIZE; sb->s_blocksize = PAGE_SIZE; From 4dffce1cc5abea2d8d37472557b8f941caa1d44f Mon Sep 17 00:00:00 2001 From: Qinyun Tan Date: Wed, 2 Sep 2026 17:32:02 +0800 Subject: [PATCH 056/539] mm/list_lru: disable memcg awareness under cgroup_disable=memory __list_lru_init() only collapses a memcg-aware list_lru into plain per-node lists when kmem accounting is disabled (cgroup.memory=nokmem). When the memory controller is disabled entirely (cgroup_disable=memory), mem_cgroup_kmem_disabled() is false, so the lru stays memcg aware even though no object will ever be charged to a memcg. This is more than a semantic inconsistency. folio_memcg_list_lru_alloc() trusts list_lru_memcg_aware() and dereferences the folio's memcg, which is always NULL with the controller disabled. The only mainline caller, folio_memcg_alloc_deferred(), papers over this with an explicit mem_cgroup_disabled() check. The shmem unused-huge shrinker conversion ("mm: shmem: make unused huge shrinker memcg aware") adds a second caller without such a guard, so booting with cgroup_disable=memory and writing to a huge=always tmpfs oopses: BUG: unable to handle page fault for address: 0000000000000488 RIP: 0010:folio_memcg_list_lru_alloc+0x41/0xf0 Call Trace: shmem_get_folio_gfp+0x1cd/0x7c0 shmem_write_begin+0x5d/0x100 generic_perform_write+0x89/0x2a0 shmem_file_write_iter+0x82/0x90 vfs_write+0x256/0x410 ksys_write+0x61/0xe0 do_syscall_64+0x8d/0x460 entry_SYSCALL_64_after_hwframe+0x76/0x7e The faulting address is the offset of mem_cgroup->kmemcg_id, dereferenced on a NULL memcg in memcg_list_lru_allocated(): folio_memcg_list_lru_alloc() list_lru_memcg_aware() <- true, only nokmem checked memcg = folio_memcg(folio) <- NULL memcg_list_lru_allocated(memcg, lru) memcg->kmemcg_id <- NULL pointer dereference Check mem_cgroup_disabled() in __list_lru_init() so that all list_lrus fall back to plain per-node lists when the controller is disabled, matching what the shrinker side already does (shrinker_memcg_alloc() bails out on mem_cgroup_disabled()). This makes the mem_cgroup_disabled() check in callers unnecessary rather than mandatory. Link: https://lore.kernel.org/20260902093202.609559-1-qinyuntan@linux.alibaba.com Signed-off-by: Qinyun Tan Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Qi Zheng Cc: Roman Gushchin Cc: Shakeel Butt --- mm/list_lru.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/list_lru.c b/mm/list_lru.c index 36662d02ff96..a4522ca93ebc 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -671,7 +671,7 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware, struct shrinker *shr else lru->shrinker_id = -1; - if (mem_cgroup_kmem_disabled()) + if (mem_cgroup_disabled() || mem_cgroup_kmem_disabled()) memcg_aware = false; #endif From 5547ee3a76b2ec0c7f1a53547d83c4e5cc3bdbd4 Mon Sep 17 00:00:00 2001 From: Wilson Felipe Pereira Date: Fri, 28 Aug 2026 03:37:31 +0000 Subject: [PATCH 057/539] selftests/cgroup: test_zswap: wait for cgroup to unpopulate in test_zswap_writeback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "selftests/cgroup: fixes for test_zswap on single core VM", v4. This series fixes two test failures in test_zswap observed when running on a single-core VM (-smp 1) with 4GB of RAM. Patch 1 addresses a race condition in test_zswap_writeback() where waitpid() returns before the exiting child process is switched away by the kernel, causing an immediate write of "+memory" to cgroup.subtree_control to fail with -EBUSY. We fix this by waiting for cgroup.events to report "populated 0". Patch 2 fixes an implicit unsigned conversion bug in test_no_kmem_bypass() where small negative timing differences between debugfs stored_pages and cgroup zswapped bytes caused the comparison to falsely fail due to unsigned promotion. This patch (of 2): When running test_zswap on a single-core VM (-smp 1) with 4GB of RAM, test_zswap_writeback intermittently fails on the initial run after boot. In test_zswap_writeback(), after waitpid() reaps the child process created by test_zswap_writeback_one(), writing "+memory" to cgroup.subtree_control can fail with -EBUSY. Under cgroup v2, enabling domain subtree controllers is forbidden while any tasks remain in cgroup.procs. When a child process exits, exit_notify() wakes the parent process, allowing waitpid() to return immediately. However, the cgroup populated task count (nr_populated_csets) is only decremented when the exiting task is switched away via finish_task_switch() -> cgroup_task_dead(). On single-core systems, the parent runs before the dead child has been switched out, causing "+memory" to fail with -EBUSY if written immediately after waitpid() returns. Fix this by waiting for cgroup.events to report "populated 0\n" via cg_read_strcmp_wait() before enabling subtree control. Link: https://lore.kernel.org/20260828033741.2184560-1-wfelipe@google.com Link: https://lore.kernel.org/20260828033741.2184560-2-wfelipe@google.com Signed-off-by: Wilson Felipe Pereira Signed-off-by: Andrew Morton Acked-by: Michal Koutný Cc: Chengming Zhou Cc: Johannes Weiner Cc: Nhat Pham Cc: Shuah Khan Cc: Tejun Heo --- tools/testing/selftests/cgroup/test_zswap.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 8df54b59513a..b1abc94317e4 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -409,6 +409,8 @@ static int test_zswap_writeback(const char *root, bool wb) * Thus, the parent's setting shall be what's in effect. */ if (cg_write(test_group, "memory.zswap.max", "max")) goto out; + if (cg_read_strcmp_wait(test_group, "cgroup.events", "populated 0\n")) + goto out; if (cg_write(test_group, "cgroup.subtree_control", "+memory")) goto out; From 0567c7eaefd7ea1b26d76c46964f9a1d92120d7d Mon Sep 17 00:00:00 2001 From: Wilson Felipe Pereira Date: Fri, 28 Aug 2026 03:37:32 +0000 Subject: [PATCH 058/539] selftests/cgroup: test_zswap: fix implicit unsigned promotion bug in test_no_kmem_bypass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In test_no_kmem_bypass(), delta (stored_pages * page_size - zswapped) is checked against stored_pages * page_size / 4 to verify that the pages pushed to zswap belong to the test memory cgroup. Due to slight stat update timing differences, delta can evaluate to a small negative number (e.g. -5MB out of 1GB). Because delta is declared as a signed int and stored_pages is an unsigned size_t, C's usual arithmetic conversions implicitly promote a negative delta to a large unsigned 64-bit integer, causing `delta < stored_pages * page_size / 4` to falsely evaluate to 0 and fail the test. Fix this by declaring zswapped and delta as signed long long and comparing against a signed threshold, ensuring negative deltas correctly evaluate to true. Link: https://lore.kernel.org/20260828033741.2184560-3-wfelipe@google.com Fixes: a549f9f31561a ("selftests: cgroup: add test_zswap with no kmem bypass test") Signed-off-by: Wilson Felipe Pereira Signed-off-by: Andrew Morton Acked-by: Michal Koutný Cc: Chengming Zhou Cc: Johannes Weiner Cc: Nhat Pham Cc: Shuah Khan Cc: Tejun Heo --- tools/testing/selftests/cgroup/test_zswap.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index b1abc94317e4..1ac779072775 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -654,11 +654,14 @@ static int test_no_kmem_bypass(const char *root) break; /* If memory was pushed to zswap, verify it belongs to memcg */ if (stored_pages > stored_pages_threshold) { - int zswapped = cg_read_key_long(test_group, "memory.stat", "zswapped "); - int delta = stored_pages * page_size - zswapped; - int result_ok = delta < stored_pages * page_size / 4; + long zswapped = cg_read_key_long( + test_group, "memory.stat", "zswapped "); + long long delta = + (long long)stored_pages * page_size - zswapped; + long long max_delta = + (long long)stored_pages * page_size / 4; - ret = result_ok ? KSFT_PASS : KSFT_FAIL; + ret = (delta < max_delta) ? KSFT_PASS : KSFT_FAIL; break; } } From d2f28a1a652e0a43f6f63de9b4e4d8d5f2716803 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Fri, 28 Aug 2026 13:50:36 -0400 Subject: [PATCH 059/539] mm/memcontrol: fix stuck FLUSHING_CACHED_CHARGE bit on isolated cpus When drain_all_stock() sets FLUSHING_CACHED_CHARGE before checking isolation, schedule_drain_work() can drop the work in a separate RCU critical section, and housekeeping_update()'s synchronize_rcu() can race that second check, leaving the flag set. drain_local_stock() only clears the bit for work that ran, so the flag remains set and the stock is never drained again. Have schedule_drain_work() return whether the work was queued, and clear FLUSHING_CACHED_CHARGE in drain_all_stock() when the remote CPU is isolated, so future drains can retry. Link: https://lore.kernel.org/20260828135036.7d44361f@fangorn Fixes: 6a792697a53a ("memcg: do not drain charge pcp caches on remote isolated cpus") Suggested-by: Michal Hocko Suggested-by: Shakeel Butt Signed-off-by: Rik van Riel Signed-off-by: Andrew Morton Acked-by: Shakeel Butt Acked-by: Michal Hocko Cc: Johannes Weiner Cc: Muchun Song Cc: Roman Gushchin Cc: --- mm/memcontrol.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1709ac96bbde..c6b85e3a5a0b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2306,7 +2306,7 @@ static bool is_memcg_drain_needed(struct memcg_stock_pcp *stock, return flush; } -static void schedule_drain_work(int cpu, struct work_struct *work) +static bool schedule_drain_work(int cpu, struct work_struct *work) { /* * Protect housekeeping cpumask read and work enqueue together @@ -2315,8 +2315,11 @@ static void schedule_drain_work(int cpu, struct work_struct *work) * pending work on newly isolated CPUs. */ guard(rcu)(); - if (!cpu_is_isolated(cpu)) - queue_work_on(cpu, memcg_wq, work); + if (cpu_is_isolated(cpu)) + return false; + + queue_work_on(cpu, memcg_wq, work); + return true; } /* @@ -2348,8 +2351,9 @@ void drain_all_stock(struct mem_cgroup *root_memcg) &memcg_st->flags)) { if (cpu == curcpu) drain_local_memcg_stock(&memcg_st->work); - else - schedule_drain_work(cpu, &memcg_st->work); + else if (!schedule_drain_work(cpu, &memcg_st->work)) + clear_bit(FLUSHING_CACHED_CHARGE, + &memcg_st->flags); } if (!test_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags) && @@ -2358,8 +2362,9 @@ void drain_all_stock(struct mem_cgroup *root_memcg) &obj_st->flags)) { if (cpu == curcpu) drain_local_obj_stock(&obj_st->work); - else - schedule_drain_work(cpu, &obj_st->work); + else if (!schedule_drain_work(cpu, &obj_st->work)) + clear_bit(FLUSHING_CACHED_CHARGE, + &obj_st->flags); } } migrate_enable(); From c21581cb7599f96f889423c4c3a1a29d4c0e93fd Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 28 Aug 2026 12:24:19 -0700 Subject: [PATCH 060/539] memcg: clear FLUSHING_CACHED_CHARGE on cpu offline Sashiko [1] reported that memcg_hotplug_cpu_dead() drains the stocks of the CPU which went away but leaves FLUSHING_CACHED_CHARGE alone. The flag can be set at that point: drain_all_stock() may have claimed the stock and queued the drain work shortly before the CPU went down. workqueue_offline_cpu() unbinds the per-cpu workers, so such a pending work item is executed by an unbound worker on some other CPU, where drain_local_memcg_stock() operates on this_cpu_ptr() and thus drains and clears the flag of that other CPU instead. Nothing clears the flag of the dead CPU, so drain_all_stock() would skip its stock forever once the CPU comes back online. Clear the flag of both stocks after draining them. Link: https://lore.kernel.org/20260828192419.3057939-1-shakeel.butt@linux.dev Link: https://sashiko.dev/#/patchset/20260828135036.7d44361f%40fangorn [1] Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Reviewed-by: Rik van Riel Reported-by: Sashiko Acked-by: Michal Hocko Cc: Johannes Weiner Cc: Muchun Song Cc: Roman Gushchin --- mm/memcontrol.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index c6b85e3a5a0b..05f5338765a9 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2373,9 +2373,21 @@ void drain_all_stock(struct mem_cgroup *root_memcg) static int memcg_hotplug_cpu_dead(unsigned int cpu) { + struct memcg_stock_pcp *memcg_st = &per_cpu(memcg_stock, cpu); + struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu); + /* no need for the local lock */ - drain_obj_stock(&per_cpu(obj_stock, cpu)); - drain_stock_fully(&per_cpu(memcg_stock, cpu)); + drain_obj_stock(obj_st); + drain_stock_fully(memcg_st); + + /* + * A drain work queued before the CPU went away is executed by an + * unbound worker on some other CPU and clears that CPU's flag, so + * clear the flags here to make these stocks drainable again once + * the CPU comes back online. + */ + clear_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags); + clear_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags); return 0; } From bbeeb4a29dc4ca6ac18ab3bafa3728bc984a4cde Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Fri, 28 Aug 2026 15:31:11 -0400 Subject: [PATCH 061/539] mm/mempolicy: take a cpuset cookie for the interleave node count alloc_pages_bulk_interleave() counts pol->nodes without a cpuset cookie: nodes = nodes_weight(pol->nodes); nr_pages_per_node = nr_pages / nodes; nodemask_t spans several words once MAX_NUMNODES exceeds BITS_PER_LONG, so a concurrent cpuset rebind can tear that read and yield an empty mask even though neither version of it was empty. The call then allocates nothing and returns 0. Some compilers will hoist the loop entry test above the division, because nr_pages_per_node is dead when the loop does not run. 682e: call ... <- nodes_weight() 6838: test %eax,%eax 683a: jle 692d <- nodes <= 0 skips the loop 684a: div %rcx So in most deployments, this div/0 is unreachable - but nothing in the source guarantees that, it's just not easily exercised. Take the cookie around the count and bail if the mask really is empty. Only the count needs it, interleave_nodes() takes the cookie itself so so a torn read there is already retried. A rebind landing mid-loop can still leave the count disagreeing with the mask, so the loop may revisit a node or skip one - but a rebind where nodes change causes migration, so a handful of misplaced pages isn't catastrophic in any sense. Measured on a 72 node VM (NODES_SHIFT=10) with a cgroup v2 cpuset flipping cpuset.mems between a word 0 and a word 1 node set, and the two word read artificially widened: 330 zero counts in 130414 calls without the cookie, and 401 retries with it. Link: https://lore.kernel.org/20260828193111.1023497-1-gourry@gourry.net Fixes: c00b6b961099 ("mm/vmalloc: introduce alloc_pages_bulk_array_mempolicy to accelerate memory allocation") Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Reported-by: Chelsy Ratnawat Closes: https://lore.kernel.org/all/20250907160829.91628-1-chelsyratnawat2001@gmail.com/ Reviewed-by: Huang Ying Assisted-by: Claude:claude-opus-5 Cc: Alistair Popple Cc: Byungchul Park Cc: Chenwandun Cc: David Hildenbrand Cc: Joshua Hahn Cc: Matthew Brost Cc: Rakie Kim Cc: "Uladzislau Rezki (Sony)" Cc: Zi Yan Cc: --- mm/mempolicy.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 79053ece02cd..060a0eb26917 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2592,6 +2592,7 @@ static unsigned long alloc_pages_bulk_interleave(gfp_t gfp, struct mempolicy *pol, unsigned long nr_pages, struct page **page_array) { + unsigned int cpuset_mems_cookie; int nodes; unsigned long nr_pages_per_node; int delta; @@ -2599,7 +2600,16 @@ static unsigned long alloc_pages_bulk_interleave(gfp_t gfp, unsigned long nr_allocated; unsigned long total_allocated = 0; - nodes = nodes_weight(pol->nodes); + /* count the nodes, retry if a rebind happened during the read */ + do { + cpuset_mems_cookie = read_mems_allowed_begin(); + nodes = nodes_weight(pol->nodes); + } while (read_mems_allowed_retry(cpuset_mems_cookie)); + + /* if the nodemask has become invalid, we cannot do anything */ + if (!nodes) + return 0; + nr_pages_per_node = nr_pages / nodes; delta = nr_pages - nodes * nr_pages_per_node; From 67f96334148d5ff30c7475346913b523506c997d Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Wed, 19 Aug 2026 10:16:09 +0800 Subject: [PATCH 062/539] tools/mm/page_owner_sort: fix --sort option being silently ignored Patch series "tools/mm/page_owner_sort: fix --sort, add module filter, improve usage", v3. This series improves the page_owner_sort tool with a bug fix, a new module-name feature, and better usage text. Patch 1 fixes a long-standing bug where --sort was silently ignored when used without a short option (-a, -m, -p, etc.). The COMP_NO_FLAG case fell through to COMP_NUM and overwrote the sort conditions configured by parse_sort_args(). Patch 2 adds kernel module name support for sort, cull, and filter operations. Page owner stack traces already contain module names in the "function+0xNN/0xNN [module]" format produced by %pS, but page_owner_sort had no way to use them. Records without module frames are assigned "vmlinux". # Aggregate page usage per module ./page_owner_sort input.txt output.txt --cull=mod # Filter to records from xfs module only ./page_owner_sort input.txt output.txt --module xfs # Sort by module name, then by pid descending ./page_owner_sort input.txt output.txt --sort=mod,-pid Patch 3 lists all available sort keys with abbreviations and examples directly in the --sort help section so users no longer need to read the source to discover valid keys. This patch (of 3): When --sort is used without any short option (-a, -m, -p, etc.), compare_flag remains COMP_NO_FLAG. The switch (compare_flag) then falls through to the COMP_NUM case and calls set_single_cmp(), which unconditionally overwrites the sort conditions that parse_sort_args() already configured. This makes --sort silently ineffective unless a short option is also supplied. Split COMP_NO_FLAG out of the COMP_NUM fallthrough so that --sort is respected when no short option is present. Reproduction: # Before fix: ascending order (ignored --sort=-pid) ./page_owner_sort --sort=-pid input.txt output.txt # After fix: descending order as expected Link: https://lore.kernel.org/20260819021611.2910835-1-ye.liu@linux.dev Link: https://lore.kernel.org/20260819021611.2910835-2-ye.liu@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Liu Jing Cc: Yichong Chen --- tools/mm/page_owner_sort.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index 22b3b500d33a..f1dc0763c25c 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -821,6 +821,9 @@ int main(int argc, char **argv) set_single_cmp(compare_stacktrace, SORT_ASC); break; case COMP_NO_FLAG: + if (sc.size > 0) + break; + /* fallthrough */ case COMP_NUM: set_single_cmp(compare_num, SORT_DESC); break; From 305a263d89c96c157ca568a4872c1eb09e90e9f1 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Wed, 19 Aug 2026 10:16:10 +0800 Subject: [PATCH 063/539] tools/mm/page_owner_sort: add module name sort/cull/filter support Page owner stack traces already contain kernel module names in the "[module]" format produced by %pS, but page_owner_sort has no way to sort, cull, or filter by module. Extract the first module name from each record's stack trace using the regex \[([a-zA-Z0-9_]+)\]. Records whose stack traces contain no module frames are assigned "vmlinux". New options: -M Sort by module name --sort=mod Sort by module name (supports +/- prefix) --cull=mod Cull (aggregate) by module name --module Filter to records matching the given module(s) The module field is also printed in cull output when relevant. Link: https://lore.kernel.org/20260819021611.2910835-3-ye.liu@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Cc: David Hildenbrand (Arm) Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Liu Jing Cc: Yichong Chen --- Documentation/mm/page_owner.rst | 8 ++- tools/mm/page_owner_sort.c | 122 +++++++++++++++++++++++++++----- 2 files changed, 110 insertions(+), 20 deletions(-) diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst index a6bd3fe6423a..bd027377dff6 100644 --- a/Documentation/mm/page_owner.rst +++ b/Documentation/mm/page_owner.rst @@ -199,6 +199,7 @@ Usage -p Sort by pid. -P Sort by tgid. -n Sort by task command name. + -M Sort by module name. -r Sort by memory release time. -s Sort by stack trace. -t Sort by times (default). @@ -240,8 +241,10 @@ Usage group ID numbers appear in . --name Select by task command name. This selects the blocks whose task command name appear in . + --module Select by module name. This selects the blocks whose + module name appear in . - , , are single arguments in the form of a comma-separated list, + , , , are single arguments in the form of a comma-separated list, which offers a way to specify individual selecting rules. @@ -249,6 +252,7 @@ Usage ./page_owner_sort --pid=1 ./page_owner_sort --tgid=1,2,3 ./page_owner_sort --name name1,name2 + ./page_owner_sort --module xfs,ext4 STANDARD FORMAT SPECIFIERS ========================== @@ -265,6 +269,7 @@ STANDARD FORMAT SPECIFIERS ft free_ts timestamp of the page when it was released at alloc_ts timestamp of the page when it was allocated ator allocator memory allocator for pages + mod module kernel module name For --cull option: @@ -275,6 +280,7 @@ STANDARD FORMAT SPECIFIERS f free whether the page has been released or not st stacktrace stack trace of the page allocation ator allocator memory allocator for pages + mod module kernel module name Filtering page_owner output ============================ diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index f1dc0763c25c..a5b61b4abc2f 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -25,11 +25,13 @@ #include #define TASK_COMM_LEN 16 +#define MODULE_NAME_LEN 64 struct block_list { char *txt; char *comm; // task command name char *stacktrace; + char *module; // kernel module name __u64 ts_nsec; int len; int num; @@ -41,7 +43,8 @@ struct block_list { enum FILTER_BIT { FILTER_PID = 1<<1, FILTER_TGID = 1<<2, - FILTER_COMM = 1<<3 + FILTER_COMM = 1<<3, + FILTER_MODULE = 1<<4 }; enum FILTER_RESULT { @@ -55,7 +58,8 @@ enum CULL_BIT { CULL_TGID = 1<<2, CULL_COMM = 1<<3, CULL_STACKTRACE = 1<<4, - CULL_ALLOCATOR = 1<<5 + CULL_ALLOCATOR = 1<<5, + CULL_MODULE = 1<<6 }; enum ALLOCATOR_BIT { ALLOCATOR_CMA = 1<<1, @@ -65,7 +69,8 @@ enum ALLOCATOR_BIT { }; enum ARG_TYPE { ARG_TXT, ARG_COMM, ARG_STACKTRACE, ARG_ALLOC_TS, ARG_CULL_TIME, - ARG_PAGE_NUM, ARG_PID, ARG_TGID, ARG_UNKNOWN, ARG_ALLOCATOR + ARG_PAGE_NUM, ARG_PID, ARG_TGID, ARG_UNKNOWN, ARG_ALLOCATOR, + ARG_MODULE }; enum SORT_ORDER { SORT_ASC = 1, @@ -79,15 +84,18 @@ enum COMP_FLAG { COMP_STACK = 1<<3, COMP_NUM = 1<<4, COMP_TGID = 1<<5, - COMP_COMM = 1<<6 + COMP_COMM = 1<<6, + COMP_MODULE = 1<<7 }; struct filter_condition { pid_t *pids; pid_t *tgids; char **comms; + char **modules; int pids_size; int tgids_size; int comms_size; + int modules_size; }; struct sort_condition { int (**cmps)(const void *, const void *); @@ -101,6 +109,7 @@ static regex_t pid_pattern; static regex_t tgid_pattern; static regex_t comm_pattern; static regex_t ts_nsec_pattern; +static regex_t module_pattern; static struct block_list *list; static int list_size; static int max_size; @@ -184,6 +193,13 @@ static int compare_comm(const void *p1, const void *p2) return strcmp(l1->comm, l2->comm); } +static int compare_module(const void *p1, const void *p2) +{ + const struct block_list *l1 = p1, *l2 = p2; + + return strcmp(l1->module, l2->module); +} + static int compare_ts(const void *p1, const void *p2) { const struct block_list *l1 = p1, *l2 = p2; @@ -207,6 +223,8 @@ static int compare_cull_condition(const void *p1, const void *p2) return compare_tgid(p1, p2); if ((cull & CULL_COMM) && compare_comm(p1, p2)) return compare_comm(p1, p2); + if ((cull & CULL_MODULE) && compare_module(p1, p2)) + return compare_module(p1, p2); if ((cull & CULL_ALLOCATOR) && compare_allocator(p1, p2)) return compare_allocator(p1, p2); return 0; @@ -411,9 +429,33 @@ static char *get_comm(char *buf) return comm_str; } +static char *get_module(char *buf) +{ + char *module_str = malloc(MODULE_NAME_LEN); + regmatch_t pmatch[2]; + int val_len; + + if (!module_str) + return NULL; + memset(module_str, 0, MODULE_NAME_LEN); + if (regexec(&module_pattern, buf, 2, pmatch, REG_NOTBOL) != 0 || pmatch[1].rm_so == -1) { + strcpy(module_str, "vmlinux"); + return module_str; + } + + val_len = pmatch[1].rm_eo - pmatch[1].rm_so; + if ((size_t)val_len >= MODULE_NAME_LEN) + val_len = MODULE_NAME_LEN - 1; + memcpy(module_str, buf + pmatch[1].rm_so, val_len); + module_str[val_len] = '\0'; + + return module_str; +} + static void free_block_list(struct block_list *block) { free(block->comm); + free(block->module); free(block->txt); } @@ -433,6 +475,8 @@ static int get_arg_type(const char *arg) return ARG_ALLOC_TS; else if (!strcmp(arg, "allocator") || !strcmp(arg, "ator")) return ARG_ALLOCATOR; + else if (!strcmp(arg, "module") || !strcmp(arg, "mod")) + return ARG_MODULE; else { return ARG_UNKNOWN; } @@ -483,25 +527,36 @@ static bool match_str_list(const char *str, char **list, int list_size) static enum FILTER_RESULT filter_record(char *buf) { - char *comm; + char *comm, *module; if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size)) return FILTER_SKIP; if ((filter & FILTER_TGID) && !match_num_list(get_tgid(buf), fc.tgids, fc.tgids_size)) return FILTER_SKIP; - if (!(filter & FILTER_COMM)) + if (!(filter & (FILTER_COMM | FILTER_MODULE))) return FILTER_MATCH; - comm = get_comm(buf); - if (!comm) - return FILTER_ERROR; - - if (!match_str_list(comm, fc.comms, fc.comms_size)) { + if (filter & FILTER_COMM) { + comm = get_comm(buf); + if (!comm) + return FILTER_ERROR; + if (!match_str_list(comm, fc.comms, fc.comms_size)) { + free(comm); + return FILTER_SKIP; + } free(comm); - return FILTER_SKIP; } - free(comm); + if (filter & FILTER_MODULE) { + module = get_module(buf); + if (!module) + return FILTER_ERROR; + if (!match_str_list(module, fc.modules, fc.modules_size)) { + free(module); + return FILTER_SKIP; + } + free(module); + } return FILTER_MATCH; } @@ -547,6 +602,12 @@ static bool add_list(char *buf, int len, char *ext_buf) list[list_size].stacktrace++; list[list_size].ts_nsec = get_ts_nsec(buf); list[list_size].allocator = get_allocator(buf, ext_buf); + list[list_size].module = get_module(buf); + if (!list[list_size].module) { + fprintf(stderr, "Out of memory\n"); + free_block_list(&list[list_size]); + return false; + } list_size++; if (list_size % 1000 == 0) { printf("loaded %d\r", list_size); @@ -573,6 +634,8 @@ static bool parse_cull_args(const char *arg_str) cull |= CULL_STACKTRACE; else if (arg_type == ARG_ALLOCATOR) cull |= CULL_ALLOCATOR; + else if (arg_type == ARG_MODULE) + cull |= CULL_MODULE; else { free_explode(args, size); return false; @@ -635,6 +698,8 @@ static bool parse_sort_args(const char *arg_str) sc.cmps[i] = compare_txt; else if (arg_type == ARG_ALLOCATOR) sc.cmps[i] = compare_allocator; + else if (arg_type == ARG_MODULE) + sc.cmps[i] = compare_module; else { free_explode(args, size); sc.size = 0; @@ -691,7 +756,8 @@ static void usage(void) "-p\t\t\tSort by pid.\n" "-P\t\t\tSort by tgid.\n" "-s\t\t\tSort by the stacktrace.\n" - "-t\t\t\tSort by number of times record is seen (default).\n\n" + "-t\t\t\tSort by number of times record is seen (default).\n" + "-M\t\t\tSort by module name.\n\n" "--pid \t\tSelect by pid. This selects the information" " of\n\t\t\tblocks whose process ID numbers appear in .\n" "--tgid \tSelect by tgid. This selects the information" @@ -700,10 +766,11 @@ static void usage(void) "--name \tSelect by command name. This selects the" " information\n\t\t\tof blocks whose command name appears in" " .\n" - "--cull \t\tCull by user-defined rules. is a " - "single\n\t\t\targument in the form of a comma-separated list " - "with some\n\t\t\tcommon fields predefined (pid, tgid, comm, " - "stacktrace, allocator)\n" + "--module \tSelect by module name. This selects the information\n" + "\t\t\tof blocks whose module name appears in .\n" + "--cull \t\tCull by user-defined rules. is a single\n" + "\t\t\targument in the form of a comma-separated list with some\n" + "\t\t\tcommon fields predefined (pid, tgid, comm, stacktrace, allocator, module)\n" "--sort \t\tSpecify sort order as: [+|-]key[,[+|-]key[,...]]\n" ); } @@ -721,13 +788,14 @@ int main(int argc, char **argv) { "name", required_argument, NULL, 3 }, { "cull", required_argument, NULL, 4 }, { "sort", required_argument, NULL, 5 }, + { "module", required_argument, NULL, 6 }, { "help", no_argument, NULL, 'h' }, { 0, 0, 0, 0}, }; compare_flag = COMP_NO_FLAG; - while ((opt = getopt_long(argc, argv, "admnpstPh", longopts, NULL)) != -1) + while ((opt = getopt_long(argc, argv, "admnpstPMh", longopts, NULL)) != -1) switch (opt) { case 'a': compare_flag |= COMP_ALLOC; @@ -753,6 +821,9 @@ int main(int argc, char **argv) case 'n': compare_flag |= COMP_COMM; break; + case 'M': + compare_flag |= COMP_MODULE; + break; case 'h': usage(); exit(0); @@ -792,6 +863,10 @@ int main(int argc, char **argv) exit(1); } break; + case 6: + filter = filter | FILTER_MODULE; + fc.modules = explode(',', optarg, &fc.modules_size); + break; default: usage(); exit(1); @@ -833,6 +908,9 @@ int main(int argc, char **argv) case COMP_COMM: set_single_cmp(compare_comm, SORT_ASC); break; + case COMP_MODULE: + set_single_cmp(compare_module, SORT_ASC); + break; default: usage(); exit(1); @@ -855,6 +933,8 @@ int main(int argc, char **argv) goto out_comm; if (!check_regcomp(&ts_nsec_pattern, "ts\\s*([0-9]*)\\s*ns")) goto out_ts; + if (!check_regcomp(&module_pattern, "\\+0x[0-9a-f]+/0x[0-9a-f]+\\s*\\[([a-zA-Z0-9_-]+)\\]")) + goto out_module; fstat(fileno(fin), &st); max_size = st.st_size / 100; /* hack ... */ @@ -920,6 +1000,8 @@ int main(int argc, char **argv) fprintf(fout, ", TGID %d", list[i].tgid); if (cull & CULL_COMM || filter & FILTER_COMM) fprintf(fout, ", task_comm_name: %s", list[i].comm); + if (cull & CULL_MODULE || filter & FILTER_MODULE) + fprintf(fout, ", module: %s", list[i].module); if (cull & CULL_ALLOCATOR) { fprintf(fout, ", "); print_allocator(fout, list[i].allocator); @@ -940,6 +1022,8 @@ out_free: free_block_list(&list[i]); free(list); } +out_module: + regfree(&module_pattern); out_ts: regfree(&ts_nsec_pattern); out_comm: From 611aebcec3f3051457619fb1b9bebc84d4136747 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Wed, 19 Aug 2026 10:16:11 +0800 Subject: [PATCH 064/539] tools/mm/page_owner_sort: show available sort keys in usage text The --sort option accepts abbreviated or complete key names, but the usage text never listed them. Users had to read the source or the documentation to discover valid keys. List all available keys (full form and abbreviation) with a brief description and examples directly in the --sort help section. Link: https://lore.kernel.org/20260819021611.2910835-4-ye.liu@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yichong Chen Cc: Liu Jing --- tools/mm/page_owner_sort.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index a5b61b4abc2f..6c2ac5d9dc4b 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -772,6 +772,15 @@ static void usage(void) "\t\t\targument in the form of a comma-separated list with some\n" "\t\t\tcommon fields predefined (pid, tgid, comm, stacktrace, allocator, module)\n" "--sort \t\tSpecify sort order as: [+|-]key[,[+|-]key[,...]]\n" + "\t\t\tAvailable keys:\n" + "\t\t\t pid(p), tgid(tg), name(n), stacktrace(st),\n" + "\t\t\t txt(T), alloc_ts(at), allocator(ator), module(mod)\n" + "\t\t\tThe \"+\" is optional since default direction is\n" + "\t\t\tincreasing numerical or lexicographic order.\n" + "\t\t\tMixed use of abbreviated and complete-form is allowed.\n" + "\t\t\tExamples:\n" + "\t\t\t --sort=n,+pid,-tgid\n" + "\t\t\t --sort=mod,at\n" ); } From f2d21fb73df638d49d78f8cca3373dfc2109d821 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 19 Aug 2026 18:20:10 -0700 Subject: [PATCH 065/539] memcg: trim the per-cpu charge stock instead of draining it Joy reported that an application generating a request/response traffic pattern spends 44.6% to 57.0% of CPU in the memcg charge/uncharge path for a range of message sizes, against 0.27% to 0.71% outside that range. Running from the root memcg, where socket memory accounting is skipped, recovers the performance. Tracing the charge path showed that the application generates a pattern where the write syscall charges one page and the read syscall uncharges two pages on the same CPU. This hits a corner case in the memcg percpu stock code that thrashes the stock continuously. In the memcg percpu stock code, MEMCG_CHARGE_BATCH (64) is both the high watermark and the emptying target, i.e. on a request to charge one page the kernel charges MEMCG_CHARGE_BATCH pages and caches (MEMCG_CHARGE_BATCH - 1) of them in the percpu stock. The following uncharge of 2 pages takes the cached count to (MEMCG_CHARGE_BATCH + 1), and refill_stock() then empties the cache completely. With such a pattern the percpu stock becomes completely ineffective. Instead of a single boundary point for charges, use the technique the page allocator uses for its own percpu caches, which keeps the watermark and the emptying target apart: nr_pcp_free() frees between batch and high - batch pages, leaving at least pcp->batch on the list. Add a high watermark MEMCG_STOCK_HIGH and, once the cached count goes over it, return only the pages above MEMCG_STOCK_LOW. The watermarks are MEMCG_CHARGE_BATCH apart, so a page_counter update still covers a full batch. For now, keep MEMCG_STOCK_HIGH same as MEMCG_CHARGE_BATCH and in future we will reevaluate if it makes sense to increase it. Link: https://lore.kernel.org/20260820012010.2016086-1-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Reported-by: Joy Chaoyue Xiong Acked-by: Michal Hocko Cc: Jakub Kacinski Cc: Johannes Weiner Cc: Joshua Hahn Cc: Muchun Song Cc: Roman Gushchin --- mm/memcontrol.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 05f5338765a9..bfd0a74fac92 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2048,6 +2048,15 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) * nr_pages in a single cacheline. This may change in future. */ #define NR_MEMCG_STOCK 7 + +/* + * Watermarks for a charge stock slot, in the spirit of pcp->high and + * pcp->batch: MEMCG_STOCK_HIGH is the high watermark at which a slot is + * trimmed, and it is trimmed down to MEMCG_STOCK_LOW rather than emptied. + */ +#define MEMCG_STOCK_LOW (MEMCG_CHARGE_BATCH / 2) +#define MEMCG_STOCK_HIGH (MEMCG_CHARGE_BATCH) + #define FLUSHING_CACHED_CHARGE 0 struct memcg_stock_pcp { local_trylock_t lock; @@ -2228,17 +2237,18 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { struct memcg_stock_pcp *stock; struct mem_cgroup *cached; - uint8_t stock_pages; + unsigned int stock_pages; bool success = false; int empty_slot = -1; int i; /* - * For now limit MEMCG_CHARGE_BATCH to 127 and less. In future if we - * decide to increase it more than 127 then we will need more careful - * handling of nr_pages[] in struct memcg_stock_pcp. + * nr_pages[] is a uint8_t and a slot's count is capped at + * MEMCG_STOCK_HIGH. Raising MEMCG_CHARGE_BATCH beyond 127 would need + * more careful handling of nr_pages[] in struct memcg_stock_pcp. */ BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S8_MAX); + BUILD_BUG_ON(MEMCG_STOCK_HIGH > U8_MAX); VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg)); @@ -2259,9 +2269,12 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) empty_slot = i; if (memcg == READ_ONCE(stock->cached[i])) { stock_pages = READ_ONCE(stock->nr_pages[i]) + nr_pages; + if (stock_pages > MEMCG_STOCK_HIGH) { + memcg_uncharge(memcg, + stock_pages - MEMCG_STOCK_LOW); + stock_pages = MEMCG_STOCK_LOW; + } WRITE_ONCE(stock->nr_pages[i], stock_pages); - if (stock_pages > MEMCG_CHARGE_BATCH) - drain_stock(stock, i); success = true; break; } From 91163e5fc8441d5f42d73c63c433a9eeeb947a7f Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 2 Sep 2026 10:43:04 -0700 Subject: [PATCH 066/539] memcg: remove v1 soft limit reclaim Patch series "memcg: remove the v1 soft limit", v2. The v1 soft limit was deprecated in v6.12 by commit 569c4f62d84a ("memcg: initiate deprecation of v1 soft limit") and nobody has reported depending on it in the ~21 months since. memory.low and memory.min in v2 have covered the same ground for far longer. The knob has since been made inert by "memcg: make the v1 soft limit knob inert", already queued in mm-hotfixes as a backportable fix for a syzbot report [1]. Nothing can enter the soft limit rbtree anymore, so this series just deletes the machinery that is now dead: the reclaim pass in kswapd and direct reclaim, mem_cgroup_shrink_node() and its tracepoints, the per-node rbtree, lru_gen_soft_reclaim() and the MEMCG_LRU_HEAD op, the per-node tree fields, mem_cgroup->soft_limit, and finally the v1 event ratelimiting which is now down to a single target. memory.soft_limit_in_bytes itself is untouched: writes stay ignored and reads keep returning the maximum value. This patch (of 8): Nothing can put a cgroup on the soft limit rbtree anymore, so the tree is always empty and both callers of memcg1_soft_limit_reclaim() are guaranteed no-ops. Remove the reclaim pass from direct reclaim and from kswapd, along with its implementation. In shrink_zones() this leaves the global reclaim branch with a last_pgdat check that is now redundant with the identical check right below it, so drop it and move the explaining comment down to the check that remains. That check could only ever fire once last_pgdat was set, which implies first_pgdat had already been assigned, so skipping it does not change which node consider_reclaim_throttle() gets. Link: https://lore.kernel.org/20260902174311.1772372-1-shakeel.butt@linux.dev Link: https://lore.kernel.org/20260902174311.1772372-2-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Muchun Song Cc: Roman Gushchin Cc: T.J. Mercier --- include/linux/memcontrol.h | 12 --- mm/memcontrol-v1.c | 175 ------------------------------------- mm/vmscan.c | 39 ++------- 3 files changed, 6 insertions(+), 220 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index da625d2edb3b..11c1fa88d6fd 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1924,10 +1924,6 @@ static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) /* Cgroup v1-related declarations */ #ifdef CONFIG_MEMCG_V1 -unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, - gfp_t gfp_mask, - unsigned long *total_scanned); - bool mem_cgroup_oom_synchronize(bool wait); static inline bool task_in_memcg_oom(struct task_struct *p) @@ -1948,14 +1944,6 @@ static inline void mem_cgroup_exit_user_fault(void) } #else /* CONFIG_MEMCG_V1 */ -static inline -unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, - gfp_t gfp_mask, - unsigned long *total_scanned) -{ - return 0; -} - static inline bool task_in_memcg_oom(struct task_struct *p) { return false; diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 05ef55cae4dc..b38b8d0f7f51 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -34,13 +34,6 @@ struct mem_cgroup_tree { static struct mem_cgroup_tree soft_limit_tree __read_mostly; -/* - * Maximum loops in mem_cgroup_soft_reclaim(), used for soft - * limit reclaim to prevent infinite loops, if they ever occur. - */ -#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 -#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 - /* for OOM */ struct mem_cgroup_eventfd_list { struct list_head list; @@ -233,174 +226,6 @@ void memcg1_remove_from_trees(struct mem_cgroup *memcg) } } -static struct mem_cgroup_per_node * -__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) -{ - struct mem_cgroup_per_node *mz; - -retry: - mz = NULL; - if (!mctz->rb_rightmost) - goto done; /* Nothing to reclaim from */ - - mz = rb_entry(mctz->rb_rightmost, - struct mem_cgroup_per_node, tree_node); - /* - * Remove the node now but someone else can add it back, - * we will to add it back at the end of reclaim to its correct - * position in the tree. - */ - __mem_cgroup_remove_exceeded(mz, mctz); - if (!soft_limit_excess(mz->memcg) || - !css_tryget(&mz->memcg->css)) - goto retry; -done: - return mz; -} - -static struct mem_cgroup_per_node * -mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) -{ - struct mem_cgroup_per_node *mz; - - spin_lock_irq(&mctz->lock); - mz = __mem_cgroup_largest_soft_limit_node(mctz); - spin_unlock_irq(&mctz->lock); - return mz; -} - -static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, - pg_data_t *pgdat, - gfp_t gfp_mask, - unsigned long *total_scanned) -{ - struct mem_cgroup *victim = NULL; - int total = 0; - int loop = 0; - unsigned long excess; - unsigned long nr_scanned; - struct mem_cgroup_reclaim_cookie reclaim = { - .pgdat = pgdat, - }; - - excess = soft_limit_excess(root_memcg); - - while (1) { - victim = mem_cgroup_iter(root_memcg, victim, &reclaim); - if (!victim) { - loop++; - if (loop >= 2) { - /* - * If we have not been able to reclaim - * anything, it might because there are - * no reclaimable pages under this hierarchy - */ - if (!total) - break; - /* - * We want to do more targeted reclaim. - * excess >> 2 is not to excessive so as to - * reclaim too much, nor too less that we keep - * coming back to reclaim from this cgroup - */ - if (total >= (excess >> 2) || - (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) - break; - } - continue; - } - total += mem_cgroup_shrink_node(victim, gfp_mask, false, - pgdat, &nr_scanned); - *total_scanned += nr_scanned; - if (!soft_limit_excess(root_memcg)) - break; - } - mem_cgroup_iter_break(root_memcg, victim); - return total; -} - -unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, - gfp_t gfp_mask, - unsigned long *total_scanned) -{ - unsigned long nr_reclaimed = 0; - struct mem_cgroup_per_node *mz, *next_mz = NULL; - unsigned long reclaimed; - int loop = 0; - struct mem_cgroup_tree_per_node *mctz; - unsigned long excess; - - if (lru_gen_enabled()) - return 0; - - if (order > 0) - return 0; - - mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id]; - - /* - * Do not even bother to check the largest node if the root - * is empty. Do it lockless to prevent lock bouncing. Races - * are acceptable as soft limit is best effort anyway. - */ - if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root)) - return 0; - - /* - * This loop can run a while, specially if mem_cgroup's continuously - * keep exceeding their soft limit and putting the system under - * pressure - */ - do { - if (next_mz) - mz = next_mz; - else - mz = mem_cgroup_largest_soft_limit_node(mctz); - if (!mz) - break; - - reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, - gfp_mask, total_scanned); - nr_reclaimed += reclaimed; - spin_lock_irq(&mctz->lock); - - /* - * If we failed to reclaim anything from this memory cgroup - * it is time to move on to the next cgroup - */ - next_mz = NULL; - if (!reclaimed) - next_mz = __mem_cgroup_largest_soft_limit_node(mctz); - - excess = soft_limit_excess(mz->memcg); - /* - * One school of thought says that we should not add - * back the node to the tree if reclaim returns 0. - * But our reclaim could return 0, simply because due - * to priority we are exposing a smaller subset of - * memory to reclaim from. Consider this as a longer - * term TODO. - */ - /* If excess == 0, no tree ops */ - __mem_cgroup_insert_exceeded(mz, mctz, excess); - spin_unlock_irq(&mctz->lock); - css_put(&mz->memcg->css); - loop++; - /* - * Could not reclaim anything and there are no more - * mem cgroups to try or we seem to be looping without - * reclaiming anything. - */ - if (!nr_reclaimed && - (next_mz == NULL || - loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) - break; - } while (!nr_reclaimed); - if (next_mz) - css_put(&next_mz->memcg->css); - return nr_reclaimed; -} - static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, struct cftype *cft) { diff --git a/mm/vmscan.c b/mm/vmscan.c index fdd13299a04a..0e04eaf64af3 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -6439,8 +6439,6 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) { struct zoneref *z; struct zone *zone; - unsigned long nr_soft_reclaimed; - unsigned long nr_soft_scanned; gfp_t orig_mask; pg_data_t *last_pgdat = NULL; pg_data_t *first_pgdat = NULL; @@ -6482,35 +6480,17 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) sc->compaction_ready = true; continue; } - - /* - * Shrink each node in the zonelist once. If the - * zonelist is ordered by zone (not the default) then a - * node may be shrunk multiple times but in that case - * the user prefers lower zones being preserved. - */ - if (zone->zone_pgdat == last_pgdat) - continue; - - /* - * This steals pages from memory cgroups over softlimit - * and returns the number of reclaimed pages and - * scanned pages. This works for global memory pressure - * and balancing, not for a memcg's limit. - */ - nr_soft_scanned = 0; - nr_soft_reclaimed = memcg1_soft_limit_reclaim(zone->zone_pgdat, - sc->order, sc->gfp_mask, - &nr_soft_scanned); - sc->nr_reclaimed += nr_soft_reclaimed; - sc->nr_scanned += nr_soft_scanned; - /* need some check for avoid more shrink_zone() */ } if (!first_pgdat) first_pgdat = zone->zone_pgdat; - /* See comment about same check for global reclaim above */ + /* + * Shrink each node in the zonelist once. If the zonelist is + * ordered by zone (not the default) then a node may be shrunk + * multiple times but in that case the user prefers lower zones + * being preserved. + */ if (zone->zone_pgdat == last_pgdat) continue; last_pgdat = zone->zone_pgdat; @@ -7171,8 +7151,6 @@ clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) { int i; - unsigned long nr_soft_reclaimed; - unsigned long nr_soft_scanned; unsigned long pflags; unsigned long nr_boost_reclaim; unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; @@ -7278,12 +7256,7 @@ restart: */ kswapd_age_node(pgdat, &sc); - /* Call soft limit reclaim before calling shrink_node. */ sc.nr_scanned = 0; - nr_soft_scanned = 0; - nr_soft_reclaimed = memcg1_soft_limit_reclaim(pgdat, sc.order, - sc.gfp_mask, &nr_soft_scanned); - sc.nr_reclaimed += nr_soft_reclaimed; /* * There should be no need to raise the scanning priority if From af88378942d37f10a1d6605f52d8e234130c9722 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 2 Sep 2026 10:43:05 -0700 Subject: [PATCH 067/539] memcg: remove mem_cgroup_shrink_node() Its only caller was soft limit reclaim, which is gone. Link: https://lore.kernel.org/20260902174311.1772372-3-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Muchun Song Cc: Roman Gushchin Cc: T.J. Mercier --- mm/internal.h | 4 ---- mm/vmscan.c | 41 ----------------------------------------- 2 files changed, 45 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 5cc220db9076..e16f1250b25c 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -78,10 +78,6 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, gfp_t gfp_mask, unsigned int reclaim_options, int *swappiness); -unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned); #ifdef CONFIG_NUMA extern int sysctl_min_unmapped_ratio; diff --git a/mm/vmscan.c b/mm/vmscan.c index 0e04eaf64af3..d66b5cd167d6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -6805,47 +6805,6 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, #ifdef CONFIG_MEMCG -/* Only used by soft limit reclaim. Do not reuse for anything else. */ -unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned) -{ - struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); - struct scan_control sc = { - .nr_to_reclaim = SWAP_CLUSTER_MAX, - .target_mem_cgroup = memcg, - .may_writepage = 1, - .may_unmap = 1, - .reclaim_idx = MAX_NR_ZONES - 1, - .may_swap = !noswap, - }; - - WARN_ON_ONCE(!current->reclaim_state); - - sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | - (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); - - trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.gfp_mask, - sc.order, - memcg); - - /* - * NOTE: Although we can get the priority field, using it - * here is not a good idea, since it limits the pages we can scan. - * if we don't reclaim here, the shrink_node from balance_pgdat - * will pick up pages from other mem cgroup's as well. We hack - * the priority and make it zero. - */ - shrink_lruvec(lruvec, &sc); - - trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed, memcg); - - *nr_scanned = sc.nr_scanned; - - return sc.nr_reclaimed; -} - unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages, gfp_t gfp_mask, From c4e11c58c31647909f78bcc950bdd4ab475cb9d7 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 2 Sep 2026 10:43:06 -0700 Subject: [PATCH 068/539] memcg: remove the soft limit reclaim tracepoints mm_vmscan_memcg_softlimit_reclaim_begin and mm_vmscan_memcg_softlimit_reclaim_end were only emitted by mem_cgroup_shrink_node(), which is gone, so they can never fire again. Link: https://lore.kernel.org/20260902174311.1772372-4-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Muchun Song Cc: Roman Gushchin Cc: T.J. Mercier --- include/trace/events/vmscan.h | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h index b4bf7b8def1f..8a872990b4be 100644 --- a/include/trace/events/vmscan.h +++ b/include/trace/events/vmscan.h @@ -214,13 +214,6 @@ DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_reclaim_be TP_ARGS(gfp_flags, order, memcg) ); - -DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_softlimit_reclaim_begin, - - TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg), - - TP_ARGS(gfp_flags, order, memcg) -); #endif /* CONFIG_MEMCG */ DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_end_template, @@ -260,13 +253,6 @@ DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_memcg_reclaim_end, TP_ARGS(nr_reclaimed, memcg) ); - -DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_memcg_softlimit_reclaim_end, - - TP_PROTO(unsigned long nr_reclaimed, struct mem_cgroup *memcg), - - TP_ARGS(nr_reclaimed, memcg) -); #endif /* CONFIG_MEMCG */ TRACE_EVENT(mm_shrink_slab_start, From 806ec7ae198d7b2e5b80369a2519ff576af25764 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 2 Sep 2026 10:43:07 -0700 Subject: [PATCH 069/539] memcg: remove the soft limit rbtree With soft limit reclaim gone, the per-node rbtree of cgroups in excess has no readers left. Remove the tree, the helpers maintaining it, and the subsys_initcall that existed only to allocate it. memcg1_check_events() no longer needs to feed it, which also drops the last caller of lru_gen_soft_reclaim(). Link: https://lore.kernel.org/20260902174311.1772372-5-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Muchun Song Cc: Roman Gushchin Cc: T.J. Mercier --- mm/memcontrol-v1.c | 176 +-------------------------------------------- mm/memcontrol-v1.h | 2 - mm/memcontrol.c | 1 - 3 files changed, 2 insertions(+), 177 deletions(-) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index b38b8d0f7f51..475f998b7643 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -17,23 +17,6 @@ #include "swap_table.h" #include "memcontrol-v1.h" -/* - * Cgroups above their limits are maintained in a RB-Tree, independent of - * their hierarchy representation - */ - -struct mem_cgroup_tree_per_node { - struct rb_root rb_root; - struct rb_node *rb_rightmost; - spinlock_t lock; -}; - -struct mem_cgroup_tree { - struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; -}; - -static struct mem_cgroup_tree soft_limit_tree __read_mostly; - /* for OOM */ struct mem_cgroup_eventfd_list { struct list_head list; @@ -99,133 +82,6 @@ static struct lockdep_map memcg_oom_lock_dep_map = { DEFINE_SPINLOCK(memcg_oom_lock); -static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz, - struct mem_cgroup_tree_per_node *mctz, - unsigned long new_usage_in_excess) -{ - struct rb_node **p = &mctz->rb_root.rb_node; - struct rb_node *parent = NULL; - struct mem_cgroup_per_node *mz_node; - bool rightmost = true; - - if (mz->on_tree) - return; - - mz->usage_in_excess = new_usage_in_excess; - if (!mz->usage_in_excess) - return; - while (*p) { - parent = *p; - mz_node = rb_entry(parent, struct mem_cgroup_per_node, - tree_node); - if (mz->usage_in_excess < mz_node->usage_in_excess) { - p = &(*p)->rb_left; - rightmost = false; - } else { - p = &(*p)->rb_right; - } - } - - if (rightmost) - mctz->rb_rightmost = &mz->tree_node; - - rb_link_node(&mz->tree_node, parent, p); - rb_insert_color(&mz->tree_node, &mctz->rb_root); - mz->on_tree = true; -} - -static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, - struct mem_cgroup_tree_per_node *mctz) -{ - if (!mz->on_tree) - return; - - if (&mz->tree_node == mctz->rb_rightmost) - mctz->rb_rightmost = rb_prev(&mz->tree_node); - - rb_erase(&mz->tree_node, &mctz->rb_root); - mz->on_tree = false; -} - -static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, - struct mem_cgroup_tree_per_node *mctz) -{ - unsigned long flags; - - spin_lock_irqsave(&mctz->lock, flags); - __mem_cgroup_remove_exceeded(mz, mctz); - spin_unlock_irqrestore(&mctz->lock, flags); -} - -static unsigned long soft_limit_excess(struct mem_cgroup *memcg) -{ - unsigned long nr_pages = page_counter_read(&memcg->memory); - unsigned long soft_limit = READ_ONCE(memcg->soft_limit); - unsigned long excess = 0; - - if (nr_pages > soft_limit) - excess = nr_pages - soft_limit; - - return excess; -} - -static void memcg1_update_tree(struct mem_cgroup *memcg, int nid) -{ - unsigned long excess; - struct mem_cgroup_per_node *mz; - struct mem_cgroup_tree_per_node *mctz; - - if (lru_gen_enabled()) { - if (soft_limit_excess(memcg)) - lru_gen_soft_reclaim(memcg, nid); - return; - } - - mctz = soft_limit_tree.rb_tree_per_node[nid]; - if (!mctz) - return; - /* - * Necessary to update all ancestors when hierarchy is used. - * because their event counter is not touched. - */ - for (; memcg; memcg = parent_mem_cgroup(memcg)) { - mz = memcg->nodeinfo[nid]; - excess = soft_limit_excess(memcg); - /* - * We have to update the tree if mz is on RB-tree or - * mem is over its softlimit. - */ - if (excess || mz->on_tree) { - unsigned long flags; - - spin_lock_irqsave(&mctz->lock, flags); - /* if on-tree, remove it */ - if (mz->on_tree) - __mem_cgroup_remove_exceeded(mz, mctz); - /* - * Insert again. mz->usage_in_excess will be updated. - * If excess is 0, no tree ops. - */ - __mem_cgroup_insert_exceeded(mz, mctz, excess); - spin_unlock_irqrestore(&mctz->lock, flags); - } - } -} - -void memcg1_remove_from_trees(struct mem_cgroup *memcg) -{ - struct mem_cgroup_tree_per_node *mctz; - struct mem_cgroup_per_node *mz; - int nid; - - for_each_node(nid) { - mz = memcg->nodeinfo[nid]; - mctz = soft_limit_tree.rb_tree_per_node[nid]; - if (mctz) - mem_cgroup_remove_exceeded(mz, mctz); - } -} - static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, struct cftype *cft) { @@ -336,7 +192,7 @@ static void mem_cgroup_threshold(struct mem_cgroup *memcg) } } -/* Cgroup1: threshold notifications & softlimit tree updates */ +/* Cgroup1: threshold notifications */ /* * Per memcg event counter is incremented at every pagein/pageout. With THP, @@ -405,17 +261,8 @@ static void memcg1_check_events(struct mem_cgroup *memcg, int nid) if (IS_ENABLED(CONFIG_PREEMPT_RT)) return; - /* threshold event is triggered in finer grain than soft limit */ - if (unlikely(memcg1_event_ratelimit(memcg, - MEM_CGROUP_TARGET_THRESH))) { - bool do_softlimit; - - do_softlimit = memcg1_event_ratelimit(memcg, - MEM_CGROUP_TARGET_SOFTLIMIT); + if (unlikely(memcg1_event_ratelimit(memcg, MEM_CGROUP_TARGET_THRESH))) mem_cgroup_threshold(memcg); - if (unlikely(do_softlimit)) - memcg1_update_tree(memcg, nid); - } } void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg) @@ -2391,22 +2238,3 @@ void memcg1_free_events(struct mem_cgroup *memcg) { free_percpu(memcg->events_percpu); } - -static int __init memcg1_init(void) -{ - int node; - - for_each_node(node) { - struct mem_cgroup_tree_per_node *rtpn; - - rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, node); - - rtpn->rb_root = RB_ROOT; - rtpn->rb_rightmost = NULL; - spin_lock_init(&rtpn->lock); - soft_limit_tree.rb_tree_per_node[node] = rtpn; - } - - return 0; -} -subsys_initcall(memcg1_init); diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index 1e394269c613..fd611e66859a 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -41,7 +41,6 @@ bool memcg1_alloc_events(struct mem_cgroup *memcg); void memcg1_free_events(struct mem_cgroup *memcg); void memcg1_memcg_init(struct mem_cgroup *memcg); -void memcg1_remove_from_trees(struct mem_cgroup *memcg); static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) { @@ -98,7 +97,6 @@ static inline bool memcg1_alloc_events(struct mem_cgroup *memcg) { return true; static inline void memcg1_free_events(struct mem_cgroup *memcg) {} static inline void memcg1_memcg_init(struct mem_cgroup *memcg) {} -static inline void memcg1_remove_from_trees(struct mem_cgroup *memcg) {} static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) {} static inline void memcg1_css_offline(struct mem_cgroup *memcg) {} diff --git a/mm/memcontrol.c b/mm/memcontrol.c index bfd0a74fac92..29def0376819 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4429,7 +4429,6 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css) vmpressure_cleanup(&memcg->vmpressure); cancel_work_sync(&memcg->high_work); - memcg1_remove_from_trees(memcg); free_shrinker_info(memcg); mem_cgroup_free(memcg); } From d5d1712c5928b46e848857037d89541fdd1e32a0 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 2 Sep 2026 10:43:08 -0700 Subject: [PATCH 070/539] memcg: remove lru_gen_soft_reclaim() The soft limit rbtree was the only caller. Dropping it leaves MEMCG_LRU_HEAD unreachable, since nothing else ever rotates a memcg with that op, so remove the op too and update the memcg LRU comment. Link: https://lore.kernel.org/20260902174311.1772372-6-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Reviewed-by: T.J. Mercier Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/mmzone.h | 30 +++++++++++------------------- mm/vmscan.c | 16 ++-------------- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 6acc14b169bb..c070b867e2f3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -638,35 +638,32 @@ struct lru_gen_mm_walk { * For each node, memcgs are divided into two generations: the old and the * young. For each generation, memcgs are randomly sharded into multiple bins * to improve scalability. For each bin, the hlist_nulls is virtually divided - * into three segments: the head, the tail and the default. + * into two segments: the tail and the default. * * An onlining memcg is added to the tail of a random bin in the old generation. * The eviction starts at the head of a random bin in the old generation. The * per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes * the old generation, is incremented when all its bins become empty. * - * There are four operations: - * 1. MEMCG_LRU_HEAD, which moves a memcg to the head of a random bin in its - * current generation (old or young) and updates its "seg" to "head"; - * 2. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its + * There are three operations: + * 1. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its * current generation (old or young) and updates its "seg" to "tail"; - * 3. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old + * 2. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old * generation, updates its "gen" to "old" and resets its "seg" to "default"; - * 4. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the + * 3. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the * young generation, updates its "gen" to "young" and resets its "seg" to * "default". * * The events that trigger the above operations are: - * 1. Exceeding the soft limit, which triggers MEMCG_LRU_HEAD; - * 2. The first attempt to reclaim a memcg below low, which triggers + * 1. The first attempt to reclaim a memcg below low, which triggers * MEMCG_LRU_TAIL; - * 3. The first attempt to reclaim a memcg offlined or below reclaimable size + * 2. The first attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_TAIL; - * 4. The second attempt to reclaim a memcg offlined or below reclaimable size + * 3. The second attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_YOUNG; - * 5. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG; - * 6. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG; - * 7. Offlining a memcg, which triggers MEMCG_LRU_OLD. + * 4. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG; + * 5. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG; + * 6. Offlining a memcg, which triggers MEMCG_LRU_OLD. * * Notes: * 1. Memcg LRU only applies to global reclaim, and the round-robin incrementing @@ -699,7 +696,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg); void lru_gen_online_memcg(struct mem_cgroup *memcg); void lru_gen_offline_memcg(struct mem_cgroup *memcg); void lru_gen_release_memcg(struct mem_cgroup *memcg); -void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid); void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid); bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid); void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); @@ -740,10 +736,6 @@ static inline void lru_gen_release_memcg(struct mem_cgroup *memcg) { } -static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) -{ -} - static inline void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid) { } diff --git a/mm/vmscan.c b/mm/vmscan.c index d66b5cd167d6..deb087c57007 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4373,7 +4373,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* see the comment on MEMCG_NR_GENS */ enum { MEMCG_LRU_NOP, - MEMCG_LRU_HEAD, MEMCG_LRU_TAIL, MEMCG_LRU_OLD, MEMCG_LRU_YOUNG, @@ -4395,9 +4394,7 @@ static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) new = old = lruvec->lrugen.gen; /* see the comment on MEMCG_NR_GENS */ - if (op == MEMCG_LRU_HEAD) - seg = MEMCG_LRU_HEAD; - else if (op == MEMCG_LRU_TAIL) + if (op == MEMCG_LRU_TAIL) seg = MEMCG_LRU_TAIL; else if (op == MEMCG_LRU_OLD) new = get_memcg_gen(pgdat->memcg_lru.seq); @@ -4411,7 +4408,7 @@ static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) hlist_nulls_del_rcu(&lruvec->lrugen.list); - if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD) + if (op == MEMCG_LRU_OLD) hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); else hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); @@ -4489,15 +4486,6 @@ unlock: } } -void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) -{ - struct lruvec *lruvec = get_lruvec(memcg, nid); - - /* see the comment on MEMCG_NR_GENS */ - if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD) - lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD); -} - bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid) { struct lruvec *lruvec = get_lruvec(memcg, nid); From fa148fc5e18e6d2b7197113ff414e68d53ea338c Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 2 Sep 2026 10:43:09 -0700 Subject: [PATCH 071/539] memcg: remove the per-node soft limit tree fields tree_node, usage_in_excess and on_tree only existed for the soft limit rbtree. They also doubled as the buffer between the read-mostly head of struct mem_cgroup_per_node and its update-often tail, so replace them with the explicit padding that CONFIG_MEMCG_V1=n already used. Link: https://lore.kernel.org/20260902174311.1772372-7-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Muchun Song Cc: Roman Gushchin Cc: T.J. Mercier --- include/linux/memcontrol.h | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 11c1fa88d6fd..1eababed16f5 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -95,20 +95,7 @@ struct mem_cgroup_per_node { struct lruvec_stats *lruvec_stats; struct shrinker_info __rcu *shrinker_info; -#ifdef CONFIG_MEMCG_V1 - /* - * Memcg-v1 only stuff in middle as buffer between read mostly fields - * and update often fields to avoid false sharing. If v1 stuff is - * not present, an explicit padding is needed. - */ - - struct rb_node tree_node; /* RB tree node */ - unsigned long usage_in_excess;/* Set to the value by which */ - /* the soft limit is exceeded*/ - bool on_tree; -#else CACHELINE_PADDING(_pad1_); -#endif /* Fields which get updated often at the end. */ struct lruvec lruvec; From 06d5d1223b6fbeff9d5ce95c89b7de720f0ee213 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 2 Sep 2026 10:43:10 -0700 Subject: [PATCH 072/539] memcg: remove mem_cgroup->soft_limit Nothing reads it anymore, so the field and the helper that reset it on css alloc and css reset can go. Link: https://lore.kernel.org/20260902174311.1772372-8-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Muchun Song Cc: Roman Gushchin Cc: T.J. Mercier --- include/linux/memcontrol.h | 2 -- mm/memcontrol-v1.h | 6 ------ mm/memcontrol.c | 2 -- 3 files changed, 10 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 1eababed16f5..c79992643556 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -280,8 +280,6 @@ struct mem_cgroup { struct memcg1_events_percpu __percpu *events_percpu; - unsigned long soft_limit; - /* protected by memcg_oom_lock */ bool oom_lock; int under_oom; diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index fd611e66859a..f48d0e22e615 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -42,11 +42,6 @@ void memcg1_free_events(struct mem_cgroup *memcg); void memcg1_memcg_init(struct mem_cgroup *memcg); -static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) -{ - WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX); -} - struct cgroup_taskset; void memcg1_css_offline(struct mem_cgroup *memcg); @@ -97,7 +92,6 @@ static inline bool memcg1_alloc_events(struct mem_cgroup *memcg) { return true; static inline void memcg1_free_events(struct mem_cgroup *memcg) {} static inline void memcg1_memcg_init(struct mem_cgroup *memcg) {} -static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) {} static inline void memcg1_css_offline(struct mem_cgroup *memcg) {} static inline bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 29def0376819..bce3962dba57 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4257,7 +4257,6 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) return ERR_CAST(memcg); page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); - memcg1_soft_limit_reset(memcg); #ifdef CONFIG_ZSWAP memcg->zswap_max = PAGE_COUNTER_MAX; WRITE_ONCE(memcg->zswap_writeback, true); @@ -4464,7 +4463,6 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) page_counter_set_min(&memcg->memory, 0); page_counter_set_low(&memcg->memory, 0); page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); - memcg1_soft_limit_reset(memcg); page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); memcg_wb_domain_size_changed(memcg); } From 4ddf67d73ea3365c52dee9004a917860ea155680 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 2 Sep 2026 10:43:11 -0700 Subject: [PATCH 073/539] memcg: simplify v1 event ratelimiting Thresholds are the only periodic v1 event left, so the target enum, the per-cpu target array and the switch in memcg1_event_ratelimit() all collapse to a single counter. memcg1_check_events() no longer needs a node id either, which lets memcg1_uncharge_batch() drop its nid argument and struct uncharge_gather drop the field feeding it. Link: https://lore.kernel.org/20260902174311.1772372-9-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Muchun Song Cc: Roman Gushchin Cc: T.J. Mercier --- mm/memcontrol-v1.c | 43 +++++++++++-------------------------------- mm/memcontrol-v1.h | 4 ++-- mm/memcontrol.c | 4 +--- 3 files changed, 14 insertions(+), 37 deletions(-) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 475f998b7643..bf2c7d53b01b 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -200,15 +200,9 @@ static void mem_cgroup_threshold(struct mem_cgroup *memcg) * to trigger some periodic events. This is straightforward and better * than using jiffies etc. to handle periodic memcg event. */ -enum mem_cgroup_events_target { - MEM_CGROUP_TARGET_THRESH, - MEM_CGROUP_TARGET_SOFTLIMIT, - MEM_CGROUP_NTARGETS, -}; - struct memcg1_events_percpu { unsigned long nr_page_events; - unsigned long targets[MEM_CGROUP_NTARGETS]; + unsigned long threshold_target; }; static void memcg1_charge_statistics(struct mem_cgroup *memcg, int nr_pages) @@ -225,43 +219,28 @@ static void memcg1_charge_statistics(struct mem_cgroup *memcg, int nr_pages) } #define THRESHOLDS_EVENTS_TARGET 128 -#define SOFTLIMIT_EVENTS_TARGET 1024 -static bool memcg1_event_ratelimit(struct mem_cgroup *memcg, - enum mem_cgroup_events_target target) +static bool memcg1_event_ratelimit(struct mem_cgroup *memcg) { unsigned long val, next; val = __this_cpu_read(memcg->events_percpu->nr_page_events); - next = __this_cpu_read(memcg->events_percpu->targets[target]); + next = __this_cpu_read(memcg->events_percpu->threshold_target); /* from time_after() in jiffies.h */ if ((long)(next - val) < 0) { - switch (target) { - case MEM_CGROUP_TARGET_THRESH: - next = val + THRESHOLDS_EVENTS_TARGET; - break; - case MEM_CGROUP_TARGET_SOFTLIMIT: - next = val + SOFTLIMIT_EVENTS_TARGET; - break; - default: - break; - } - __this_cpu_write(memcg->events_percpu->targets[target], next); + __this_cpu_write(memcg->events_percpu->threshold_target, + val + THRESHOLDS_EVENTS_TARGET); return true; } return false; } -/* - * Check events in order. - * - */ -static void memcg1_check_events(struct mem_cgroup *memcg, int nid) +static void memcg1_check_events(struct mem_cgroup *memcg) { if (IS_ENABLED(CONFIG_PREEMPT_RT)) return; - if (unlikely(memcg1_event_ratelimit(memcg, MEM_CGROUP_TARGET_THRESH))) + if (unlikely(memcg1_event_ratelimit(memcg))) mem_cgroup_threshold(memcg); } @@ -271,7 +250,7 @@ void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg) local_irq_save(flags); memcg1_charge_statistics(memcg, folio_nr_pages(folio)); - memcg1_check_events(memcg, folio_nid(folio)); + memcg1_check_events(memcg); local_irq_restore(flags); } @@ -344,7 +323,7 @@ void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci) VM_WARN_ON_IRQS_ENABLED(); memcg1_charge_statistics(memcg, -folio_nr_pages(folio)); preempt_enable_nested(); - memcg1_check_events(memcg, folio_nid(folio)); + memcg1_check_events(memcg); rcu_read_unlock(); obj_cgroup_put(objcg); @@ -398,14 +377,14 @@ void memcg1_swapin(struct folio *folio) #endif void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, - unsigned long nr_memory, int nid) + unsigned long nr_memory) { unsigned long flags; local_irq_save(flags); count_memcg_events(memcg, PGPGOUT, pgpgout); __this_cpu_add(memcg->events_percpu->nr_page_events, nr_memory); - memcg1_check_events(memcg, nid); + memcg1_check_events(memcg); local_irq_restore(flags); } diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index f48d0e22e615..b9a21f0fd2c3 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -59,7 +59,7 @@ void memcg1_oom_recover(struct mem_cgroup *memcg); void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg); void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, - unsigned long nr_memory, int nid); + unsigned long nr_memory); void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s); void reparent_memcg1_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent); @@ -107,7 +107,7 @@ static inline void memcg1_commit_charge(struct folio *folio, static inline void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, - unsigned long nr_memory, int nid) {} + unsigned long nr_memory) {} static inline void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) {} diff --git a/mm/memcontrol.c b/mm/memcontrol.c index bce3962dba57..30636b9d9673 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5328,7 +5328,6 @@ struct uncharge_gather { unsigned long nr_memory; unsigned long pgpgout; unsigned long nr_kmem; - int nid; }; static inline void uncharge_gather_clear(struct uncharge_gather *ug) @@ -5351,7 +5350,7 @@ static void uncharge_batch(const struct uncharge_gather *ug) memcg1_oom_recover(memcg); } - memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory, ug->nid); + memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory); rcu_read_unlock(); /* drop reference from uncharge_folio */ @@ -5380,7 +5379,6 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug) uncharge_gather_clear(ug); } ug->objcg = objcg; - ug->nid = folio_nid(folio); /* pairs with obj_cgroup_put in uncharge_batch */ obj_cgroup_get(objcg); From 0361b8f820c135639b9213ff18492e7d00045613 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Sat, 29 Aug 2026 15:09:00 -0400 Subject: [PATCH 074/539] mm/page_io: convert write completion handlers to folios Patch series "mm/page_io: folio conversion cleanups", v2. Convert the remaining struct page usage in mm/page_io.c to folios. This removes one of the last callers of end_page_writeback(), along with the last caller of ClearPageReclaim(). This allows us to remove the PG_reclaim page accessors entirely. Clean up a few other stale references to pages throughout while at it. The rename of mm/page_io.c to mm/swap_io.c is deferred to a separate series. This patch (of 6): Convert swap_write_end() and swap_fs_write_complete() to operate on folios directly instead of going through the folio-compat page APIs. This removes calls to end_page_writeback() and set_page_dirty(), and the last caller of ClearPageReclaim(), saving two calls to compound_head() per folio on the write error path. Link: https://lore.kernel.org/20260829-b4-page_io-folios-v2-0-649728091117@columbia.edu Link: https://lore.kernel.org/20260829-b4-page_io-folios-v2-1-649728091117@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Lorenzo Stoakes (ARM) Cc: Baoquan He Cc: Barry Song Cc: Chengming Zhou Cc: Chris Li Cc: Christoph Hellwig Cc: David Hildenbrand Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/page_io.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index 88962571cb93..fbcf58ff292d 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -497,13 +497,13 @@ static void swap_write_end(struct swap_iocb *sio, bool failed) int p; for (p = 0; p < sio->nr_bvecs; p++) { - struct page *page = sio->bvecs[p].bv_page; + struct folio *folio = bvec_folio(&sio->bvecs[p]); if (failed) { - set_page_dirty(page); - ClearPageReclaim(page); + folio_mark_dirty(folio); + folio_clear_reclaim(folio); } - end_page_writeback(page); + folio_end_writeback(folio); } mempool_free(sio, sio_pool); } @@ -514,16 +514,16 @@ static void swap_fs_write_complete(struct kiocb *iocb, long ret) bool failed = ret != sio->len; if (failed) { - struct page *page = sio->bvecs[0].bv_page; + struct folio *folio = bvec_folio(&sio->bvecs[0]); /* * In the case of swap-over-nfs, this can be a temporary failure * if the system has limited memory for allocating transmit - * buffers. Mark the page dirty and avoid + * buffers. Mark the folio dirty and avoid * folio_rotate_reclaimable but rate-limit the messages. */ pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", - ret, swap_dev_pos(page_swap_entry(page))); + ret, swap_dev_pos(folio->swap)); } swap_write_end(sio, failed); From 50a8b41589fb21613ed51587eb92149da199beeb Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Sat, 29 Aug 2026 15:09:01 -0400 Subject: [PATCH 075/539] mm: remove PageReclaim This flag is now only used on folios, so we can remove all the page accessors. folio_test_clear_reclaim() is not used, so don't add FOLIO_TEST_CLEAR_FLAG() for it. Link: https://lore.kernel.org/20260829-b4-page_io-folios-v2-2-649728091117@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) --- include/linux/page-flags.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7a863572adce..ae2ebaed6d4d 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -593,8 +593,7 @@ TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL) FOLIO_FLAG(mappedtodisk, FOLIO_HEAD_PAGE) /* PG_readahead is only used for reads; PG_reclaim is only for writes */ -PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL) - TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL) +FOLIO_FLAG(reclaim, FOLIO_HEAD_PAGE) FOLIO_FLAG(readahead, FOLIO_HEAD_PAGE) FOLIO_TEST_CLEAR_FLAG(readahead, FOLIO_HEAD_PAGE) From 43999735491ea1e10c4665d1036dc23ef0d7b1ba Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Sat, 29 Aug 2026 15:09:02 -0400 Subject: [PATCH 076/539] mm/page_io: use swap entries directly in zeromap helpers Increment swp_entry_t::val directly instead of recomputing each entry with page_swap_entry(). This removes the last struct page usage in page_io.c and saves one call to compound_head() per page. Link: https://lore.kernel.org/20260829-b4-page_io-folios-v2-3-649728091117@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Lorenzo Stoakes (ARM) --- mm/page_io.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index fbcf58ff292d..dd95cdb0cd7a 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -160,7 +160,7 @@ static void swap_zeromap_folio_set(struct folio *folio) struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio); int nr_pages = folio_nr_pages(folio); struct swap_cluster_info *ci; - swp_entry_t entry; + swp_entry_t entry = folio->swap; unsigned int i; VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); @@ -168,8 +168,8 @@ static void swap_zeromap_folio_set(struct folio *folio) ci = swap_cluster_get_and_lock(folio); for (i = 0; i < folio_nr_pages(folio); i++) { - entry = page_swap_entry(folio_page(folio, i)); __swap_table_set_zero(ci, swp_cluster_offset(entry)); + entry.val++; } swap_cluster_unlock(ci); @@ -183,7 +183,7 @@ static void swap_zeromap_folio_set(struct folio *folio) static void swap_zeromap_folio_clear(struct folio *folio) { struct swap_cluster_info *ci; - swp_entry_t entry; + swp_entry_t entry = folio->swap; unsigned int i; VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); @@ -191,8 +191,8 @@ static void swap_zeromap_folio_clear(struct folio *folio) ci = swap_cluster_get_and_lock(folio); for (i = 0; i < folio_nr_pages(folio); i++) { - entry = page_swap_entry(folio_page(folio, i)); __swap_table_clear_zero(ci, swp_cluster_offset(entry)); + entry.val++; } swap_cluster_unlock(ci); } From 48d54b07157c65c056df09ff7006c3bad9ce937e Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Sat, 29 Aug 2026 15:09:03 -0400 Subject: [PATCH 077/539] mm/page_io: rename bio_associate_blkg_from_page() This function takes a folio. Rename it to bio_associate_blkg_from_folio() accordingly. While at it, convert the macro in the !CONFIG_MEMCG || !CONFIG_BLK_CGROUP case to a function. Link: https://lore.kernel.org/20260829-b4-page_io-folios-v2-4-649728091117@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Lorenzo Stoakes (ARM) --- mm/page_io.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index dd95cdb0cd7a..295cc6ac6244 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -277,7 +277,7 @@ static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) return can_merge; } -static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) +static void bio_associate_blkg_from_folio(struct bio *bio, struct folio *folio) { struct cgroup_subsys_state *css; @@ -298,7 +298,9 @@ static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) { return true; } -#define bio_associate_blkg_from_page(bio, folio) do { } while (0) +static void bio_associate_blkg_from_folio(struct bio *bio, struct folio *folio) +{ +} #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ static mempool_t *sio_pool; @@ -596,7 +598,7 @@ static void swap_bdev_submit_write(struct swap_io_ctx *ctx) REQ_OP_WRITE | REQ_SWAP); bio->bi_iter.bi_size = sio->len; bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio)); - bio_associate_blkg_from_page(bio, bio_first_folio_all(bio)); + bio_associate_blkg_from_folio(bio, bio_first_folio_all(bio)); if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) { submit_bio_wait(bio); From 7be17d6d935c6b3e3b29fb6eef795b9eed2dca91 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Sat, 29 Aug 2026 15:09:04 -0400 Subject: [PATCH 078/539] mm/page_io: refer to folios in swap_writeout() comments swap_writeout() operates on folios, not pages. Update its comments accordingly. Link: https://lore.kernel.org/20260829-b4-page_io-folios-v2-5-649728091117@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Lorenzo Stoakes (ARM) --- mm/page_io.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index 295cc6ac6244..36466159cf19 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -209,7 +209,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) goto out_unlock; /* - * Arch code may have to preserve more data than just the page + * Arch code may have to preserve more data than just the folio * contents, e.g. memory tags. */ ret = arch_prepare_to_swap(folio); @@ -220,7 +220,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) /* * Use the swap table zero mark to avoid doing IO for zero-filled - * pages. The zero mark is protected by the cluster lock, which is + * folios. The zero mark is protected by the cluster lock, which is * acquired internally by swap_zeromap_folio_set/clear. */ if (is_folio_zero_filled(folio)) { From a6479602b783b6a2e6067afdb06159b21b4f9ad0 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Sat, 29 Aug 2026 15:09:05 -0400 Subject: [PATCH 079/539] mm/swap: rename __swap_writepage() to __swap_writeout() Commit 84798514db50 ("mm: Remove swap_writepage() and shmem_writepage()") renamed swap_writepage() to swap_writeout(). Rename __swap_writepage(), which operates on a folio, to match its caller. Update a stale reference to swap_writepage() in swapfile.c as well. Link: https://lore.kernel.org/20260829-b4-page_io-folios-v2-6-649728091117@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Lorenzo Stoakes (ARM) --- mm/page_io.c | 4 ++-- mm/swap.h | 2 +- mm/swapfile.c | 2 +- mm/zswap.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index 36466159cf19..1da4ff484f09 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -248,7 +248,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) } rcu_read_unlock(); - __swap_writepage(ctx, folio); + __swap_writeout(ctx, folio); return 0; out_unlock: folio_unlock(folio); @@ -369,7 +369,7 @@ static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw) } } -void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio) +void __swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) { VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); diff --git a/mm/swap.h b/mm/swap.h index fddba7a87500..0b5d507739bc 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -258,7 +258,7 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio); void swap_read_submit(struct swap_io_ctx *ctx); void swap_write_submit(struct swap_io_ctx *ctx); int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio); -void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio); +void __swap_writeout(struct swap_io_ctx *ctx, struct folio *folio); /* linux/mm/swap_state.c */ extern struct address_space swap_space __read_mostly; diff --git a/mm/swapfile.c b/mm/swapfile.c index 601979b97f95..3b2279a16d1c 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2928,7 +2928,7 @@ EXPORT_SYMBOL_GPL(add_swap_extent); /* * A `swap extent' is a simple thing which maps a contiguous range of pages * onto a contiguous range of disk blocks. A rbtree of swap extents is - * built at swapon time and is then used at swap_writepage/swap_read_folio + * built at swapon time and is then used at swap_writeout/swap_read_folio * time for locating where on disk a page belongs. * * If the swapfile is an S_ISBLK block device, a single extent is installed. diff --git a/mm/zswap.c b/mm/zswap.c index c1dc60926bad..fc869d60ef1b 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1037,7 +1037,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, folio_set_reclaim(folio); /* start writeback */ - __swap_writepage(&ctx, folio); + __swap_writeout(&ctx, folio); swap_write_submit(&ctx); out: From 715d7fbe42393658c42ceaea7f61b147895c9e3a Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Sat, 29 Aug 2026 15:42:03 +0800 Subject: [PATCH 080/539] mm/mglru: make type fallback logic explicit in isolate_folios() Patch series "mm/mglru: clean up isolate_folios for readability and clarity", v2. Right now, isolate_folios() is quite difficult to follow: 1. It uses for_each_evictable_type(i, swappiness) to iterate over the types, but 'i' is not actually used as the type within the loop body. 2. It retries the same type when folios were scanned but none could be isolated, but the retry is implemented in a rather subtle way that is difficult to understand. This patchset makes both behaviors explicit and much easier to follow. There are no functional changes for swappiness values from 1 to 200. There is a slight functional change for 0 and 201: with the existing code, there is no chance to retry for these values because for_each_evictable_type() only iterates once. After this patch, 0 and 201 have behavior that is more consistent with the 1-200 range. This patch (of 2): The for_each_evictable_type() loop in isolate_folios() is misleading: it does not actually iterate over each evictable type. Instead, get_type_to_scan() selects the type to scan, while the iterator `i` merely bounds the number of attempts. Make the fallback behavior explicit in the code and remove the opaque for_each_evictable_type(i, swappiness). Link: https://lore.kernel.org/20260829074204.45304-1-baohua@kernel.org Link: https://lore.kernel.org/20260829074204.45304-2-baohua@kernel.org Signed-off-by: Ridong Chen Co-developed-by: Barry Song (Xiaomi) Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Reviewed-by: Lian Wang Reviewed-by: Baoquan He Cc: Axel Rasmussen Cc: Baolin Wang Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index deb087c57007..1b40c63706f2 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4826,35 +4826,41 @@ static int get_type_to_scan(struct lruvec *lruvec, int swappiness) return positive_ctrl_err(&sp, &pv); } +static inline bool is_single_type_reclaim(int swappiness) +{ + return swappiness == MIN_SWAPPINESS || + swappiness == SWAPPINESS_ANON_ONLY; +} + static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec, struct scan_control *sc, int swappiness, struct list_head *list, int *isolated, int *isolate_type, int *isolate_scanned) { - int i; - int total_scanned = 0; + bool type_fallback_allowed = !is_single_type_reclaim(swappiness); int type = get_type_to_scan(lruvec, swappiness); + int total_scanned = 0, scanned, tier; - for_each_evictable_type(i, swappiness) { - int scanned; - int tier = get_tier_idx(lruvec, type); +retry: + tier = get_tier_idx(lruvec, type); + scanned = scan_folios(nr_to_scan, lruvec, sc, + type, tier, list, isolated); - scanned = scan_folios(nr_to_scan, lruvec, sc, - type, tier, list, isolated); + total_scanned += scanned; + if (*isolated) { + *isolate_type = type; + *isolate_scanned = scanned; + return total_scanned; + } - total_scanned += scanned; - if (*isolated) { - *isolate_type = type; - *isolate_scanned = scanned; - break; - } - /* - * If scanned > 0 and isolated == 0, avoid falling back to the - * other type, as this type remains sufficient. Falling back - * too readily can disrupt the positive_ctrl_err() bias. - */ - if (!scanned) - type = !type; + /* + * We are running out of the current reclaim type. Fall back to + * the other type if allowed. + */ + if (!scanned && type_fallback_allowed) { + type = !type; + type_fallback_allowed = false; + goto retry; } return total_scanned; From e006ab5a0f1f411202bc6a20061663b1954d79eb Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Sat, 29 Aug 2026 15:42:04 +0800 Subject: [PATCH 081/539] mm/mglru: make retry logic explicit in isolate_folios() The existing mainline code retries the same type once in a rather subtle way. `for_each_evictable_type()` may provide one more iteration, allowing the same type to be retried if we scanned some folios but failed to isolate any due to protections, promotions, or races. This patch makes the retry behavior explicit. Link: https://lore.kernel.org/20260829074204.45304-3-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Cc: Axel Rasmussen Cc: Baoquan He Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Ridong Chen Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mm/vmscan.c b/mm/vmscan.c index 1b40c63706f2..413efe44d1f6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4840,6 +4840,7 @@ static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec, bool type_fallback_allowed = !is_single_type_reclaim(swappiness); int type = get_type_to_scan(lruvec, swappiness); int total_scanned = 0, scanned, tier; + bool tried = false; retry: tier = get_tier_idx(lruvec, type); @@ -4859,9 +4860,18 @@ retry: */ if (!scanned && type_fallback_allowed) { type = !type; + tried = true; type_fallback_allowed = false; goto retry; } + /* + * We scanned some folios but failed to isolate any due to promotions, + * protections, or races. Retry once to avoid a larger loop. + */ + if (scanned && !tried) { + tried = true; + goto retry; + } return total_scanned; } From 6c54eb9543b60230692cdf0ed6264034aab069b6 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Thu, 3 Sep 2026 14:48:52 +0800 Subject: [PATCH 082/539] mm: revert slight behavior change for swappiness 1-200 Baoquan's review found that we unexpectedly introduced a slight behavior change for swappiness 1-200. We could now have a case like: 1. First scan -> `scanned != 0` 2. Second scan -> `scanned = 0` 3. Type fallback Step 3 was impossible before. Let's remove this possibility. Link: https://lore.kernel.org/20260903070500.76379-1-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Reported-by: Baoquan He Closes: https://lore.kernel.org/linux-mm/apfZQE1X6zGAsBb_@fedora/ Cc: Axel Rasmussen Cc: Baolin Wang Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Ridong Chen Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 413efe44d1f6..42fcdcd3d2e4 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4858,7 +4858,7 @@ retry: * We are running out of the current reclaim type. Fall back to * the other type if allowed. */ - if (!scanned && type_fallback_allowed) { + if (!scanned && !tried && type_fallback_allowed) { type = !type; tried = true; type_fallback_allowed = false; From 1cb3621c6e9a13da36907e65a25bc7b1f11fcbc4 Mon Sep 17 00:00:00 2001 From: Avi Weiss Date: Sat, 29 Aug 2026 20:11:12 +0300 Subject: [PATCH 083/539] mm/memory: simplify error handling in insert_pages() Patch series "mm/memory: improve insert_pages() error handling", v3. Improve insert_pages() error handling. The first patch simplifies error handling by initializing the error status to zero and assigning error codes at their respective failure sites. The second patch returns -ENOMEM when walk_to_pmd() fails. A NULL return from walk_to_pmd() indicates failure to allocate an upper page-table level, so -ENOMEM is more appropriate than -EFAULT and is consistent with the subsequent pte_alloc() failure. This patch (of 2): Initialize error return status to zero and then set it as needed at each point of failure. Assign -ENOMEM explicitly when pte_alloc() fails as the pte_alloc() macro returns a boolean. Link: https://lore.kernel.org/cover.1788022178.git.thnkslprpt@gmail.com Link: https://lore.kernel.org/dd3a672c858b38c7525541b19a919e120c4e5a0e.1788022178.git.thnkslprpt@gmail.com Signed-off-by: Avi Weiss Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/memory.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 9cbce5c90bff..2561dc6bdde6 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2565,20 +2565,22 @@ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, unsigned long curr_page_idx = 0; unsigned long remaining_pages_total = *num; unsigned long pages_to_write_in_pmd; - int ret; + int err = 0; more: - ret = -EFAULT; pmd = walk_to_pmd(mm, addr); - if (!pmd) + if (!pmd) { + err = -EFAULT; goto out; + } pages_to_write_in_pmd = min_t(unsigned long, remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); /* Allocate the PTE if necessary; takes PMD lock once only. */ - ret = -ENOMEM; - if (pte_alloc(mm, pmd)) + if (pte_alloc(mm, pmd)) { + err = -ENOMEM; goto out; + } while (pages_to_write_in_pmd) { int pte_idx = 0; @@ -2586,15 +2588,14 @@ more: start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); if (!start_pte) { - ret = -EFAULT; + err = -EFAULT; goto out; } for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { - int err = insert_page_in_batch_locked(vma, pte, - addr, pages[curr_page_idx], prot); + err = insert_page_in_batch_locked(vma, pte, addr, + pages[curr_page_idx], prot); if (unlikely(err)) { pte_unmap_unlock(start_pte, pte_lock); - ret = err; remaining_pages_total -= pte_idx; goto out; } @@ -2607,10 +2608,9 @@ more: } if (remaining_pages_total) goto more; - ret = 0; out: *num = remaining_pages_total; - return ret; + return err; } /** From 3830ec6477e234a869731aa53bbb06ee2959346f Mon Sep 17 00:00:00 2001 From: Avi Weiss Date: Sat, 29 Aug 2026 20:11:13 +0300 Subject: [PATCH 084/539] mm/memory: return -ENOMEM for page-table allocation failure in insert_pages() walk_to_pmd() returns NULL only when p4d_alloc(), pud_alloc(), or pmd_alloc() fails. These are page-table allocation failures, but insert_pages() currently reports them as -EFAULT. Return -ENOMEM instead, consistent with the subsequent pte_alloc() failure and with the single-page insert_page() path, which reports failure of the same page-table allocation chain as -ENOMEM. Address and range validation failures in vm_insert_pages() continue to return -EFAULT. Keep the later -EFAULT return for pte_offset_map_lock(), which is not an allocation failure. Link: https://lore.kernel.org/9d990c3ed43608e674d4b12a8c221a09fd200f49.1788022178.git.thnkslprpt@gmail.com Signed-off-by: Avi Weiss Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memory.c b/mm/memory.c index 2561dc6bdde6..27ffe1a99a08 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2569,7 +2569,7 @@ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, more: pmd = walk_to_pmd(mm, addr); if (!pmd) { - err = -EFAULT; + err = -ENOMEM; goto out; } From cbbed3c303f9ea63937ffaa1f6f023cd3dcd12a1 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Sat, 29 Aug 2026 02:58:47 +0000 Subject: [PATCH 085/539] mm: adjust out-dated document of __GFP_NOFAIL Commit ee040cbd6e48 ("mm/page_alloc: don't warn about large allocations with __GFP_NOFAIL") remove a warning on allocating large folio with __GFP_NOFAIL, which was placed there by commit 903edea6c53f ("mm: warn about illegal __GFP_NOFAIL usage in a more appropriate location and manner"). While in that commit, it also documented this behavior which is out-dated now. Adjust the document to align to current code, and adjust the comment while at it. Link: https://lore.kernel.org/20260829025847.26779-1-richard.weiyang@gmail.com Signed-off-by: Wei Yang Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: David Hildenbrand Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan --- include/linux/gfp_types.h | 2 +- mm/page_alloc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 190191411009..bfd4c43ed777 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -244,7 +244,7 @@ enum { * definitely preferable to use the flag rather than opencode endless * loop around allocator. * Allocating pages from the buddy with __GFP_NOFAIL and order > 1 is - * not supported. Please consider using kvmalloc() instead. + * discouraged. Please consider using kvmalloc() instead if possible. */ #define __GFP_IO ((__force gfp_t)___GFP_IO) #define __GFP_FS ((__force gfp_t)___GFP_FS) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ab385bc252cc..146f7e0a9462 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4804,7 +4804,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, if (unlikely(nofail)) { /* - * Also we don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM, + * We don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM, * otherwise, we may result in lockup. */ WARN_ON_ONCE(!can_direct_reclaim); From 0c92f74c4e280dae833d131d5263d18d7f021d05 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Thu, 17 Sep 2026 20:12:02 -0400 Subject: [PATCH 086/539] mm/mempolicy: use SRCU for the weighted interleave state Patch series "mm/mempolicy: stop copying state in the interleave paths", v2. The interleave node selectors and bulk allocators take copies of nodemasks and node weights (for weighted interleave) in the fault path. Both of these copies can be entirely eliminated. For node weights, use SRCU to pin the weights in place. This eliminates a copy and a kmalloc from the bulk allocator path. For nodemasks, we can operate directly on pol->nodes as long as we bounds check the walk. A concurrent rebind can shrink the mask, or tear the read of it so the mask appears empty. - The interleave node selectors fall back to numa_node_id() when that happens, which is what they already did when a copy came back empty. - The bulk allocator simply returns what it managed to allocate. The node count and weight totals are read separately from the nodemask walk that consumes them - creating a time-of-check / time-of-use race. Just clamp the walk to a single pass (number of nodes), and clamp each bulk allocation chunk to the space left in the request. The cost is distribution accuracy during a rebind. The copies never corrected for that either - they only kept the code from dividing by zero and overrunning the allocation request. This patch (of 2): alloc_pages_bulk_weighted_interleave() copies iw_table into a scratch array on every call so it can walk the weights outside of RCU. The copy exists only because the loop may sleep in the page allocator and so cannot hold rcu_read_lock(). Use SRCU to pin the global iw_table object and use it in-place instead. Retire through both flavors - call_srcu() for the sleeping readers, then kfree_rcu() for the reference-less ones - so writers no longer block on synchronize_rcu() either. Tested in a VM with KASAN, PROVE_LOCKING and DEBUG_OBJECTS_RCU_HEAD, with a udelay() injected into the read section to widen the race against concurrent sysfs weight writers, and placement checked against the configured weights. Every retired state reached its callback. Swapping the deferred free for a bare kfree() in the same test reports a use-after-free immediately. Link: https://lore.kernel.org/20260918001203.3389165-1-gourry@gourry.net Link: https://lore.kernel.org/20260918001203.3389165-2-gourry@gourry.net Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Suggested-by: Andrew Morton Suggested-by: Matthew Wilcox Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Alistair Popple Cc: Byungchul Park Cc: "Huang, Ying" Cc: Joshua Hahn Cc: Matthew Brost Cc: Rakie Kim Cc: Zi Yan --- mm/mempolicy.c | 70 ++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 060a0eb26917..2643915dc966 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -112,6 +112,7 @@ #include #include #include +#include #include #include @@ -157,6 +158,7 @@ static const int weightiness = 32; */ struct weighted_interleave_state { bool mode_auto; + struct rcu_head rcu; u8 iw_table[]; }; static struct weighted_interleave_state __rcu *wi_state; @@ -168,6 +170,24 @@ static unsigned int *node_bw_table; */ static DEFINE_MUTEX(wi_state_lock); +/* Readers that sleep while walking iw_table hold this instead */ +DEFINE_STATIC_SRCU_FAST(wi_srcu); + +static void wi_state_free_rcu(struct rcu_head *head) +{ + struct weighted_interleave_state *state = + container_of(head, struct weighted_interleave_state, rcu); + + kfree_rcu(state, rcu); +} + +/* Retire through both flavors: sleeping readers use SRCU, the rest RCU */ +static void wi_state_retire(struct weighted_interleave_state *state) +{ + if (state) + call_srcu(&wi_srcu, &state->rcu, wi_state_free_rcu); +} + static u8 get_il_weight(int node) { struct weighted_interleave_state *state; @@ -266,10 +286,7 @@ int mempolicy_set_node_perf(unsigned int node, struct access_coordinate *coords) rcu_assign_pointer(wi_state, new_wi_state); mutex_unlock(&wi_state_lock); - if (old_wi_state) { - synchronize_rcu(); - kfree(old_wi_state); - } + wi_state_retire(old_wi_state); out: kfree(old_bw); return 0; @@ -2644,7 +2661,8 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, unsigned long nr_allocated = 0; unsigned long rounds; unsigned long node_pages, delta; - u8 *weights, weight; + struct srcu_ctr __percpu *scp; + u8 *table, weight; unsigned int weight_total = 0; unsigned long rem_pages = nr_pages; nodemask_t nodes; @@ -2688,25 +2706,14 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, me->il_weight = 0; prev_node = node; - /* create a local copy of node weights to operate on outside rcu */ - weights = kmalloc(nr_node_ids, gfp & GFP_RECLAIM_MASK); - if (!weights) - return total_allocated; - - rcu_read_lock(); - state = rcu_dereference(wi_state); - if (state) { - memcpy(weights, state->iw_table, nr_node_ids * sizeof(u8)); - rcu_read_unlock(); - } else { - rcu_read_unlock(); - for (i = 0; i < nr_node_ids; i++) - weights[i] = 1; - } + /* The page allocator may sleep, pin the weight table with SRCU */ + scp = srcu_read_lock_fast(&wi_srcu); + state = srcu_dereference(wi_state, &wi_srcu); + table = state ? state->iw_table : NULL; /* calculate total, detect system default usage */ for_each_node_mask(node, nodes) - weight_total += weights[node]; + weight_total += table ? table[node] : 1; /* * Calculate rounds/partial rounds to minimize __alloc_pages_bulk calls. @@ -2718,10 +2725,10 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, rounds = rem_pages / weight_total; delta = rem_pages % weight_total; resume_node = next_node_in(prev_node, nodes); - resume_weight = weights[resume_node]; + resume_weight = table ? table[resume_node] : 1; for (i = 0; i < nnodes; i++) { node = next_node_in(prev_node, nodes); - weight = weights[node]; + weight = table ? table[node] : 1; node_pages = weight * rounds; /* If a delta exists, add this node's portion of the delta */ if (delta > weight) { @@ -2747,7 +2754,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, } me->il_prev = resume_node; me->il_weight = resume_weight; - kfree(weights); + srcu_read_unlock_fast(&wi_srcu, scp); return total_allocated; } @@ -3673,10 +3680,7 @@ static ssize_t node_store(struct kobject *kobj, struct kobj_attribute *attr, rcu_assign_pointer(wi_state, new_wi_state); mutex_unlock(&wi_state_lock); - if (old_wi_state) { - synchronize_rcu(); - kfree(old_wi_state); - } + wi_state_retire(old_wi_state); return count; } @@ -3742,10 +3746,7 @@ static ssize_t weighted_interleave_auto_store(struct kobject *kobj, update_wi_state: rcu_assign_pointer(wi_state, new_wi_state); mutex_unlock(&wi_state_lock); - if (old_wi_state) { - synchronize_rcu(); - kfree(old_wi_state); - } + wi_state_retire(old_wi_state); return count; } @@ -3789,10 +3790,7 @@ static void wi_state_free(void) rcu_assign_pointer(wi_state, NULL); mutex_unlock(&wi_state_lock); - if (old_wi_state) { - synchronize_rcu(); - kfree(old_wi_state); - } + wi_state_retire(old_wi_state); } static struct kobj_attribute wi_auto_attr = { From 3a5f742b44c390be80886b89aa2ae42a9c1f08ee Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Thu, 17 Sep 2026 20:12:03 -0400 Subject: [PATCH 087/539] mm/mempolicy: stop copying the nodemask in the interleave paths The interleave node selectors copy pol->nodes onto the stack so the mask cannot change while they walk it. nodemask_t is 128 bytes at MAX_NUMNODES=1024, and two of the three run per folio fault. The copy only buys consistency between the node count and the walk. Drop the consistency and just bounds check the walk instead. The nodelist access is racy by design, but safe as long as we handle the scenario where a cpuset rebind causes a torn nodemask read to perceive the nodemask as empty (weight_total == 0). If an empty nodelist or weight is perceived, fall back to numa_node_id(), which is what the functions already did when the copy came back empty Otherwise, iterating the nodelist during the actual allocation loop is perfectly safe - a concurrent rebind may simply cause a skew in in the distribution of memory (or fail and fall back the same as any other error condition). weighted_interleave_nid() counts the nodes as we sum the weights. We use that node count to limit the maximum skew a single node can host. interleave_nid() walks with next_node_in() rather than next_node(), so a mask that shrank mid-walk wraps to a node still in the policy. alloc_pages_bulk_weighted_interleave() derives per-node counts from a weight total summed over the mask, so a changing mask can make them exceed the request. Clamp each chunk to the space left in page_array. A cpuset cookie will not work here: two of these take VMA policies, which mpol_rebind_mm() rebinds under mmap_write_lock(), not mems_allowed_seq. Cost is distribution accuracy during a rebind - but the copy never corrected this anyway, it was just a safety mechanism to prevent div/0 and overrunning the alloc request buffer. Remove read_once_policy_nodemask(), now unused. -fstack-usage at MAX_NUMNODES=1024: weighted_interleave_nid 184 -> 56 interleave_nid 168 -> 32 alloc_pages_bulk_mempolicy_noprof 360 -> 136 Link: https://lore.kernel.org/20260918001203.3389165-3-gourry@gourry.net Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Rakie Kim Assisted-by: LLM Cc: Alistair Popple Cc: Byungchul Park Cc: "Huang, Ying" Cc: Joshua Hahn Cc: Matthew Brost Cc: Matthew Wilcox Cc: Zi Yan --- mm/mempolicy.c | 93 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 2643915dc966..fd97fb0289bc 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2197,34 +2197,15 @@ unsigned int mempolicy_slab_node(void) } } -static unsigned int read_once_policy_nodemask(struct mempolicy *pol, - nodemask_t *mask) -{ - /* - * barrier stabilizes the nodemask locally so that it can be iterated - * over safely without concern for changes. Allocators validate node - * selection does not violate mems_allowed, so this is safe. - */ - barrier(); - memcpy(mask, &pol->nodes, sizeof(nodemask_t)); - barrier(); - return nodes_weight(*mask); -} - static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx) { struct weighted_interleave_state *state; - nodemask_t nodemask; - unsigned int target, nr_nodes; + unsigned int target, nnodes = 0; u8 *table = NULL; unsigned int weight_total = 0; u8 weight; int nid = 0; - nr_nodes = read_once_policy_nodemask(pol, &nodemask); - if (!nr_nodes) - return numa_node_id(); - rcu_read_lock(); state = rcu_dereference(wi_state); @@ -2232,22 +2213,45 @@ static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx) if (state) table = state->iw_table; - /* calculate the total weight */ - for_each_node_mask(nid, nodemask) + /* calculate the total weight and the node count */ + for_each_node_mask(nid, pol->nodes) { weight_total += table ? table[nid] : 1; + nnodes++; + } + + /* the mask is empty */ + if (!weight_total) { + rcu_read_unlock(); + return numa_node_id(); + } /* Calculate the node offset based on totals */ target = ilx % weight_total; - nid = first_node(nodemask); - while (target) { + nid = first_node(pol->nodes); + + /* + * The target was calculated in a separate loop, and a concurrent + * rebind can change the contents of pol->nodes as we calculate. + * Access is safe, in the worst case we suddenly perceive an empty + * nodemask and return numa_node_id() below - otherwise we may + * simply cause a skew in allocations. + * + * Clamp this loop to a single pass (nnodes) to keep the walk + * bounded by node count. + */ + while (target && nnodes-- && nid < MAX_NUMNODES) { /* detect system default usage */ weight = table ? table[nid] : 1; if (target < weight) break; target -= weight; - nid = next_node_in(nid, nodemask); + nid = next_node_in(nid, pol->nodes); } rcu_read_unlock(); + + /* the mask emptied under the walk */ + if (nid >= MAX_NUMNODES) + return numa_node_id(); return nid; } @@ -2258,18 +2262,23 @@ static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx) */ static unsigned int interleave_nid(struct mempolicy *pol, pgoff_t ilx) { - nodemask_t nodemask; unsigned int target, nnodes; int i; int nid; - nnodes = read_once_policy_nodemask(pol, &nodemask); + nnodes = nodes_weight(pol->nodes); if (!nnodes) return numa_node_id(); target = ilx % nnodes; - nid = first_node(nodemask); - for (i = 0; i < target; i++) - nid = next_node(nid, nodemask); + nid = first_node(pol->nodes); + + /* A concurrent cpuset rebind may cause us to see an empty nodemask */ + for (i = 0; i < target && nid < MAX_NUMNODES; i++) + nid = next_node_in(nid, pol->nodes); + + /* the mask emptied under the walk */ + if (nid >= MAX_NUMNODES) + return numa_node_id(); return nid; } @@ -2665,7 +2674,6 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, u8 *table, weight; unsigned int weight_total = 0; unsigned long rem_pages = nr_pages; - nodemask_t nodes; int nnodes, node; int resume_node = MAX_NUMNODES - 1; u8 resume_weight = 0; @@ -2675,10 +2683,10 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, if (!nr_pages) return 0; - /* read the nodes onto the stack, retry if done during rebind */ + /* count the nodes, retry if a rebind happened during the read */ do { cpuset_mems_cookie = read_mems_allowed_begin(); - nnodes = read_once_policy_nodemask(pol, &nodes); + nnodes = nodes_weight(pol->nodes); } while (read_mems_allowed_retry(cpuset_mems_cookie)); /* if the nodemask has become invalid, we cannot do anything */ @@ -2688,7 +2696,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, /* Continue allocating from most recent node and adjust the nr_pages */ node = me->il_prev; weight = me->il_weight; - if (weight && node_isset(node, nodes)) { + if (weight && node_isset(node, pol->nodes)) { node_pages = min(rem_pages, weight); nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages, page_array); @@ -2712,9 +2720,13 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, table = state ? state->iw_table : NULL; /* calculate total, detect system default usage */ - for_each_node_mask(node, nodes) + for_each_node_mask(node, pol->nodes) weight_total += table ? table[node] : 1; + /* the mask emptied since it was counted */ + if (!weight_total) + goto out; + /* * Calculate rounds/partial rounds to minimize __alloc_pages_bulk calls. * Track which node weighted interleave should resume from. @@ -2724,10 +2736,14 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, */ rounds = rem_pages / weight_total; delta = rem_pages % weight_total; - resume_node = next_node_in(prev_node, nodes); + resume_node = next_node_in(prev_node, pol->nodes); + if (resume_node >= MAX_NUMNODES) + goto out; resume_weight = table ? table[resume_node] : 1; for (i = 0; i < nnodes; i++) { - node = next_node_in(prev_node, nodes); + node = next_node_in(prev_node, pol->nodes); + if (node >= MAX_NUMNODES) + break; weight = table ? table[node] : 1; node_pages = weight * rounds; /* If a delta exists, add this node's portion of the delta */ @@ -2744,6 +2760,8 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, /* node_pages can be 0 if an allocation fails and rounds == 0 */ if (!node_pages) break; + /* a rebind can invalidate the counts: never overrun page_array */ + node_pages = min(node_pages, nr_pages - total_allocated); nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages, page_array); page_array += nr_allocated; @@ -2754,6 +2772,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, } me->il_prev = resume_node; me->il_weight = resume_weight; +out: srcu_read_unlock_fast(&wi_srcu, scp); return total_allocated; } From 07526a538863f91293fca97a12f8dd692c7ceedd Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Wed, 19 Aug 2026 01:48:30 +0900 Subject: [PATCH 088/539] percpu: fix the comment about which sizes share a slot A percpu allocation is at least PCPU_MIN_ALLOC_SIZE bytes, and __pcpu_size_to_slot() returns 1 for sizes below 16 bytes and 2 for sizes from 16 to 31 bytes. So fix the wrong comment. Link: https://lore.kernel.org/20260818164831.3138490-1-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Signed-off-by: Andrew Morton Cc: Dennis Zhou Cc: Tejun Heo Cc: Christoph Lameter --- mm/percpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/percpu.c b/mm/percpu.c index a802d72c116f..47a903fe3b51 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -100,7 +100,7 @@ /* * The slots are sorted by the size of the biggest continuous free area. - * 1-31 bytes share the same slot. + * [PCPU_MIN_ALLOC_SIZE..15] bytes share the same slot. */ #define PCPU_SLOT_BASE_SHIFT 5 /* chunks in slots below this are subject to being sidelined on failed alloc */ From d5e1dc71ce3871d94bc65d3cefc461d957811dcc Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 18 Aug 2026 03:06:23 -0700 Subject: [PATCH 089/539] mm, swap: distinguish a malformed swap entry from a dying device Patch series "mm, swap: don't spin on a bad swap entry", v3. I've seen some machines at Meta fleet that show the following type of problem: 1) It gets some weird warning: BUG: Bad page map in process khugepaged pte:f000eef300000017 pmd:00000067 addr:00007f57c0a01000 vm_flags:20200073 anon_vma:ffff88829af7c340 mapping:0000000000000000 index:7f57c0a01 The corruption is most likely the collapse/PT_RECLAIM race fixed by commit 366a4532d96f ("mm: fix the race between collapse and PT_RECLAIM under per-vma lock"). But this series is not about this one. 2) Then the fault never makes progress. do_swap_page() returns 0 when get_swap_device() fails, so the fault is retried, reads the same entry and faults again. Nothing in the round trip changes the PTE, and the same line comes out on every pass: get_swap_device: Bad swap offset entry 3ffffffc043c5 Patch 1 makes get_swap_device() return ERR_PTR(-EIO) for a malformed entry, keeping NULL for a device swapoff is taking away, and converts the callers. No functional change expected. Patch 2 uses that to return VM_FAULT_SIGBUS instead of retrying. This patch (of 2): get_swap_device() returns NULL for two different things: an entry whose type names no swap device or whose offset is past the end of one, and a device that swapoff is taking away. The first never becomes valid, the second does, and callers cannot tell them apart. Return ERR_PTR(-EIO) for the two malformed cases and keep NULL for swapoff. copy_nonpresent_pte() already reports -EIO for an entry whose type names no device. Callers bail out on failure either way, so switch them to IS_ERR_OR_NULL(), and let the two paths that drop the reference skip an error pointer. No functional change. Link: https://lore.kernel.org/20260818-swap-v3-0-d3fa52598a59@debian.org Link: https://lore.kernel.org/20260818-swap-v3-1-d3fa52598a59@debian.org Signed-off-by: Breno Leitao Signed-off-by: Andrew Morton Reviewed-by: Barry Song Acked-by: Kairui Song Reviewed-by: Nhat Pham Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Chengming Zhou Cc: Chris Li Cc: Hugh Dickins Cc: Jann Horn Cc: Johannes Weiner Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Pedro Falcato Cc: Peter Xu Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/memory.c | 6 +++--- mm/mincore.c | 2 +- mm/shmem.c | 2 +- mm/swap_state.c | 4 ++-- mm/swapfile.c | 15 ++++++++++----- mm/userfaultfd.c | 4 ++-- mm/zswap.c | 2 +- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 27ffe1a99a08..e59d6c2a3432 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4956,9 +4956,9 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out; } - /* Prevent swapoff from happening to us. */ + /* Prevent swapoff from happening to us, and reject a bad entry. */ si = get_swap_device(entry); - if (unlikely(!si)) + if (IS_ERR_OR_NULL(si)) goto out; folio = swap_cache_get_folio(entry); @@ -5268,7 +5268,7 @@ unlock: if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl); out: - if (si) + if (!IS_ERR_OR_NULL(si)) put_swap_device(si); return ret; out_nomap: diff --git a/mm/mincore.c b/mm/mincore.c index ff4ac8281768..c086836bc4bc 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -71,7 +71,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem) */ if (shmem) { si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) return 0; } folio = swap_cache_get_folio(entry); diff --git a/mm/shmem.c b/mm/shmem.c index ae39966fc730..d3f24b5977bd 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2481,7 +2481,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, si = get_swap_device(index_entry); order = shmem_confirm_swap(mapping, index, index_entry); - if (unlikely(!si)) { + if (IS_ERR_OR_NULL(si)) { if (order < 0) return -EEXIST; else diff --git a/mm/swap_state.c b/mm/swap_state.c index f3961fdd857d..305877e1f4d7 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -716,7 +716,7 @@ struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, struct folio *folio; si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) return NULL; mpol = get_vma_policy(vma, addr, 0, &ilx); @@ -952,7 +952,7 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, */ if (swp_type(entry) != swp_type(targ_entry)) { si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) continue; } folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx, diff --git a/mm/swapfile.c b/mm/swapfile.c index 3b2279a16d1c..408f6c72fb5a 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1504,7 +1504,7 @@ int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp) unsigned long offset = swp_offset(entry); si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) return 0; ci = __swap_offset_to_cluster(si, offset); @@ -1859,7 +1859,10 @@ void folio_put_swap(struct folio *folio, struct page *page) * Check whether swap entry is valid in the swap device. If so, * return pointer to swap_info_struct, and keep the swap entry valid * via preventing the swap device from being swapoff, until - * put_swap_device() is called. Otherwise return NULL. + * put_swap_device() is called. Return NULL for an empty entry or a + * device that is going away, and ERR_PTR(-EIO) if the entry's type + * names no swap device or its offset is past the end of one. These EIOs + * are preceded by pr_err(). * * Notice that swapoff or swapoff+swapon can still happen before the * percpu_ref_tryget_live() in get_swap_device() or after the @@ -1900,12 +1903,14 @@ struct swap_info_struct *get_swap_device(swp_entry_t entry) return si; bad_nofile: pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val); + return ERR_PTR(-EIO); + out: return NULL; put_out: pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val); percpu_ref_put(&si->users); - return NULL; + return ERR_PTR(-EIO); } /* @@ -2001,7 +2006,7 @@ int swp_swapcount(swp_entry_t entry) int count; si = get_swap_device(entry); - if (!si) + if (IS_ERR_OR_NULL(si)) return 0; ci = swap_cluster_lock(si, swp_offset(entry)); @@ -2127,7 +2132,7 @@ void swap_put_entries_direct(swp_entry_t entry, int nr) struct swap_info_struct *si; si = get_swap_device(entry); - if (WARN_ON_ONCE(!si)) + if (WARN_ON_ONCE(IS_ERR_OR_NULL(si))) return; if (WARN_ON_ONCE(end_offset > si->max)) goto out; diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 74f04c323c50..95c1ed96df9a 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1700,7 +1700,7 @@ retry: } si = get_swap_device(entry); - if (unlikely(!si)) { + if (IS_ERR_OR_NULL(si)) { ret = -EAGAIN; goto out; } @@ -1757,7 +1757,7 @@ out: if (dst_pte) pte_unmap(dst_pte); mmu_notifier_invalidate_range_end(&range); - if (si) + if (!IS_ERR_OR_NULL(si)) put_swap_device(si); return ret; diff --git a/mm/zswap.c b/mm/zswap.c index fc869d60ef1b..f3ae3c81e48e 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -984,7 +984,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, /* try to allocate swap cache folio */ si = get_swap_device(swpentry); - if (!si) + if (IS_ERR_OR_NULL(si)) return -EEXIST; mpol = get_task_policy(current); From fbc63fccf46da5802f7dfa5c5cb77faa00bbc770 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 18 Aug 2026 03:06:24 -0700 Subject: [PATCH 090/539] mm: fail the fault on a malformed swap entry instead of retrying it do_swap_page() returns 0 when get_swap_device() fails, which the fault handler reads as "handled". For an entry that can never become valid the retry takes the same fault again, so the thread spins forever, retrying on the same fault. Return VM_FAULT_SIGBUS (Bad access) for a malformed entry (pr_err() was called at get_swap_device()). Link: https://lore.kernel.org/20260818-swap-v3-2-d3fa52598a59@debian.org Signed-off-by: Breno Leitao Signed-off-by: Andrew Morton Acked-by: Kairui Song Reviewed-by: Barry Song Reviewed-by: Nhat Pham Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Chengming Zhou Cc: Chris Li Cc: Hugh Dickins Cc: Jann Horn Cc: Johannes Weiner Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Pedro Falcato Cc: Peter Xu Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/memory.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/memory.c b/mm/memory.c index e59d6c2a3432..347db2acd0f8 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4958,8 +4958,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) /* Prevent swapoff from happening to us, and reject a bad entry. */ si = get_swap_device(entry); - if (IS_ERR_OR_NULL(si)) + if (IS_ERR_OR_NULL(si)) { + if (IS_ERR(si)) + ret = VM_FAULT_SIGBUS; goto out; + } folio = swap_cache_get_folio(entry); if (folio) From 3a09df924fa2ba75ac80acb9bce4eed79be218fa Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Mon, 17 Aug 2026 18:08:08 -0400 Subject: [PATCH 091/539] mm/huge_memory: skip zone device folios in madvise_free_huge_pmd() Patch series "mm: reject zone device folios in more folio walkers", v2. Several LRU-oriented mm walkers resolve the folio backing a PMD entry (or a physical pfn) and then reclaim, age, migrate, or lazyfree it without ever checking for ZONE_DEVICE memory. This series adds missing folio_is_zone_device() rejections, matching the checks that comparable walkers already perform. - mm/huge_memory, mm/madvise: the !pmd_present branch above these sites only filters device-private entries (which are non-present). A present zone device PMD (e.g. device-coherent) would still reach the folio and be lazyfreed / aged / paged out. Add an explicit check. - mm/mempolicy: queue_folios_pmd() can see a present zone device PMD (e.g. device-coherent) and queue it for migration. No crash reproducer - this is a correctness/hardening cleanup found by inspection. All checks are placed after the folio is resolved and before it is acted upon, on paths that already hold the relevant page-table lock, so no locking or refcount changes are involved. This patch (of 3): madvise_free_huge_pmd() resolves the folio backing a PMD via pmd_folio() and marks it lazyfree without checking for zone device memory. The surrounding guards do not cover every zone device case: - MADV_FREE only operates on anonymous VMAs (DAX mappings are excluded) - !pmd_present() branch rejects device-private and migration entries - present zone device PMD (device coherent THP) is not filtered. Unlike vm_normal_page_pmd(), it performs no special/pfnmap check, and would be marked lazyfree here. Bail out when the folio is a zone device folio. Link: https://lore.kernel.org/20260817220810.1175596-1-gourry@gourry.net Link: https://lore.kernel.org/20260817220810.1175596-2-gourry@gourry.net Fixes: a30b48bf1b24 ("mm/migrate_device: implement THP migration of zone device pages") Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Tested-by: Lance Yang Cc: Alistair Popple Cc: Balbir Singh Cc: Baolin Wang Cc: Barry Song Cc: Byungchul Park Cc: Dev Jain Cc: "Huang, Ying" Cc: Jann Horn Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Matthew Brost Cc: Rakie Kim Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Zi Yan Cc: --- mm/huge_memory.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1e5d68acf62a..54494c3fa983 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2423,6 +2423,10 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, } folio = pmd_folio(orig_pmd); + + if (folio_is_zone_device(folio)) + goto out; + /* * If other processes are mapping this folio, we couldn't discard * the folio unless they all do MADV_FREE so let's skip the folio. From ab7ace574f3da7f167fe1883116480a00c52150c Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Mon, 17 Aug 2026 18:08:09 -0400 Subject: [PATCH 092/539] mm/madvise: skip zone device folios in cold/pageout PMD range madvise_cold_or_pageout_pte_range() resolves the folio backing a PMD via pmd_folio() and ages or reclaims it without checking for zone device memory. The surrounding guards do not cover every zone device case: - can_madv_lru_vma() excludes VM_PFNMAP and VM_HUGETLB VMAs (so device DAX is filtered) - !pmd_present() branch above rejects device-private and migration entries, which are non-present. - A present zone device PMD - e.g. a device-coherent THP - is not filtered by any of these, nor by pmd_folio() (unlike vm_normal_page_pmd(), it performs no special/pfnmap check), and would be aged or paged out here. Skip ZONE_DEVICE folios explicitly during MADV_COLD/PAGEOUT. Link: https://lore.kernel.org/20260817220810.1175596-3-gourry@gourry.net Fixes: a30b48bf1b24 ("mm/migrate_device: implement THP migration of zone device pages") Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Balbir Singh Tested-by: Lance Yang Cc: Alistair Popple Cc: Baolin Wang Cc: Barry Song Cc: Byungchul Park Cc: Dev Jain Cc: "Huang, Ying" Cc: Jann Horn Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Matthew Brost Cc: Rakie Kim Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Zi Yan Cc: --- mm/madvise.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/madvise.c b/mm/madvise.c index eeee82cf2b3f..73c2901b9adb 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -404,6 +404,9 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, folio = pmd_folio(orig_pmd); + if (folio_is_zone_device(folio)) + goto huge_unlock; + /* Do not interfere with other mappings of this folio */ if (folio_maybe_mapped_shared(folio)) goto huge_unlock; From 1f834ba9b42e3f3c298c7e8ef1df21b9947c92b4 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Mon, 17 Aug 2026 18:08:10 -0400 Subject: [PATCH 093/539] mm/mempolicy: skip zone device folios when queueing folios queue_folios_pte_range() already pairs vm_normal_folio() with an explicit folio_is_zone_device() check before adding folios to the migration pagelist. vm_normal_folio() alone does not reject zone device memory (a present device-coherent page in a normal VMA is returned as "normal"). Mirror the explicit check in queue_folios_pmd() as well. queue_folios_pmd() uses pmd_folio() directly and can encounter a present zone device PMD - e.g. a device-coherent THP. This is not filtered by existing checks: !pmd_present() - only rejects non-present device-private and migration entries vma_migratable() - excludes DAX and VM_PFNMAP. The early return also means such a folio is no longer counted in qp->nr_failed under MPOL_MF_STRICT. This is the same pattern used by queue_folios_pte_range() (skipping zone device without failing). Link: https://lore.kernel.org/20260817220810.1175596-4-gourry@gourry.net Fixes: a30b48bf1b24 ("mm/migrate_device: implement THP migration of zone device pages") Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Reviewed-by: Balbir Singh Acked-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Tested-by: Lance Yang Cc: Alistair Popple Cc: Baolin Wang Cc: Barry Song Cc: Byungchul Park Cc: Dev Jain Cc: "Huang, Ying" Cc: Jann Horn Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Matthew Brost Cc: Rakie Kim Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Zi Yan Cc: --- mm/mempolicy.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index fd97fb0289bc..95dba5d919e9 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -679,6 +679,8 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) return; } folio = pmd_folio(pmdval); + if (folio_is_zone_device(folio)) + return; if (is_huge_zero_folio(folio)) { walk->action = ACTION_CONTINUE; return; From 7ae9c1d82ed722432c66f35fc99ede9c4778d8aa Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 17 Aug 2026 14:19:55 +0800 Subject: [PATCH 094/539] selftests/mm: khugepaged: consolidate error exits via kselftest helpers Replace the perror()+exit(EXIT_FAILURE) pattern with ksft_exit_fail_perror() so failures are reported through the kselftest framework, consistent with the rest of the file. Link: https://lore.kernel.org/20260817061955.45454-1-hongfu.li@linux.dev Signed-off-by: Hongfu Li Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Zi Yan Reviewed-by: Lance Yang Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Ryan Roberts Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/khugepaged.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 83d27d069c41..f82673f5f6b4 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -337,21 +337,15 @@ static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) ksft_exit_fail_perror("open()"); size = nr_hpages * hpage_pmd_size; - if (ftruncate(fd, size)) { - perror("ftruncate()"); - exit(EXIT_FAILURE); - } + if (ftruncate(fd, size)) + ksft_exit_fail_perror("ftruncate()"); p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (p != BASE_ADDR) { - perror("mmap()"); - exit(EXIT_FAILURE); - } + if (p != BASE_ADDR) + ksft_exit_fail_perror("mmap()"); fill_memory(p, 0, size); - if (msync(p, size, MS_SYNC)) { - perror("msync()"); - exit(EXIT_FAILURE); - } + if (msync(p, size, MS_SYNC)) + ksft_exit_fail_perror("msync()"); close(fd); munmap(p, size); success("OK"); @@ -426,7 +420,7 @@ static bool file_check_huge(void *addr, size_t len, int nr_hpages, case VMA_SHMEM: return check_huge_shmem(addr, len, nr_hpages, hpage_size); default: - exit(EXIT_FAILURE); + ksft_exit_fail_msg("Unknown VMA type\n"); return false; } } From 3307103acd4769b295a495f0740a543e894dac5d Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Sun, 30 Aug 2026 08:20:43 +0800 Subject: [PATCH 095/539] memcg: acquire peaks_lock when reading memory.peak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm, memcg: fix memory.peak reset clobbering other fds' watermark", v4. The memory.peak / memory.swap.peak per-fd watermark tracking has two issues. Each open fd is a watcher and reads back max(its own value, the shared local_watermark); both bugs live in that scheme. Worst case for both is the same and is userspace-visible: a reader of memory.peak (or memory.swap.peak) gets a value lower than the true peak, so a tool that sizes or bills a cgroup by its peak usage under-reports it. Patch 1 (read side) fixes the race Sashiko pointed out in the v1 review [1]: peak_show() inspects local_watermark and the per-fd values without holding peaks_lock, so a reader that races an unrelated peak_write() reset briefly observes the lowered value. Transient. It takes peaks_lock in the show path. Patch 2 (write side) fixes peak_write(): on a reset it stores the current usage into the other watchers instead of the old watermark, so once usage has dropped from a peak a reset on one fd drags every other fd's peak down too, even fds that never reset. This patch (of 2): Sashiko reported that a reader can transiently observe a lower peak within a race window [1]. peak_show() returns max(local_watermark, ofp->value), but peak_write() updates those two under peaks_lock while the reader takes no lock. The interleaving is: writer (reset on fd A) reader (fd B) ---------------------- ------------- usage = page_counter_read(pc) WRITE_ONCE(local_watermark, usage) // watermark lowered to usage lw = READ_ONCE(local_watermark) // sees the lowered usage val = READ_ONCE(ofp->value) // B's value not updated yet return max(lw, val) // both low -> low peak WRITE_ONCE(peer_ctx->value, usage) // B updated, but too late Fix it by acquiring peaks_lock when reading the peak, so the reader sees a consistent snapshot of local_watermark and the per-fd values. The same race applies to memory.swap.peak, which shares peaks_lock and the peak_write() path, so take the lock there as well. Link: https://lore.kernel.org/20260830002044.1938621-1-ridong.chen@linux.dev Link: https://lore.kernel.org/20260830002044.1938621-2-ridong.chen@linux.dev Link: https://sashiko.dev/#/patchset/20260730115314.1069089-1-ridong.chen@linux.dev?part=1 [1] Fixes: c6f53ed8f213 ("mm, memcg: cg2 memory{.swap,}.peak write handlers") Signed-off-by: Ridong Chen Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Acked-by: Shakeel Butt Reviewed-by: Muchun Song Assisted-by: Claude:claude-opus-4-8 Cc: David Finkel Cc: Michal Hocko Cc: Michal Koutný Cc: Roman Gushchin Cc: Tejun Heo Cc: Tao Cui --- mm/memcontrol.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 30636b9d9673..1ee974cb6d37 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4745,6 +4745,7 @@ static int memory_peak_show(struct seq_file *sf, void *v) { struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); + guard(spinlock)(&memcg->peaks_lock); return peak_show(sf, v, &memcg->memory); } @@ -5888,6 +5889,7 @@ static int swap_peak_show(struct seq_file *sf, void *v) { struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); + guard(spinlock)(&memcg->peaks_lock); return peak_show(sf, v, &memcg->swap); } From 8e7caac55a6f71559925860595198489cb3641c0 Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Sun, 30 Aug 2026 08:20:44 +0800 Subject: [PATCH 096/539] mm, memcg: fix memory.peak reset clobbering other fds' watermark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Writing to memory.peak resets the peak for that fd only. Each fd is a watcher and reads back max(its own value, the shared local_watermark). peak_write() resets by lowering local_watermark to the current usage. To keep the other watchers' peaks it then walks the watcher list, but it stores the current usage into them instead of the old watermark. So once usage has dropped from a peak, a reset on one fd wrongly drags every other fd's peak down too, even fds that never reset. Reproduced on 7.2.0-rc5-next under QEMU, two fds A and B on one cgroup: B sees the peak (410624 KB), usage drops, then A resets -- and B's peak collapses to 1060 KB although B never reset. With this patch B keeps reading 410624 KB. Fix: save the old watermark before lowering it and use that to floor the other watchers, so a reset only affects the fd that issued it. Link: https://lore.kernel.org/20260830002044.1938621-3-ridong.chen@linux.dev Fixes: c6f53ed8f213 ("mm, memcg: cg2 memory{.swap,}.peak write handlers") Signed-off-by: Ridong Chen Signed-off-by: Andrew Morton Closes: https://sashiko.dev/#/patchset/20260807090000.1532495-1-ridong.chen@linux.dev Acked-by: Tao Cui Acked-by: Johannes Weiner Acked-by: Shakeel Butt Assisted-by: Claude:claude-opus-4-8 Cc: David Finkel Cc: Michal Hocko Cc: Michal Koutný Cc: Muchun Song Cc: Roman Gushchin Cc: Tejun Heo --- mm/memcontrol.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1ee974cb6d37..d5ebe83eae3e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4775,7 +4775,7 @@ static ssize_t peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off, struct page_counter *pc, struct list_head *watchers) { - unsigned long usage; + unsigned long usage, old_watermark; struct cgroup_of_peak *peer_ctx; struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); struct cgroup_of_peak *ofp = of_peak(of); @@ -4783,11 +4783,12 @@ static ssize_t peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes, spin_lock(&memcg->peaks_lock); usage = page_counter_read(pc); + old_watermark = READ_ONCE(pc->local_watermark); WRITE_ONCE(pc->local_watermark, usage); list_for_each_entry(peer_ctx, watchers, list) - if (usage > peer_ctx->value) - WRITE_ONCE(peer_ctx->value, usage); + if (peer_ctx != ofp && old_watermark > peer_ctx->value) + WRITE_ONCE(peer_ctx->value, old_watermark); /* initial write, register watcher */ if (ofp->value == OFP_PEAK_UNSET) From 4886cec0b283ff89c87c7b92f22e328439bf7b1f Mon Sep 17 00:00:00 2001 From: Eamon Sippy Date: Sat, 15 Aug 2026 10:32:45 +0000 Subject: [PATCH 097/539] mm: cma: make mm/cma.h self-contained and conditionalize includes mm/cma.h uses types from , , and without explicitly including them, violating the kernel header self-containment guidelines. and are also included unconditionally even though they are only needed under CONFIG_CMA_DEBUGFS and CONFIG_CMA_SYSFS respectively. Move the struct cma_kobject definition and inside the CONFIG_CMA_SYSFS block, and move inside CONFIG_CMA_DEBUGFS. Remove spurious trailing semicolons after the empty inline function bodies in the CONFIG_CMA_SYSFS #else branch. Add so that MAX_CMA_AREAS and CMA_MAX_NAME are always available when this header is included. Link: https://lore.kernel.org/20260815103246.5315-1-eamon112009@gmail.com Signed-off-by: Eamon Sippy Signed-off-by: Andrew Morton Reviewed-by: Barry Song --- mm/cma.h | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/mm/cma.h b/mm/cma.h index ab6d39898ea5..68e574b0f951 100644 --- a/mm/cma.h +++ b/mm/cma.h @@ -2,14 +2,24 @@ #ifndef __MM_CMA_H__ #define __MM_CMA_H__ +#include #include +#include +#include +#include + +#ifdef CONFIG_CMA_DEBUGFS #include +#endif + +#ifdef CONFIG_CMA_SYSFS #include struct cma_kobject { struct kobject kobj; struct cma *cma; }; +#endif /* * Multi-range support. This can be useful if the size of the allocation @@ -38,10 +48,10 @@ struct cma_memrange { #define CMA_MAX_RANGES 8 struct cma { - unsigned long count; - unsigned long available_count; + unsigned long count; + unsigned long available_count; unsigned int order_per_bit; /* Order of pages represented by one bit */ - spinlock_t lock; + spinlock_t lock; struct mutex alloc_mutex; #ifdef CONFIG_CMA_DEBUGFS struct hlist_head mem_head; @@ -87,10 +97,11 @@ void cma_sysfs_account_fail_pages(struct cma *cma, unsigned long nr_pages); void cma_sysfs_account_release_pages(struct cma *cma, unsigned long nr_pages); #else static inline void cma_sysfs_account_success_pages(struct cma *cma, - unsigned long nr_pages) {}; + unsigned long nr_pages) {} static inline void cma_sysfs_account_fail_pages(struct cma *cma, - unsigned long nr_pages) {}; + unsigned long nr_pages) {} static inline void cma_sysfs_account_release_pages(struct cma *cma, - unsigned long nr_pages) {}; + unsigned long nr_pages) {} #endif + #endif From f1dff3456c9afd90408e77f7edc2ec960e179217 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Thu, 13 Aug 2026 17:38:09 +0800 Subject: [PATCH 098/539] mm/oom_kill: remove unreachable __GFP_THISNODE check in constrained_alloc() The __GFP_THISNODE check in constrained_alloc() is dead code: global OOM is never triggered with __GFP_THISNODE (blocked in __alloc_pages_may_oom before out_of_memory() is called), and memcg OOM returns CONSTRAINT_MEMCG at the top of the function before reaching this point. Remove the check, its stale comment, and update the following comment that referenced __GFP_THISNODE. Link: https://lore.kernel.org/20260813093810.573302-1-ye.liu@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Acked-by: Michal Hocko Acked-by: Shakeel Butt Cc: David Rientjes --- mm/oom_kill.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 5f372f6e26fa..fd3c476846a3 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -267,18 +267,11 @@ static enum oom_constraint constrained_alloc(struct oom_control *oc) if (!oc->zonelist) return CONSTRAINT_NONE; - /* - * Reach here only when __GFP_NOFAIL is used. So, we should avoid - * to kill current.We have to random task kill in this case. - * Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now. - */ - if (oc->gfp_mask & __GFP_THISNODE) - return CONSTRAINT_NONE; /* - * This is not a __GFP_THISNODE allocation, so a truncated nodemask in - * the page allocator means a mempolicy is in effect. Cpuset policy - * is enforced in get_page_from_freelist(). + * A truncated nodemask in the page allocator means a mempolicy + * is in effect. Cpuset policy is enforced in + * get_page_from_freelist(). */ if (oc->nodemask && !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) { From abef8c99527a2e8deb41e9b814a920580c49eff5 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 11 Aug 2026 11:36:08 +0800 Subject: [PATCH 099/539] mm/oom_kill, proc: replace magic number 1000 with OOM_SCORE_ADJ_MAX In oom_badness() and proc_oom_score(), the oom_score_adj normalization uses a hardcoded 1000, which is the value of OOM_SCORE_ADJ_MAX defined in include/uapi/linux/oom.h. Other code in the kernel (e.g. fs/proc/base.c oom_adj handling) already uses OOM_SCORE_ADJ_MAX for the same purpose. Replace the magic number with the macro for consistency and readability. No functional change. Link: https://lore.kernel.org/20260811033609.3992348-1-ye.liu@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Acked-by: Michal Hocko Cc: David Rientjes Cc: Shakeel Butt Cc: Song Hu --- fs/proc/base.c | 3 ++- mm/oom_kill.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index 6a39de424f62..58be38942460 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -594,7 +594,8 @@ static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns, * exporting for a long time so userspace might depend on it. */ if (badness != LONG_MIN) - points = (1000 + badness * 1000 / (long)totalpages) * 2 / 3; + points = (OOM_SCORE_ADJ_MAX + + badness * OOM_SCORE_ADJ_MAX / (long)totalpages) * 2 / 3; seq_printf(m, "%lu\n", points); diff --git a/mm/oom_kill.c b/mm/oom_kill.c index fd3c476846a3..5d48bd862c27 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -230,7 +230,7 @@ long oom_badness(struct task_struct *p, unsigned long totalpages) task_unlock(p); /* Normalize to oom_score_adj units */ - adj *= totalpages / 1000; + adj *= totalpages / OOM_SCORE_ADJ_MAX; points += adj; return points; From 0c39f49a22bf39c0077e9939fda3c8afb889c3cd Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Tue, 11 Aug 2026 18:21:55 +0100 Subject: [PATCH 100/539] mm: replace custom bad page map ratelimiting logic Patch series "mm: replace custom ratelimiting logic". The kernel has a perfectly cromulent and mostly-equivalent variant in lib/ratelimit.c that can be used. This patch (of 2): The current logic (allow up to $BURST prints per minute) can be entirely replaced by the generic version in lib/ratelimit.c, used around the kernel. Do so. The only functional difference should be that the new logs will read something like: KERN_WARNING "print_bad_page_map: %d callbacks suppressed\n", ... But that should be fine enough. Link: https://lore.kernel.org/20260811172156.356053-1-pfalcato@suse.de Link: https://lore.kernel.org/20260811172156.356053-2-pfalcato@suse.de Signed-off-by: Pedro Falcato Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Acked-by: Zi Yan Reviewed-by: SJ Park Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: Vlastimil Babka (SUSE) Acked-by: Mike Rapoport (Microsoft) Cc: Brendan Jackman Cc: Liam R. Howlett Cc: Michal Hocko Cc: Suren Baghdasaryan --- mm/memory.c | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 347db2acd0f8..09ac784f8b7b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -492,32 +492,8 @@ static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) add_mm_counter(mm, i, rss[i]); } -static bool is_bad_page_map_ratelimited(void) -{ - static unsigned long resume; - static unsigned long nr_shown; - static unsigned long nr_unshown; - - /* - * Allow a burst of 60 reports, then keep quiet for that minute; - * or allow a steady drip of one report per second. - */ - if (nr_shown == 60) { - if (time_before(jiffies, resume)) { - nr_unshown++; - return true; - } - if (nr_unshown) { - pr_alert("BUG: Bad page map: %lu messages suppressed\n", - nr_unshown); - nr_unshown = 0; - } - nr_shown = 0; - } - if (nr_shown++ == 0) - resume = jiffies + 60 * HZ; - return false; -} +/* Allow a burst of 60 bad page map reports per minute. */ +static DEFINE_RATELIMIT_STATE(bad_page_map_ratelimit, 60 * HZ, 60); static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size) { @@ -633,7 +609,7 @@ static void print_bad_page_map(struct vm_area_struct *vma, char entry_str[PTVAL_STR_MAX]; pgoff_t index, anon_index; - if (is_bad_page_map_ratelimited()) + if (!__ratelimit(&bad_page_map_ratelimit)) return; mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; From 0685b37a7098d47586441c874dfd5bdc23d6e587 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Tue, 11 Aug 2026 18:21:56 +0100 Subject: [PATCH 101/539] mm/page_alloc: replace custom bad page ratelimiting logic The current logic (allow up to $BURST prints per minute) can be entirely replaced by the generic version in lib/ratelimit.c, used around the kernel. Do so. The only functional difference should be that the new logs will read something like: KERN_WARNING "bad_page: %d callbacks suppressed\n", ... But that should be fine enough. Link: https://lore.kernel.org/20260811172156.356053-3-pfalcato@suse.de Signed-off-by: Pedro Falcato Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Acked-by: Zi Yan Reviewed-by: SJ Park Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: Vlastimil Babka (SUSE) Acked-by: Mike Rapoport (Microsoft) Cc: Brendan Jackman Cc: Liam R. Howlett Cc: Michal Hocko Cc: Suren Baghdasaryan --- mm/page_alloc.c | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 146f7e0a9462..c4dc61ec663e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -613,31 +613,13 @@ static inline bool __maybe_unused bad_range(struct zone *zone, struct page *page } #endif +/* Allow a burst of 60 reports per minute */ +static DEFINE_RATELIMIT_STATE(bad_page_ratelimit, 60 * HZ, 60); + static void bad_page(struct page *page, const char *reason) { - static unsigned long resume; - static unsigned long nr_shown; - static unsigned long nr_unshown; - - /* - * Allow a burst of 60 reports, then keep quiet for that minute; - * or allow a steady drip of one report per second. - */ - if (nr_shown == 60) { - if (time_before(jiffies, resume)) { - nr_unshown++; - goto out; - } - if (nr_unshown) { - pr_alert( - "BUG: Bad page state: %lu messages suppressed\n", - nr_unshown); - nr_unshown = 0; - } - nr_shown = 0; - } - if (nr_shown++ == 0) - resume = jiffies + 60 * HZ; + if (!__ratelimit(&bad_page_ratelimit)) + goto out; pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", current->comm, page_to_pfn(page)); From 0041eef981cc55bb1f316a9bbda993adf9079402 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Mon, 31 Aug 2026 14:42:11 +0100 Subject: [PATCH 102/539] mm/vmpressure: remove window size TODO There has been a steady stream of patches that have been submitted by newcomers to core mm 'fixing' this TODO, with all but the original having very likely been generated by LLMs. It appears that TODOs to LLMs are like red rags to a bull. In addition, TODOs in code often bitrot and are distracting - those who understand the code know what could be improved in future. Therefore remove the TODO. The work required to actually fix this TODO requires somebody who both has understanding of the code and significant real-world data to back their changes. Such a person doesn't require a TODO prompt to implement this change, so nothing of value is being lost here. Link: https://lore.kernel.org/all/20260831130316.448-1-tahasezer.is@gmail.com/ Link: https://lore.kernel.org/linux-mm/20260724054305.516126-1-cui.tao@linux.dev/ Link: https://lore.kernel.org/linux-mm/20260715143646.15828-1-gaikwad.dcg@gmail.com/ Link: https://lore.kernel.org/all/20260227221555.29969-1-mcq@disroot.org/ Link: https://lore.kernel.org/20260831-remove-vmpressure-todo-v1-1-498515e59cdf@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Vlastimil Babka (SUSE) Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan --- mm/vmpressure.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/mm/vmpressure.c b/mm/vmpressure.c index 9629240d77ad..3de99fef3928 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -30,9 +30,6 @@ * * As the vmscan reclaimer logic works with chunks which are multiple of * SWAP_CLUSTER_MAX, it makes sense to use it for the window size as well. - * - * TODO: Make the window size depend on machine size, as we do for vmstat - * thresholds. Currently we set it to 512 pages (2MB for 4KB pages). */ const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; From 5554a09b66356f94d1e41bc249ba21bbb32c325f Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Mon, 31 Aug 2026 15:18:23 +0100 Subject: [PATCH 103/539] tools/testing/selftests/mm: add missing .gitignore entries Commit 2bee308f3adb ("selftests/mm: use pattern matching in .gitignore") switched to a pattern-matching mechanism to reduce churn in .gitignore. It however accidentally excluded the page_frag test's-generated module intermediate C file with .mod.c extension, and also the local_config.h header generated if liburing is available locally. Explicitly fix both the issues, fixing the module-generated C file as a general pattern as these are always intermediate files that should be ignored. Since this is a trivial .gitignore change it doesn't seem necessary to treat it as a hotfix. Link: https://lore.kernel.org/20260831-fix-mm-selftests-gitignore-v1-1-c984bbd4c5e4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Reviewed-by: Gregory Price (Meta) Reviewed-by: Sarthak Sharma Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/mm/.gitignore b/tools/testing/selftests/mm/.gitignore index fcd892ed21e3..a306d7754786 100644 --- a/tools/testing/selftests/mm/.gitignore +++ b/tools/testing/selftests/mm/.gitignore @@ -2,7 +2,9 @@ * !/**/ !*.c +*.mod.c !*.h +local_config.h !*.sh !.gitignore !Makefile From 47e725702b0b7c01d4ae4ef231df8f63e5f305d5 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 31 Aug 2026 13:30:52 -0700 Subject: [PATCH 104/539] mm: make per-VMA locks available universally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm: Unconditional per-VMA locks and cleanups", v7. tl;dr: Make per-VMA locks available in all configs. Simplify some of the per-VMA lock users now that they can rely on them being always available. Binder and networking folks: Your code is the target of the cleanups. I'm cc'ing you now on v2 because there's emerging consensus on the mm side that the approach here is sane. I'm not quite sure how this pile would get merged, but ack/review tags would be appreciated if this looks good to you. Longer version: When working on some x86 shadow stack code, it was a real pain to avoid causing recursive locking problems with mmap_lock. One way to avoid those was to avoid mmap_lock and use per-VMA locks instead. They are great, but they are not available in all configs which makes them unusable in generic code, or if you want to completely avoid mmap_lock. Make per-VMA locks available in all configs. Right now, they are only available on select architectures when SMP and MMU are enabled. But all of the primitives that per-VMA locks are built on (RCU, maple trees, refcounts) work just fine without SMP or MMU. The only real downside is that making VMAs a wee bit bigger on !MMU and !SMP builds. The upside is much cleaner code, lower complexity and less #ifdeffery. Clean up a binder VMA locking site now that it can rely on per-VMA locks. Building on top of universally-available per-VMA locks, introduce a new helper. Since the new API does not require callers to have a fallback to mmap_lock, it's much easier to use. Callers can potentially replace this very common kernel idiom: mmap_read_lock(mm); vma = vma_lookup() // fiddle with vma mmap_read_unlock(mm); with: vma = vma_start_read_unlocked(mm, address); // fiddle with vma vma_end_read(vma); Which avoids mmap_lock entirely in the fast path. Use that new API for another binder site and one in the TCP code. This patch (of 7): The per-VMA locks have been around for several years. They've had some bugs worked out of them and have seen quite wide use. However, they are still only available when architectures explicitly enable them. Remove the conditional compilation around the per-VMA locks, making them available on all architectures and configs. The approach up to now seemed to be to add ARCH_SUPPORTS_PER_VMA_LOCK when the architecture started using per-VMA locks in the fault handler. But, contrary to the naming, the Kconfig option does not really indicate whether the architecture supports per-VMA locks or not. It is more of a marker for whether the architecture is likely to benefit from per-VMA locks. To me, the most important thing side-effect of universal availability is letting per-VMA locks be used in SMP=n configs. This lets us use per-VMA locking in all x86 code without fallbacks. Overall, this just generally makes the kernel simpler. Just look at the diffstat. It also opens the door to users that want to use the per-VMA locks in common code. Doing *that* brings additional simplifications. The downside of this is adding some fields to vm_area_struct and mm_struct. There are likely ways to optimize this, especially for things like SMP=n configs. For now, do the simplest thing: use the same implementation everywhere. == Considerations for NOMMU config == NOMMU systems do not write-lock VMAs, therefore read-locking a VMA would always succeed unless VMA is detached. Therefore for NOMMU config we make vma_mark_attached() a NOOP, which keeps VMAs always in detached state. This causes VMA read-locking to always fail and the caller falls back to locking mmap_lock. The following functions will have a different implementation in NOMMU config: - vma_mark_attached(), vma_mark_detached() are made NOOPs, keeping VMAs always in a detached state and preventing assertions and refcount underflows; - vma_start_write(), vma_start_write_killable() are made NOOPs to avoid warnings in __vma_start_write() due to VMAs being detached. These functions are not used in NOMMU code but __vma_start_write() is an exported function, therefore might be used by drivers. - vma_assert_attached() is made NOOP because it's reachable from NOMMU code via split_vma()->vma_iter_store_new()->vma_iter_store_overwrite(); - vma_assert_write_locked() is asserting vma->vm_mm is write-locked, as was done before this change; - vma_assert_locked() is asserting vma->vm_mm is locked, as was done before this change; The following functions work for both MMU and NOMMU configs: - vma_lock_init() performs the same initialization as for MMU config; - mm_lock_seqcount_init(), mm_lock_seqcount_begin(), mm_lock_seqcount_end() are called from mmap_write_{lock|unlock} and update mm_lock_seq correctly. - mmap_lock_speculate_try_begin(), mmap_lock_speculate_retry() work as is because mm_lock_seq is updated correctly; - vma_start_read(), vma_start_read_locked() will always fail because VMAs are always detached; - vma_end_read() will never be called because vma_start_read() never succeeds; - vma_is_attached() always return false because VMAs are always detached; - vma_assert_detached() will never trigger because VMAs are never attached; - vma_start_read_locked() always return false because VMAs are always detached; - lock_vma_under_rcu() will be safe as the attempted read lock will bail; Changes in the following files are not affecting NOMMU config: task_mmu.c - not compiled when CONFIG_MMU=n; pagewalk.c - not compiled when CONFIG_MMU=n; userfaultfd.c - not compiled when CONFIG_MMU=n (CONFIG_USERFAULTFD depends on CONFIG_MMU); The following changes in the BPF code are made to keep NOMMU config working like before: stack_map_lock_vma() - keeps mmap_lock in NOMMU config; bpf_iter_task_vma_new() - bails out in NOMMU config; Link: https://lore.kernel.org/20260831203056.838265-1-surenb@google.com Link: https://lore.kernel.org/20260831203056.838265-2-surenb@google.com Signed-off-by: Dave Hansen Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: Vlastimil Babka (SUSE) Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Shakeel Butt Cc: Greg Kroah-Hartman Cc: Todd Kjos Cc: Christian Brauner Cc: Carlos Llamas Cc: Alice Ryhl Cc: David S. Miller Cc: David Ahern Cc: Arve Hjønnevåg --- arch/arm/Kconfig | 1 - arch/arm64/Kconfig | 1 - arch/loongarch/Kconfig | 1 - arch/powerpc/platforms/powernv/Kconfig | 1 - arch/powerpc/platforms/pseries/Kconfig | 1 - arch/riscv/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/x86/Kconfig | 2 - fs/proc/internal.h | 2 - fs/proc/task_mmu.c | 93 -------------------------- include/linux/mm.h | 12 ---- include/linux/mm_types.h | 8 +-- include/linux/mmap_lock.h | 75 +++++++-------------- kernel/bpf/stackmap.c | 17 ++--- kernel/bpf/task_iter.c | 2 +- kernel/fork.c | 2 - mm/Kconfig | 12 ---- mm/Kconfig.debug | 1 - mm/debug.c | 4 -- mm/init-mm.c | 2 - mm/memory.c | 2 - mm/mmap_lock.c | 26 +------ mm/pagewalk.c | 2 - mm/rmap.c | 2 - mm/userfaultfd.c | 55 --------------- rust/kernel/mm.rs | 32 +++------ tools/testing/vma/include/dup.h | 5 +- tools/testing/vma/vma_internal.h | 1 - 28 files changed, 48 insertions(+), 316 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index ffbc7f386131..408aa58a2a5b 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -42,7 +42,6 @@ config ARM select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_CFI select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_RT select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b5a51b0ef944..2bbeded33da0 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -81,7 +81,6 @@ config ARM64 select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SCHED_SMT diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 2067d1f2ad7a..1d8fb1e456d6 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -69,7 +69,6 @@ config LOONGARCH select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS select ARCH_HAS_PTE_PROTNONE if 64BIT select ARCH_SUPPORTS_NUMA_BALANCING if NUMA - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SCHED_SMT if SMP select ARCH_SUPPORTS_SCHED_MC if SMP diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig index b5ad7c173ef0..dd8f6060fb7a 100644 --- a/arch/powerpc/platforms/powernv/Kconfig +++ b/arch/powerpc/platforms/powernv/Kconfig @@ -17,7 +17,6 @@ config PPC_POWERNV select PPC_DOORBELL select MMU_NOTIFIER select FORCE_SMP - select ARCH_SUPPORTS_PER_VMA_LOCK select PPC_RADIX_BROADCAST_TLBIE if PPC_RADIX_MMU default y diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index 74910ce3a541..7d125e288f6e 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -23,7 +23,6 @@ config PPC_PSERIES select HOTPLUG_CPU select FORCE_SMP select SWIOTLB - select ARCH_SUPPORTS_PER_VMA_LOCK select PPC_RADIX_BROADCAST_TLBIE if PPC_RADIX_MMU default y diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index d6c2dbf8455c..5965666194b0 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -72,7 +72,6 @@ config RISCV select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU - select ARCH_SUPPORTS_PER_VMA_LOCK if MMU select ARCH_HAS_PTE_PROTNONE if MMU select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 4b51bc6e8948..b88b85042136 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -156,7 +156,6 @@ config S390 select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_USES_CFI_GENERIC_LLVM_PASS if CC_IS_CLANG select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 7aa74bcc72f9..a8c3b3d31a27 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -27,7 +27,6 @@ config X86_64 select ARCH_HAS_GIGANTIC_PAGE select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 - select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE select HAVE_ARCH_SOFT_DIRTY select MODULES_USE_ELF_RELA @@ -1848,7 +1847,6 @@ config X86_USER_SHADOW_STACK bool "X86 userspace shadow stack" depends on AS_WRUSS depends on X86_64 - depends on PER_VMA_LOCK select ARCH_USES_HIGH_VMA_FLAGS select ARCH_HAS_USER_SHADOW_STACK select X86_CET diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 04bd6c9e65a7..623bb43ede55 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -385,10 +385,8 @@ struct mem_size_stats; struct proc_maps_locking_ctx { struct mm_struct *mm; -#ifdef CONFIG_PER_VMA_LOCK bool mmap_locked; struct vm_area_struct *locked_vma; -#endif }; struct proc_maps_private { diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 5c54aebe2118..e671b4fd8ded 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -130,8 +130,6 @@ static void release_task_mempolicy(struct proc_maps_private *priv) } #endif -#ifdef CONFIG_PER_VMA_LOCK - static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) { int ret = mmap_read_lock_killable(lock_ctx->mm); @@ -233,46 +231,6 @@ static inline void reacquire_rcu(struct proc_maps_private *priv) vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end); } -#else /* CONFIG_PER_VMA_LOCK */ - -static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) -{ - return mmap_read_lock_killable(lock_ctx->mm); -} - -static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) -{ - mmap_read_unlock(lock_ctx->mm); -} - -static inline bool lock_vma_range(struct seq_file *m, - struct proc_maps_locking_ctx *lock_ctx) -{ - return lock_ctx_mm(lock_ctx) == 0; -} - -static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) -{ - unlock_ctx_mm(lock_ctx); -} - -static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, - loff_t last_pos) -{ - return vma_next(&priv->iter); -} - -static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, - loff_t pos) -{ - return false; -} - -static inline void drop_rcu(struct proc_maps_private *priv) {} -static inline void reacquire_rcu(struct proc_maps_private *priv) {} - -#endif /* CONFIG_PER_VMA_LOCK */ - static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) { struct proc_maps_private *priv = m->private; @@ -560,8 +518,6 @@ static int pid_maps_open(struct inode *inode, struct file *file) PROCMAP_QUERY_VMA_FLAGS \ ) -#ifdef CONFIG_PER_VMA_LOCK - static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) { reset_lock_ctx(lock_ctx); @@ -612,26 +568,6 @@ static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ct return vma; } -#else /* CONFIG_PER_VMA_LOCK */ - -static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) -{ - return mmap_read_lock_killable(lock_ctx->mm); -} - -static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) -{ - mmap_read_unlock(lock_ctx->mm); -} - -static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, - unsigned long addr) -{ - return find_vma(lock_ctx->mm, addr); -} - -#endif /* CONFIG_PER_VMA_LOCK */ - static struct vm_area_struct *query_matching_vma(struct proc_maps_locking_ctx *lock_ctx, unsigned long addr, u32 flags) { @@ -1314,8 +1250,6 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; -#ifdef CONFIG_PER_VMA_LOCK - static const struct mm_walk_ops smaps_walk_vma_lock_ops = { .pmd_entry = smaps_pte_range, .hugetlb_entry = smaps_hugetlb_range, @@ -1345,22 +1279,6 @@ get_smaps_shmem_walk_ops(struct proc_maps_private *priv) return &smaps_shmem_walk_vma_lock_ops; } -#else /* CONFIG_PER_VMA_LOCK */ - -static inline const struct mm_walk_ops * -get_smaps_walk_ops(struct proc_maps_private *priv) -{ - return &smaps_walk_ops; -} - -static inline const struct mm_walk_ops * -get_smaps_shmem_walk_ops(struct proc_maps_private *priv) -{ - return &smaps_shmem_walk_ops; -} - -#endif /* CONFIG_PER_VMA_LOCK */ - /* * Gather mem stats from @vma with the indicated beginning * address @start, and keep them in @mss. @@ -3497,7 +3415,6 @@ static const struct mm_walk_ops show_numa_ops = { .walk_lock = PGWALK_RDLOCK, }; -#ifdef CONFIG_PER_VMA_LOCK static const struct mm_walk_ops show_numa_vma_lock_ops = { .hugetlb_entry = gather_hugetlb_stats, .pmd_entry = gather_pte_stats, @@ -3512,16 +3429,6 @@ get_show_numa_ops(struct proc_maps_private *priv) return &show_numa_vma_lock_ops; } -#else /* CONFIG_PER_VMA_LOCK */ - -static inline const struct mm_walk_ops * -get_show_numa_ops(struct proc_maps_private *priv) -{ - return &show_numa_ops; -} - -#endif /* CONFIG_PER_VMA_LOCK */ - /* * Display pages allocated per node and memory policy via /proc. */ diff --git a/include/linux/mm.h b/include/linux/mm.h index b19711b6dbc6..a9fbe26536f4 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -928,7 +928,6 @@ static inline void vma_numab_state_free(struct vm_area_struct *vma) {} * These must be here rather than mmap_lock.h as dependent on vm_fault type, * declared in this header. */ -#ifdef CONFIG_PER_VMA_LOCK static inline void release_fault_lock(struct vm_fault *vmf) { if (vmf->flags & FAULT_FLAG_VMA_LOCK) @@ -944,17 +943,6 @@ static inline void assert_fault_locked(const struct vm_fault *vmf) else mmap_assert_locked(vmf->vma->vm_mm); } -#else -static inline void release_fault_lock(struct vm_fault *vmf) -{ - mmap_read_unlock(vmf->vma->vm_mm); -} - -static inline void assert_fault_locked(const struct vm_fault *vmf) -{ - mmap_assert_locked(vmf->vma->vm_mm); -} -#endif /* CONFIG_PER_VMA_LOCK */ static inline bool mm_flags_test(int flag, const struct mm_struct *mm) { diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index f3e5a2fadbe5..2a3988178adf 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -950,7 +950,6 @@ struct vm_area_struct { vma_flags_t flags; }; -#ifdef CONFIG_PER_VMA_LOCK /* * Can only be written (using WRITE_ONCE()) while holding both: * - mmap_lock (in write mode) @@ -966,7 +965,7 @@ struct vm_area_struct { * slowpath. */ unsigned int vm_lock_seq; -#endif + /* * Low 32-bits of anonymous page offset. * See vma_start_anon_pgoff() comment for details. @@ -1003,7 +1002,6 @@ struct vm_area_struct { #ifdef CONFIG_NUMA_BALANCING struct vma_numab_state *numab_state; /* NUMA Balancing state */ #endif -#ifdef CONFIG_PER_VMA_LOCK /* * Used to keep track of firstly, whether the VMA is attached, secondly, * if attached, how many read locks are taken, and thirdly, if the @@ -1046,7 +1044,6 @@ struct vm_area_struct { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map vmlock_dep_map; #endif -#endif #ifdef CONFIG_64BIT /* * High 32-bits of anonymous page offset. @@ -1254,7 +1251,6 @@ struct mm_struct { * init_mm.mmlist, and are protected * by mmlist_lock */ -#ifdef CONFIG_PER_VMA_LOCK struct rcuwait vma_writer_wait; /* * This field has lock-like semantics, meaning it is sometimes @@ -1274,7 +1270,7 @@ struct mm_struct { * mmap_lock. */ seqcount_t mm_lock_seq; -#endif + struct futex_mm_data futex; unsigned long hiwater_rss; /* High-watermark of RSS usage */ diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index b8a13b8d36a4..6a0a8cf501bd 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -76,8 +76,6 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) rwsem_assert_held_write(&mm->mmap_lock); } -#ifdef CONFIG_PER_VMA_LOCK - #ifdef CONFIG_LOCKDEP #define __vma_lockdep_map(vma) (&vma->vmlock_dep_map) #else @@ -297,6 +295,9 @@ int __vma_start_write(struct vm_area_struct *vma, int state); */ static inline void vma_start_write(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + if (__is_vma_write_locked(vma)) return; @@ -319,6 +320,9 @@ static inline void vma_start_write(struct vm_area_struct *vma) static inline __must_check int vma_start_write_killable(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return 0; + if (__is_vma_write_locked(vma)) return 0; @@ -331,6 +335,11 @@ int vma_start_write_killable(struct vm_area_struct *vma) */ static inline void vma_assert_write_locked(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) { + mmap_assert_write_locked(vma->vm_mm); + return; + } + VM_WARN_ON_ONCE_VMA(!__is_vma_write_locked(vma), vma); } @@ -343,6 +352,11 @@ static inline void vma_assert_locked(struct vm_area_struct *vma) { unsigned int refcnt; + if (!IS_ENABLED(CONFIG_MMU)) { + mmap_assert_locked(vma->vm_mm); + return; + } + if (IS_ENABLED(CONFIG_LOCKDEP)) { if (!lock_is_held(__vma_lockdep_map(vma))) vma_assert_write_locked(vma); @@ -432,6 +446,9 @@ static inline bool vma_is_attached(struct vm_area_struct *vma) */ static inline void vma_assert_attached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + WARN_ON_ONCE(!vma_is_attached(vma)); } @@ -442,6 +459,9 @@ static inline void vma_assert_detached(struct vm_area_struct *vma) static inline void vma_mark_attached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + vma_assert_write_locked(vma); vma_assert_detached(vma); refcount_set_release(&vma->vm_refcnt, 1); @@ -451,6 +471,9 @@ void __vma_exclude_readers_for_detach(struct vm_area_struct *vma); static inline void vma_mark_detached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + vma_assert_write_locked(vma); vma_assert_attached(vma); @@ -484,54 +507,6 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm, struct vma_iterator *iter, unsigned long address); -#else /* CONFIG_PER_VMA_LOCK */ - -static inline void mm_lock_seqcount_init(struct mm_struct *mm) {} -static inline void mm_lock_seqcount_begin(struct mm_struct *mm) {} -static inline void mm_lock_seqcount_end(struct mm_struct *mm) {} - -static inline bool mmap_lock_speculate_try_begin(struct mm_struct *mm, unsigned int *seq) -{ - return false; -} - -static inline bool mmap_lock_speculate_retry(struct mm_struct *mm, unsigned int seq) -{ - return true; -} -static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) {} -static inline void vma_end_read(struct vm_area_struct *vma) {} -static inline void vma_start_write(struct vm_area_struct *vma) {} -static inline __must_check -int vma_start_write_killable(struct vm_area_struct *vma) { return 0; } -static inline void vma_assert_write_locked(struct vm_area_struct *vma) - { mmap_assert_write_locked(vma->vm_mm); } -static inline bool vma_is_attached(struct vm_area_struct *vma) - { return true; } -static inline void vma_assert_attached(struct vm_area_struct *vma) {} -static inline void vma_assert_detached(struct vm_area_struct *vma) {} -static inline void vma_mark_attached(struct vm_area_struct *vma) {} -static inline void vma_mark_detached(struct vm_area_struct *vma) {} - -static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, - unsigned long address) -{ - return NULL; -} - -static inline void vma_assert_locked(struct vm_area_struct *vma) -{ - mmap_assert_locked(vma->vm_mm); -} - -static inline void vma_assert_stabilised(struct vm_area_struct *vma) -{ - /* If no VMA locks, then either mmap lock suffices to stabilise. */ - mmap_assert_locked(vma->vm_mm); -} - -#endif /* CONFIG_PER_VMA_LOCK */ - static inline void vma_assert_can_modify(struct vm_area_struct *vma) { if (vma_is_attached(vma)) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index d09d4c3fe547..f7e8d766d128 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -272,13 +272,10 @@ struct stack_map_vma_lock { /* * Acquire a stable read-side reference on the VMA covering @ip. * - * With CONFIG_PER_VMA_LOCK=y this returns a VMA with its per-VMA read - * lock held and mmap_lock dropped, so the caller may sleep. - * - * With CONFIG_PER_VMA_LOCK=n it returns a VMA with mmap_lock still - * held; the caller must snapshot any fields it needs and pin vm_file - * with get_file() before stack_map_unlock_vma() drops mmap_lock, as - * the VMA may be split, merged, or freed after that. + * On NOMMU configurations, returns with the mmap_lock held. If the MMU + * is enabled, the per-VMA lock will be held instead. The lock + * should be released with stack_map_unlock_vma() which will release the + * appropriate lock. Once the lock is released, the VMA may be freed. * * Returns NULL on failure, in which case no lock is held. */ @@ -288,7 +285,6 @@ stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip) struct mm_struct *mm = lock->mm; struct vm_area_struct *vma; - /* noop under !CONFIG_PER_VMA_LOCK */ vma = lock_vma_under_rcu(mm, ip); if (vma) { lock->vma = vma; @@ -308,21 +304,20 @@ stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip) return NULL; } -#ifdef CONFIG_PER_VMA_LOCK +#ifdef CONFIG_MMU if (!vma_start_read_locked(vma)) { mmap_read_unlock(mm); return NULL; } mmap_read_unlock(mm); #endif - lock->vma = vma; return vma; } static void stack_map_unlock_vma(struct stack_map_vma_lock *lock) { -#ifdef CONFIG_PER_VMA_LOCK +#ifdef CONFIG_MMU vma_end_read(lock->vma); #else mmap_read_unlock(lock->mm); diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index 13e1aabe6f88..c65ba1dcd866 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -869,7 +869,7 @@ __bpf_kfunc int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it, BUILD_BUG_ON(sizeof(struct bpf_iter_task_vma_kern) != sizeof(struct bpf_iter_task_vma)); BUILD_BUG_ON(__alignof__(struct bpf_iter_task_vma_kern) != __alignof__(struct bpf_iter_task_vma)); - if (!IS_ENABLED(CONFIG_PER_VMA_LOCK)) { + if (!IS_ENABLED(CONFIG_MMU)) { kit->data = NULL; return -EOPNOTSUPP; } diff --git a/kernel/fork.c b/kernel/fork.c index 10f2d05d816a..22eaf5fb844d 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1083,9 +1083,7 @@ static void mmap_init_lock(struct mm_struct *mm) { init_rwsem(&mm->mmap_lock); mm_lock_seqcount_init(mm); -#ifdef CONFIG_PER_VMA_LOCK rcuwait_init(&mm->vma_writer_wait); -#endif } static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) diff --git a/mm/Kconfig b/mm/Kconfig index 2c385f8b2944..c1ddf59c0d71 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1425,18 +1425,6 @@ config LRU_GEN_WALKS_MMU depends on LRU_GEN && ARCH_HAS_HW_PTE_YOUNG # } -config ARCH_SUPPORTS_PER_VMA_LOCK - def_bool n - -config PER_VMA_LOCK - def_bool y - depends on ARCH_SUPPORTS_PER_VMA_LOCK && MMU && SMP - help - Allow per-vma locking during page fault handling. - - This feature allows locking each virtual memory area separately when - handling page faults instead of taking mmap_lock. - config LOCK_MM_AND_FIND_VMA bool depends on !STACK_GROWSUP diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 15dca19dd07d..9eaa25d1cf23 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -310,7 +310,6 @@ config DEBUG_KMEMLEAK_VERBOSE config PER_VMA_LOCK_STATS bool "Statistics for per-vma locks" - depends on PER_VMA_LOCK help Say Y here to enable success, retry and failure counters of page faults handled under protection of per-vma locks. When enabled, the diff --git a/mm/debug.c b/mm/debug.c index 9a0297b3988d..655e6bcc0e8d 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -157,17 +157,13 @@ void dump_vma(const struct vm_area_struct *vma) pr_emerg("vma %px start %px end %px mm %px\n" "prot %lx anon_vma %px vm_ops %px\n" "pgoff %lx file %px private_data %px\n" -#ifdef CONFIG_PER_VMA_LOCK "refcnt %x\n" -#endif "flags: %#lx(%pGv)\n", vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, (unsigned long)pgprot_val(vma->vm_page_prot), vma->anon_vma, vma->vm_ops, vma_start_pgoff(vma), vma->vm_file, vma->vm_private_data, -#ifdef CONFIG_PER_VMA_LOCK refcount_read(&vma->vm_refcnt), -#endif vma->vm_flags, &vma->vm_flags); } EXPORT_SYMBOL(dump_vma); diff --git a/mm/init-mm.c b/mm/init-mm.c index 3e792aad7626..a1bb2c2d0284 100644 --- a/mm/init-mm.c +++ b/mm/init-mm.c @@ -39,10 +39,8 @@ struct mm_struct init_mm = { .page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock), .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist = LIST_HEAD_INIT(init_mm.mmlist), -#ifdef CONFIG_PER_VMA_LOCK .vma_writer_wait = __RCUWAIT_INITIALIZER(init_mm.vma_writer_wait), .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), -#endif #ifdef CONFIG_SCHED_MM_CID .mm_cid.lock = __RAW_SPIN_LOCK_UNLOCKED(init_mm.mm_cid.lock), #endif diff --git a/mm/memory.c b/mm/memory.c index 09ac784f8b7b..bc14cae3c49d 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6799,7 +6799,6 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, !vma_is_cow_mapping(vma))) return VM_FAULT_SIGSEGV; } -#ifdef CONFIG_PER_VMA_LOCK /* * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of * the assumption that lock is dropped on VM_FAULT_RETRY. @@ -6808,7 +6807,6 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) == (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT))) return VM_FAULT_SIGSEGV; -#endif return 0; } diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c index 898c2ef1e958..272f9ac762b9 100644 --- a/mm/mmap_lock.c +++ b/mm/mmap_lock.c @@ -43,9 +43,6 @@ void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write) EXPORT_SYMBOL(__mmap_lock_do_trace_released); #endif /* CONFIG_TRACING */ -#ifdef CONFIG_MMU -#ifdef CONFIG_PER_VMA_LOCK - /* State shared across __vma_[start, end]_exclude_readers. */ struct vma_exclude_readers_state { /* Input parameters. */ @@ -299,6 +296,8 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, MA_STATE(mas, &mm->mm_mt, address, address); struct vm_area_struct *vma; + if (!IS_ENABLED(CONFIG_MMU)) + return NULL; retry: rcu_read_lock(); vma = mas_walk(&mas); @@ -431,7 +430,6 @@ fallback: return vma; } -#endif /* CONFIG_PER_VMA_LOCK */ #ifdef CONFIG_LOCK_MM_AND_FIND_VMA #include @@ -548,23 +546,3 @@ fail: return NULL; } #endif /* CONFIG_LOCK_MM_AND_FIND_VMA */ - -#else /* CONFIG_MMU */ - -/* - * At least xtensa ends up having protection faults even with no - * MMU.. No stack expansion, at least. - */ -struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, - unsigned long addr, struct pt_regs *regs) -{ - struct vm_area_struct *vma; - - mmap_read_lock(mm); - vma = vma_lookup(mm, addr); - if (!vma) - mmap_read_unlock(mm); - return vma; -} - -#endif /* CONFIG_MMU */ diff --git a/mm/pagewalk.c b/mm/pagewalk.c index cc07fcf50e87..7411702a37f5 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -444,7 +444,6 @@ static inline void process_mm_walk_lock(struct mm_struct *mm, static inline void process_vma_walk_lock(struct vm_area_struct *vma, enum page_walk_lock walk_lock) { -#ifdef CONFIG_PER_VMA_LOCK switch (walk_lock) { case PGWALK_WRLOCK: vma_start_write(vma); @@ -459,7 +458,6 @@ static inline void process_vma_walk_lock(struct vm_area_struct *vma, /* PGWALK_RDLOCK is handled by process_mm_walk_lock */ break; } -#endif } /* diff --git a/mm/rmap.c b/mm/rmap.c index f3b21aaa34ee..3c67ad0e9562 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -264,11 +264,9 @@ static void check_anon_vma_clone(struct vm_area_struct *dst, /* For the anon_vma to be compatible, it can only be singular. */ VM_WARN_ON_ONCE(operation == VMA_OP_MERGE_UNFAULTED && !list_is_singular(&src->anon_vma_chain)); -#ifdef CONFIG_PER_VMA_LOCK /* Only merging an unfaulted VMA leaves the destination attached. */ VM_WARN_ON_ONCE(operation != VMA_OP_MERGE_UNFAULTED && vma_is_attached(dst)); -#endif } static void maybe_reuse_anon_vma(struct vm_area_struct *dst, diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 95c1ed96df9a..4ccf472b4163 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -122,7 +122,6 @@ struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, return vma; } -#ifdef CONFIG_PER_VMA_LOCK /* * uffd_lock_vma() - Lookup and lock vma corresponding to @address. * @mm: mm to search vma in. @@ -182,34 +181,6 @@ static void uffd_mfill_unlock(struct vm_area_struct *vma) vma_end_read(vma); } -#else - -static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, - unsigned long dst_start, - unsigned long len) -{ - struct vm_area_struct *dst_vma; - - mmap_read_lock(dst_mm); - dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start); - if (IS_ERR(dst_vma)) - goto out_unlock; - - if (validate_dst_vma(dst_vma, dst_start + len)) - return dst_vma; - - dst_vma = ERR_PTR(-ENOENT); -out_unlock: - mmap_read_unlock(dst_mm); - return dst_vma; -} - -static void uffd_mfill_unlock(struct vm_area_struct *vma) -{ - mmap_read_unlock(vma->vm_mm); -} -#endif - static void mfill_put_vma(struct mfill_state *state) { if (!state->vma) @@ -1850,7 +1821,6 @@ out_success: return 0; } -#ifdef CONFIG_PER_VMA_LOCK static int uffd_move_lock(struct mm_struct *mm, unsigned long dst_start, unsigned long src_start, @@ -1925,31 +1895,6 @@ static void uffd_move_unlock(struct vm_area_struct *dst_vma, vma_end_read(dst_vma); } -#else - -static int uffd_move_lock(struct mm_struct *mm, - unsigned long dst_start, - unsigned long src_start, - struct vm_area_struct **dst_vmap, - struct vm_area_struct **src_vmap) -{ - int err; - - mmap_read_lock(mm); - err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); - if (err) - mmap_read_unlock(mm); - return err; -} - -static void uffd_move_unlock(struct vm_area_struct *dst_vma, - struct vm_area_struct *src_vma) -{ - mmap_assert_locked(src_vma->vm_mm); - mmap_read_unlock(dst_vma->vm_mm); -} -#endif - /** * move_pages - move arbitrary anonymous pages of an existing vma * @ctx: pointer to the userfaultfd context diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index 4764d7b68f2a..f4fa54616085 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -170,30 +170,20 @@ impl MmWithUser { /// /// This is an optimistic trylock operation, so it may fail if there is contention. In that /// case, you should fall back to taking the mmap read lock. - /// - /// When per-vma locks are disabled, this always returns `None`. #[inline] pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option> { - #[cfg(CONFIG_PER_VMA_LOCK)] - { - // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where - // `mm_users` is non-zero. - let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; - if !vma.is_null() { - return Some(VmaReadGuard { - // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a - // valid vma. The vma is stable for as long as the vma read lock is held. - vma: unsafe { VmaRef::from_raw(vma) }, - _nts: NotThreadSafe, - }); - } + // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where + // `mm_users` is non-zero. + let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; + if vma.is_null() { + return None; } - - // Silence warnings about unused variables. - #[cfg(not(CONFIG_PER_VMA_LOCK))] - let _ = vma_addr; - - None + Some(VmaReadGuard { + // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }) } /// Lock the mmap read lock. diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 4c58487b764e..57046d8ac81d 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -560,7 +560,6 @@ struct vm_area_struct { vma_flags_t flags; }; -#ifdef CONFIG_PER_VMA_LOCK /* * Can only be written (using WRITE_ONCE()) while holding both: * - mmap_lock (in write mode) @@ -576,7 +575,7 @@ struct vm_area_struct { * slowpath. */ unsigned int vm_lock_seq; -#endif + unsigned int __vm_anon_pgoff_lo; /* @@ -610,10 +609,8 @@ struct vm_area_struct { #ifdef CONFIG_NUMA_BALANCING struct vma_numab_state *numab_state; /* NUMA Balancing state */ #endif -#ifdef CONFIG_PER_VMA_LOCK /* Unstable RCU readers are allowed to read this. */ refcount_t vm_refcnt; -#endif #ifdef CONFIG_64BIT unsigned int __vm_anon_pgoff_hi; #endif diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h index 8a48b231aa7a..54d5c3360aa2 100644 --- a/tools/testing/vma/vma_internal.h +++ b/tools/testing/vma/vma_internal.h @@ -15,7 +15,6 @@ #include #define CONFIG_MMU 1 -#define CONFIG_PER_VMA_LOCK 1 #ifdef __CONCAT #undef __CONCAT From 5542d17a216dfc390b0349d8e43b30f337bbbc01 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 31 Aug 2026 13:30:53 -0700 Subject: [PATCH 105/539] binder: make shrinker rely solely on per-VMA lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tl;dr: lock_vma_under_rcu() is already a trylock. No need to do both it and mmap_read_trylock(). Long Version: == Background == Historically, binder used an mmap_read_trylock() in its shrinker code. This ensures that reclaim is not blocked on an mmap_lock. Commit 95bc2d4a9020 ("binder: use per-vma lock in page reclaiming") added support for the per-VMA lock, but left mmap_read_trylock() as a fallback. This was presumably because the per-VMA locking can fail for several reasons and most (all?) lock_vma_under_rcu() callers have a fallback to mmap_read_trylock(). == Problem == The fallback is not worth the complexity here. lock_vma_under_rcu() is essentially already a non-blocking trylock. The main reason it fails is also the reason mmap_read_trylock() fails: something is holding mmap_write_lock(). The only remedy for a collision with mmap_write_lock() is to wait, which this code can not do. So the "fallback" after lock_vma_under_rcu() failure is not really a fallback: it is really likely to just be retrying in vain. That retry in an of itself isn't horrible. But it adds complexity. == Solution == Now that per-VMA locks are universally available, lock_vma_under_rcu() will not persistently fail. Rely on it alone and simplify the code. The removal of the fallback does not affect NOMMU case because binder driver depends on CONFIG_MMU. While at it we also make the handling of the cases where the original binder VMA is gone consistent. There are two cases to consider when Binder VMA is gone: 1. there is no VMA at that location anymore. 2. there is now another unrelated VMA at that location. Before this change we handle case 1 by having the shrinker proceed to free the page, and just skip the zap_vma_range() call. And we handle case 2 by having the shrinker return LRU_SKIP. While either behavior is acceptable, we need to handle them in a consistent way. Handle both cases by freeing the page without touching the VMA (skipping the zap_vma_range()). Full disclosure: I originally tried to do this with lock_vma_under_rcu_wait(), but it did not fit well with the mmap_lock trylock semantics. Claude caught this in a review and suggested the approach in this path. It seemed sane to me. So, Suggesed-by: Claude, I guess. Link: https://lore.kernel.org/20260831203056.838265-3-surenb@google.com Signed-off-by: Dave Hansen Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Alice Ryhl Acked-by: Lorenzo Stoakes (ARM) Acked-by: Carlos Llamas Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Shakeel Butt Cc: Greg Kroah-Hartman Cc: Todd Kjos Cc: Christian Brauner Cc: David S. Miller Cc: David Ahern Cc: Arve Hjønnevåg Cc: David Hildenbrand (Arm) --- drivers/android/binder_alloc.c | 46 ++++++++++++++++------------------ 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index e4488ad86a65..fcb744088e77 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -1142,7 +1142,6 @@ enum lru_status binder_alloc_free_page(struct list_head *item, struct vm_area_struct *vma; struct page *page_to_free; unsigned long page_addr; - int mm_locked = 0; size_t index; if (!mmget_not_zero(mm)) @@ -1151,27 +1150,25 @@ enum lru_status binder_alloc_free_page(struct list_head *item, index = mdata->page_index; page_addr = alloc->vm_start + index * PAGE_SIZE; - /* attempt per-vma lock first */ + /* + * Attempt per-vma lock. This is essentially a + * "trylock". It can fail even if the VMA exists + * for 'page_addr'. + */ vma = lock_vma_under_rcu(mm, page_addr); if (!vma) { - /* fall back to mmap_lock */ - if (!mmap_read_trylock(mm)) - goto err_mmap_read_lock_failed; - mm_locked = 1; - vma = vma_lookup(mm, page_addr); + /* + * If the vma exists, we can't continue because we cannot + * remove the page from the vma. However, if the vma was + * unmapped, it's okay to continue. + */ + if (binder_alloc_is_mapped(alloc)) + goto err_vma_lock_failed; } if (!mutex_trylock(&alloc->mutex)) goto err_get_alloc_mutex_failed; - /* - * Since a binder_alloc can only be mapped once, we ensure - * the vma corresponds to this mapping by checking whether - * the binder_alloc is still mapped. - */ - if (vma && !binder_alloc_is_mapped(alloc)) - goto err_invalid_vma; - trace_binder_unmap_kernel_start(alloc, index); page_to_free = alloc->pages[index]; @@ -1182,7 +1179,12 @@ enum lru_status binder_alloc_free_page(struct list_head *item, list_lru_isolate(lru, item); spin_unlock(&lru->lock); - if (vma) { + /* + * Since a binder_alloc can only be mapped once, we ensure + * the vma corresponds to this mapping by checking whether + * the binder_alloc is still mapped. + */ + if (vma && binder_alloc_is_mapped(alloc)) { trace_binder_unmap_user_start(alloc, index); zap_vma_range(vma, page_addr, PAGE_SIZE); @@ -1191,23 +1193,17 @@ enum lru_status binder_alloc_free_page(struct list_head *item, } mutex_unlock(&alloc->mutex); - if (mm_locked) - mmap_read_unlock(mm); - else + if (vma) vma_end_read(vma); mmput_async(mm); binder_free_page(page_to_free); return LRU_REMOVED_RETRY; -err_invalid_vma: - mutex_unlock(&alloc->mutex); err_get_alloc_mutex_failed: - if (mm_locked) - mmap_read_unlock(mm); - else + if (vma) vma_end_read(vma); -err_mmap_read_lock_failed: +err_vma_lock_failed: mmput_async(mm); err_mmget: return LRU_SKIP; From d96082aa35043cd1856e2213e683db778482a7d8 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 31 Aug 2026 13:30:54 -0700 Subject: [PATCH 106/539] mm: add RCU-based VMA lookup helper that waits for writers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are basically two parallel ways to look up a VMA: the traditional way, which is protected by mmap_read_lock, and the RCU-based per-VMA lock way which is based on RCU and refcounts. However, per-VMA locks will fail if the lock is help by a writer and therefore never waits. In a number of places we need to wait for the lock and it's done by falling back to mmap_read_lock, locking the VMA and releasing the mmap_lock once VMA is locked. Add vma_start_read_unlocked() - a variant of the RCU-based lookup that waits for writers. This is basically the same as the existing RCU-based lookup, but on a failure to lock it temporarily takes mmap_lock for read and waits for writers to finish before locking the VMA, dropping the mmap_read_lock and returning the locked VMA. This has some advantages: 1. Callers do not need to have a fallback path for when they collide with writers. 2. Its fast path does not require taking mmap_lock for read. Basically, when applied correctly, this approach results in faster *and* simpler code. While at it, fix the comments for vma_start_read_locked(), vma_start_read_locked_nested(), and uffd_lock_vma(). Link: https://lore.kernel.org/20260831203056.838265-4-surenb@google.com Signed-off-by: Dave Hansen Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Suggested-by: Lorenzo Stoakes (ARM) Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: Vlastimil Babka (SUSE) Cc: Liam R. Howlett Cc: Shakeel Butt Cc: Greg Kroah-Hartman Cc: Todd Kjos Cc: Christian Brauner Cc: Carlos Llamas Cc: Alice Ryhl Cc: David S. Miller Cc: David Ahern Cc: Arve Hjønnevåg Cc: David Hildenbrand (Arm) --- include/linux/mmap_lock.h | 19 +++++++++++++++---- mm/mmap_lock.c | 35 +++++++++++++++++++++++++++++++++++ mm/userfaultfd.c | 6 ++++-- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 6a0a8cf501bd..28e3696ce9ff 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -228,10 +228,14 @@ static inline void vma_refcount_put(struct vm_area_struct *vma) } /* - * Use only while holding mmap read lock which guarantees that locking will not - * fail (nobody can concurrently write-lock the vma). vma_start_read() should + * Use only while holding mmap read lock which guarantees that vma lock is not + * contended (nobody can concurrently write-lock the vma). vma_start_read() should * not be used in such cases because it might fail due to mm_lock_seq overflow. * This functionality is used to obtain vma read lock and drop the mmap read lock. + * + * VMA can't be detached while we are holding mmap lock, therefore in practice this + * function can fail only when there are so many readers that vm_refcnt overflows. + * The failure case is very unlikely and is already annotated as such internally. */ static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int subclass) { @@ -247,16 +251,23 @@ static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int } /* - * Use only while holding mmap read lock which guarantees that locking will not - * fail (nobody can concurrently write-lock the vma). vma_start_read() should + * Use only while holding mmap read lock which guarantees that vma lock is not + * contended (nobody can concurrently write-lock the vma). vma_start_read() should * not be used in such cases because it might fail due to mm_lock_seq overflow. * This functionality is used to obtain vma read lock and drop the mmap read lock. + * + * VMA can't be detached while we are holding mmap lock, therefore in practice this + * function can fail only when there are so many readers that vm_refcnt overflows. + * The failure case is very unlikely and is already annotated as such internally. */ static inline bool vma_start_read_locked(struct vm_area_struct *vma) { return vma_start_read_locked_nested(vma, 0); } +struct vm_area_struct *vma_start_read_unlocked(struct mm_struct *mm, + unsigned long address); + static inline void vma_end_read(struct vm_area_struct *vma) { vma_refcount_put(vma); diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c index 272f9ac762b9..2f94ee0fdee2 100644 --- a/mm/mmap_lock.c +++ b/mm/mmap_lock.c @@ -340,6 +340,41 @@ inval: return NULL; } +/** + * vma_start_read_unlocked() - Find the VMA covering 'address' and read-lock it. + * @mm: the mm_struct of the address space to search + * @address: address that the vma should contain + * + * The fast path does not take mmap_lock. Waits for writers to finish if the + * VMA is being modified by taking mmap_lock. + * Use when mmap_lock is not held, otherwise use vma_start_read_locked(). + * Nothing prevents VMAs being unmapped/mapped before or after the VMA is + * looked up, if a stronger guarantee is required, take an mmap_lock. + * + * Return: If a VMA exists which spans @address, return that VMA, read-locked. + * If no VMA is mapped there or, very unlikely, a reference count overflow + * occurred, return NULL. + */ +struct vm_area_struct *vma_start_read_unlocked(struct mm_struct *mm, + unsigned long address) +{ + struct vm_area_struct *vma; + + /* Fast path: return stable VMA covering 'address': */ + vma = lock_vma_under_rcu(mm, address); + if (vma) + return vma; + + /* Slow path: preclude VMA writers by temporarily getting mmap read lock. */ + mmap_read_lock(mm); + vma = vma_lookup(mm, address); + if (vma && !vma_start_read_locked(vma)) + vma = NULL; + mmap_read_unlock(mm); + + return vma; +} + static struct vm_area_struct *lock_next_vma_under_mmap_lock(struct mm_struct *mm, struct vma_iterator *vmi, unsigned long from_addr) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 4ccf472b4163..79cc7b546f13 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -129,8 +129,10 @@ struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, * * Should be called without holding mmap_lock. * - * Return: A locked vma containing @address, -ENOENT if no vma is found, or - * -ENOMEM if anon_vma couldn't be allocated. + * Return: A locked vma containing @address, -ENOENT if no vma is found, + * -ENOMEM if anon_vma couldn't be allocated, or -EAGAIN if vma refcount + * overflow happened due to high number of readers and the caller should + * retry later. */ static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm, unsigned long address) From a3bbbd68bf2a4baf972efb734d97c1b241c3881a Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 31 Aug 2026 13:30:55 -0700 Subject: [PATCH 107/539] binder: remove mmap_lock fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, the per-VMA locking could fail in the face of writers which necessitate a fallback to mmap_lock. The new vma_start_read_unlocked() will wait for writers instead of failing. Use the new helper. Wait for writers. Remove the fallback to mmap_lock. Link: https://lore.kernel.org/20260831203056.838265-5-surenb@google.com Signed-off-by: Dave Hansen Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Alice Ryhl Acked-by: Lorenzo Stoakes (ARM) Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Shakeel Butt Cc: Greg Kroah-Hartman Cc: Todd Kjos Cc: Christian Brauner Cc: Carlos Llamas Cc: David S. Miller Cc: David Ahern Cc: Arve Hjønnevåg Cc: David Hildenbrand (Arm) --- drivers/android/binder/page_range.rs | 19 +++---------------- drivers/android/binder_alloc.c | 17 +++++------------ rust/kernel/mm.rs | 28 ++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs index 52ffbf3504e7..71febd3d5b07 100644 --- a/drivers/android/binder/page_range.rs +++ b/drivers/android/binder/page_range.rs @@ -439,22 +439,9 @@ impl ShrinkablePageRange { // workqueue. let mm = MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?); { - let vma_read; - let mmap_read; - let vma = if let Some(ret) = mm.lock_vma_under_rcu(vma_addr) { - vma_read = ret; - check_vma(&vma_read, self) - } else { - mmap_read = mm.mmap_read_lock(); - mmap_read - .vma_lookup(vma_addr) - .and_then(|vma| check_vma(vma, self)) - }; - - match vma { - Some(vma) => vma.vm_insert_page(user_page_addr, &new_page)?, - None => return Err(ESRCH), - } + let vma_read_guard = mm.vma_start_read_unlocked(vma_addr).ok_or(ESRCH)?; + let vma = check_vma(&vma_read_guard, self).ok_or(ESRCH)?; + vma.vm_insert_page(user_page_addr, &new_page)?; } let inner = self.lock.lock(); diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index fcb744088e77..d6eae0aa7085 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -259,21 +259,14 @@ static int binder_page_insert(struct binder_alloc *alloc, struct vm_area_struct *vma; int ret = -ESRCH; - /* attempt per-vma lock first */ - vma = lock_vma_under_rcu(mm, addr); - if (vma) { - if (binder_alloc_is_mapped(alloc)) - ret = vm_insert_page(vma, addr, page); - vma_end_read(vma); + vma = vma_start_read_unlocked(mm, addr); + if (!vma) return ret; - } - /* fall back to mmap_lock */ - mmap_read_lock(mm); - vma = vma_lookup(mm, addr); - if (vma && binder_alloc_is_mapped(alloc)) + if (binder_alloc_is_mapped(alloc)) ret = vm_insert_page(vma, addr, page); - mmap_read_unlock(mm); + + vma_end_read(vma); return ret; } diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index f4fa54616085..58bc1793fdaf 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -186,6 +186,34 @@ impl MmWithUser { }) } + /// Find the VMA covering 'address' and read-lock it. + /// + /// The fast path does not take mmap_lock. Waits for writers to finish if the + /// VMA is being modified by taking mmap_lock. + /// Use when mmap_lock is not held, otherwise use vma_start_read_locked(). + /// Nothing prevents VMAs being unmapped/mapped before or after the VMA is + /// looked up, if a stronger guarantee is required, take an mmap_lock. + /// + /// Return: If a VMA exists which spans @address, return that VMA, read-locked. + /// If no VMA is mapped there or, very unlikely, a reference count overflow + /// occurred, return NULL. + #[inline] + pub fn vma_start_read_unlocked(&self, vma_addr: usize) -> Option> { + // SAFETY: We may invoke `vma_start_read_unlocked` because we know this `mm` has non-zero + // `mm_users`. + let vma = unsafe { bindings::vma_start_read_unlocked(self.as_raw(), vma_addr) }; + if vma.is_null() { + return None; + } + // INVARIANT: We just acquired the VMA read lock. + Some(VmaReadGuard { + // SAFETY: If `vma_start_read_unlocked` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }) + } + /// Lock the mmap read lock. #[inline] pub fn mmap_read_lock(&self) -> MmapReadGuard<'_> { From 1371a2c2720948ea3fee24e6204d4c75bf50d312 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 31 Aug 2026 13:30:56 -0700 Subject: [PATCH 108/539] tcp: remove mmap_lock fallback path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, the per-VMA locking could fail in the face of writers which necessitates a fallback to mmap_lock. The new vma_start_read_unlocked() will wait for writers instead of failing. Use the new helper. Wait for writers. Remove the fallback to mmap_lock. The fallback removal does not affect NOMMU case because TCP_ZEROCOPY is gated on CONFIG_MMU. This really is a nice cleanup. It removes the need to pass the lock state back and forth to find_tcp_vma(). Link: https://lore.kernel.org/20260831203056.838265-6-surenb@google.com Signed-off-by: Dave Hansen Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Acked-by: Lorenzo Stoakes Acked-by: Vlastimil Babka (SUSE) Tested-by: syzbot@syzkaller.appspotmail.com Cc: Liam R. Howlett Cc: Shakeel Butt Cc: Greg Kroah-Hartman Cc: Arve Hjønnevåg Cc: Todd Kjos Cc: Christian Brauner Cc: Carlos Llamas Cc: Alice Ryhl Cc: David S. Miller Cc: David Ahern Cc: David Hildenbrand (Arm) --- net/ipv4/tcp.c | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 562752352afe..5588310bc648 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -2167,27 +2167,18 @@ static void tcp_zc_finalize_rx_tstamp(struct sock *sk, } static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm, - unsigned long address, - bool *mmap_locked) + unsigned long address) { - struct vm_area_struct *vma = lock_vma_under_rcu(mm, address); + struct vm_area_struct *vma = vma_start_read_unlocked(mm, address); - if (vma) { - if (vma->vm_ops != &tcp_vm_ops) { - vma_end_read(vma); - return NULL; - } - *mmap_locked = false; - return vma; - } + if (!vma) + return NULL; - mmap_read_lock(mm); - vma = vma_lookup(mm, address); - if (!vma || vma->vm_ops != &tcp_vm_ops) { - mmap_read_unlock(mm); + if (vma->vm_ops != &tcp_vm_ops) { + vma_end_read(vma); return NULL; } - *mmap_locked = true; + return vma; } @@ -2208,7 +2199,6 @@ static int tcp_zerocopy_receive(struct sock *sk, u32 seq = tp->copied_seq; u32 total_bytes_to_map; int inq = tcp_inq(sk); - bool mmap_locked; int ret; zc->copybuf_len = 0; @@ -2233,7 +2223,7 @@ static int tcp_zerocopy_receive(struct sock *sk, return 0; } - vma = find_tcp_vma(current->mm, address, &mmap_locked); + vma = find_tcp_vma(current->mm, address); if (!vma) return -EINVAL; @@ -2315,10 +2305,7 @@ static int tcp_zerocopy_receive(struct sock *sk, zc, total_bytes_to_map); } out: - if (mmap_locked) - mmap_read_unlock(current->mm); - else - vma_end_read(vma); + vma_end_read(vma); /* Try to copy straggler data. */ if (!ret) copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss); From 374f65b1481992eb6b9c6bffd53f3afa5c2e94cf Mon Sep 17 00:00:00 2001 From: Joe Damato Date: Mon, 31 Aug 2026 10:48:35 -0700 Subject: [PATCH 109/539] mm: memcontrol: raise MEMCG_MAX for charges that fail without reclaiming Charges that exceed memory.max and return through the nomem label can raise no event and simply return -ENOMEM. A non-blocking charge can hit the limit, get rejected, but is not visible in memory.events. This was noticed in a production setting where bpf_mem_alloc() attempted to refill its per-cpu freelists, which triggered a non-blocking charge while at the limit. Commit d6e103a757fa ("mm: memcontrol: do not miss MEMCG_MAX events for enforced allocations") added raised_max_event to cover charges that are force charged without ever reaching reclaim, but charges that are rejected outright were left out. Getting an allocation failure without the corresponding MEMCG_MAX event is unexpected and makes debugging and monitoring harder. Raise the event on the way out for rejected charges as well, by routing the -ENOMEM return through the same exit path that already covers forced charges. The existing behavior of raising a MEMCG_MAX event on every charge/reclaim/retry iteration is left unchanged. Tested with a module that performs accounted GFP_NOWAIT page allocations from a task in a cgroup at its memory.max, and measures the resulting memory.events:max delta. Without this patch the rejected charges raise no event at all; with it the delta matches the number of rejected charges exactly. A GFP_KERNEL|__GFP_NORETRY control, which reaches reclaim, raises the same two events per failed charge before and after, confirming the existing charge/reclaim/retry accounting is unchanged. Link: https://lore.kernel.org/20260831174836.3102406-1-joe@dama.to Fixes: d6e103a757fa ("mm: memcontrol: do not miss MEMCG_MAX events for enforced allocations") Signed-off-by: Joe Damato Signed-off-by: Andrew Morton Suggested-by: Shakeel Butt Acked-by: Shakeel Butt Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: --- mm/memcontrol.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index d5ebe83eae3e..aeaa09e01d70 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2685,10 +2685,11 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, bool raised_max_event = false; unsigned long pflags; bool allow_spinning = gfpflags_allow_spinning(gfp_mask); + int ret = 0; retry: if (consume_stock(memcg, nr_pages)) - return 0; + return ret; if (!allow_spinning) /* Avoid the refill and flush of the older stock */ @@ -2799,16 +2800,11 @@ nomem: * put the burden of reclaim on regular allocation requests * and let these go through as privileged allocations. */ - if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH))) - return -ENOMEM; + if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH))) { + ret = -ENOMEM; + goto out; + } force: - /* - * If the allocation has to be enforced, don't forget to raise - * a MEMCG_MAX event. - */ - if (!raised_max_event) - __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); - /* * The allocation either can't fail or will lead to more memory * being freed very soon. Allow memory usage go over the limit @@ -2818,7 +2814,15 @@ force: if (do_memsw_account()) page_counter_charge(&memcg->memsw, nr_pages); - return 0; +out: + /* + * Don't forget to raise a MEMCG_MAX event for forced or rejected + * requests. + */ + if (!raised_max_event) + __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); + + return ret; done_restock: if (batch > nr_pages) @@ -2877,7 +2881,7 @@ done_restock: !(current->flags & PF_MEMALLOC) && gfpflags_allow_blocking(gfp_mask)) __mem_cgroup_handle_over_high(gfp_mask); - return 0; + return ret; } static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, From 1d62ec1548d1c6ba4fd86c7945f6f4949d316c8f Mon Sep 17 00:00:00 2001 From: Jason Angelov Date: Mon, 31 Aug 2026 08:06:48 -0700 Subject: [PATCH 110/539] mm/damon/core-kunit: test probe_hits handling at region split and merge Patch series "mm/damon: add kunit tests for probe_hits handling and probe params validation", v2. DAMON recently introduced probes and probe weights. Add kunit tests for the propagation of probe_hits at region split and merge, and the rejection of invalid probe parameters by damon_valid_probe_params(). This patch (of 2): damon_split_region_at() copies probe_hits[] and last_probe_hits[] to the new split region. damon_merge_two_regions() sets probe_hits[] to the size-weighted average of the merged regions. Extend damon_test_split_at() and damon_test_merge_two() tests to cover those fields. Link: https://lore.kernel.org/20260831150650.84829-1-sj@kernel.org Link: https://lore.kernel.org/20260831150650.84829-2-sj@kernel.org Signed-off-by: Jason Angelov Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: David Gow Cc: Brendan Higgins --- mm/damon/tests/core-kunit.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 4a536d41cdb2..2db94d49c9ba 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -152,6 +152,8 @@ static void damon_test_split_at(struct kunit *test) } r->nr_accesses = 42; r->last_nr_accesses = 15; + r->probe_hits[0] = 7; + r->last_probe_hits[0] = 3; r->age = 10; damon_add_region(r, t); damon_split_region_at(t, r, 25); @@ -168,6 +170,8 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses); KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); + KUNIT_EXPECT_EQ(test, r->probe_hits[0], r_new->probe_hits[0]); + KUNIT_EXPECT_EQ(test, r->last_probe_hits[0], r_new->last_probe_hits[0]); KUNIT_EXPECT_EQ(test, r->age, r_new->age); out: @@ -189,6 +193,7 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = 10; + r->probe_hits[0] = 6; r->age = 9; damon_add_region(r, t); r2 = damon_new_region(100, 300); @@ -197,6 +202,7 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "second region alloc fail"); } r2->nr_accesses = 20; + r2->probe_hits[0] = 14; r2->age = 21; damon_add_region(r2, t); @@ -204,6 +210,7 @@ static void damon_test_merge_two(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 300ul); KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u); + KUNIT_EXPECT_EQ(test, r->probe_hits[0], 11); KUNIT_EXPECT_EQ(test, r->age, 17u); i = 0; From 83f66bed09f774a9f69c3d46fb724f2264c89206 Mon Sep 17 00:00:00 2001 From: Jason Angelov Date: Mon, 31 Aug 2026 08:06:49 -0700 Subject: [PATCH 111/539] mm/damon/core-kunit: test damon_valid_probe_params() damon_valid_probe_params() makes damon_commit_ctx() reject probe configurations that could overflow a probe_hits counter, a single (weight * probe_hits) product, or the sum of those products. Add a kunit test covering each rejection at its boundary: - samples per aggregation interval: U8_MAX is allowed, one more could overflow a probe_hits counter - single weight: the largest whose product fits in unsigned int is allowed, one larger is rejected - multiple probes: each product fits, but their sum overflows - no weight set: the validation is skipped Link: https://lore.kernel.org/20260831150650.84829-3-sj@kernel.org Signed-off-by: Jason Angelov Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Brendan Higgins Cc: David Gow --- mm/damon/tests/core-kunit.h | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 2db94d49c9ba..b1ca4c8e03f0 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1338,6 +1338,62 @@ static void damon_test_commit_ctx(struct kunit *test) damon_destroy_ctx(dst); } +static void damon_test_valid_probe_params(struct kunit *test) +{ + struct damon_ctx *ctx; + struct damon_probe *probe, *probe2; + + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + probe = damon_new_probe(); + if (!probe) { + damon_destroy_ctx(ctx); + kunit_skip(test, "probe alloc fail"); + } + damon_add_probe(ctx, probe); + + /* Parameters are validated only if any probe weight is set. */ + ctx->attrs.sample_interval = 1; + ctx->attrs.aggr_interval = 1000000; + KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx)); + + /* Up to U8_MAX samples per aggregation interval are allowed. */ + probe->weight = 100; + ctx->attrs.aggr_interval = 255; + KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx)); + + /* More samples could overflow the probe_hits counters. */ + ctx->attrs.aggr_interval = 256; + KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx)); + + /* The largest weight whose weighted hit count fits in unsigned int. */ + ctx->attrs.aggr_interval = 255; + probe->weight = UINT_MAX / 255; + KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx)); + + /* Any larger weight could overflow its weighted hit count. */ + probe->weight = UINT_MAX / 255 + 1; + KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx)); + + /* With one sample per aggregation, even the largest weight fits. */ + ctx->attrs.aggr_interval = 1; + probe->weight = UINT_MAX; + KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx)); + + /* The sum of all probes' weighted hit counts could also overflow. */ + probe2 = damon_new_probe(); + if (!probe2) { + damon_destroy_ctx(ctx); + kunit_skip(test, "probe2 alloc fail"); + } + probe2->weight = 1; + damon_add_probe(ctx, probe2); + KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx)); + + damon_destroy_ctx(ctx); +} + static void damos_test_filter_out(struct kunit *test) { struct damon_target *t; @@ -1664,6 +1720,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damos_test_commit_migrate_hot), KUNIT_CASE(damon_test_commit_target_regions), KUNIT_CASE(damon_test_commit_ctx), + KUNIT_CASE(damon_test_valid_probe_params), KUNIT_CASE(damos_test_filter_out), KUNIT_CASE(damon_test_feed_loop_next_input), KUNIT_CASE(damon_test_set_filters_default_reject), From b9a7f3fe3cc78b4d1a4cf571f2a9737834780784 Mon Sep 17 00:00:00 2001 From: Liew Rui Yan Date: Mon, 31 Aug 2026 08:02:24 -0700 Subject: [PATCH 112/539] docs/mm/damon/design: accurate semantics of nr_snapshots Patch series "docs/mm/damon/design: add explanation of nr_snapshots", v3. Add an explanation of nr_snapshots to avoid misunderstandings. This patch (of 3): Change "tried to be applied" -> "completely tried to be applied" to maintain consistency between the documentation and the code. Link: https://lore.kernel.org/20260831150227.83416-1-sj@kernel.org Link: https://lore.kernel.org/20260831150227.83416-2-sj@kernel.org Signed-off-by: Liew Rui Yan Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 4 ++-- include/linux/damon.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index aed6cb1cf483..1739aeec6eb9 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -846,8 +846,8 @@ scheme's execution. - ``nr_applied``: Total number of regions that the scheme is applied. - ``sz_applied``: Total size of regions that the scheme is applied. - ``qt_exceeds``: Total number of times the quota of the scheme has exceeded. -- ``nr_snapshots``: Total number of DAMON snapshots that the scheme is tried to - be applied. +- ``nr_snapshots``: Total number of DAMON snapshots that the scheme is + completely tried to be applied. - ``max_nr_snapshots``: Upper limit of ``nr_snapshots``. "A scheme is tried to be applied to a region" means DAMOS core logic determined diff --git a/include/linux/damon.h b/include/linux/damon.h index 0c8b7ddef9ab..cbdf5f77978e 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -357,7 +357,8 @@ struct damos_watermarks { * Total bytes that passed ops layer-handled DAMOS filters. * @qt_exceeds: Total number of times the quota of the scheme has exceeded. * @nr_snapshots: - * Total number of DAMON snapshots that the scheme has tried. + * Total number of DAMON snapshots that the scheme is completely + * tried to be applied. * * "Tried an action to a region" in this context means the DAMOS core logic * determined the region as eligible to apply the action. The access pattern From 40b16b5d26ecf2578d8c513b82bf53162381be94 Mon Sep 17 00:00:00 2001 From: Liew Rui Yan Date: Mon, 31 Aug 2026 08:02:25 -0700 Subject: [PATCH 113/539] docs/mm/damon/design: difference between watermarks and nr_snapshots Explain the difference between nr_snapshots reaches max_nr_snapshots and watermarks. Link: https://lore.kernel.org/20260831150227.83416-3-sj@kernel.org Signed-off-by: Liew Rui Yan Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 1739aeec6eb9..e7977f005ac0 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -872,7 +872,8 @@ the action to the region will fail. Unlike normal stats, ``max_nr_snapshots`` is set by users. If it is set as non-zero and ``nr_snapshots`` be same to or greater than ``nr_snapshots``, the -scheme is deactivated. +scheme is deactivated. Note that, unlike watermarks, even if a scheme's +``nr_snapshots`` reaches ``max_nr_snapshots``, monitoring will not stop. To know how user-space can read the stats via :ref:`DAMON sysfs interface `, refer to :ref:s`stats ` part of the From d308767257f5fbd4591908322a7e449850f80b62 Mon Sep 17 00:00:00 2001 From: Liew Rui Yan Date: Mon, 31 Aug 2026 08:02:26 -0700 Subject: [PATCH 114/539] docs/mm/damon/design: fix typo of max_nr_snapshots Fix a typo (nr_snapshots -> max_nr_snapshots) and corrects a grammar error. Link: https://lore.kernel.org/20260831150227.83416-4-sj@kernel.org Signed-off-by: Liew Rui Yan Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 4 ++-- include/linux/damon.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index e7977f005ac0..d1dd9050ebf4 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -871,8 +871,8 @@ action is ``pageout`` while all pages of the region are unreclaimable, applying the action to the region will fail. Unlike normal stats, ``max_nr_snapshots`` is set by users. If it is set as -non-zero and ``nr_snapshots`` be same to or greater than ``nr_snapshots``, the -scheme is deactivated. Note that, unlike watermarks, even if a scheme's +non-zero and ``nr_snapshots`` equals or is greater than ``max_nr_snapshots``, +the scheme is deactivated. Note that, unlike watermarks, even if a scheme's ``nr_snapshots`` reaches ``max_nr_snapshots``, monitoring will not stop. To know how user-space can read the stats via :ref:`DAMON sysfs interface diff --git a/include/linux/damon.h b/include/linux/damon.h index cbdf5f77978e..4b0d2d2e4ea4 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -549,7 +549,7 @@ struct damos_migrate_dests { * * After applying the &action to each region, &stat is updated. * - * If &max_nr_snapshots is set as non-zero and &stat.nr_snapshots be same to or + * If &max_nr_snapshots is set as non-zero and &stat.nr_snapshots equals or is * greater than it, the scheme is deactivated. */ struct damos { From 38ad91e57a5f68a57a3fa23fd49f47e128465e0e Mon Sep 17 00:00:00 2001 From: Cheng-Han Wu Date: Mon, 31 Aug 2026 07:57:21 -0700 Subject: [PATCH 115/539] mm/damon/core: remove unused damon_targets_empty() Patch series "mm/damon/core: remove unused helper functions", v2. Both damon_targets_empty() and damon_nr_running_ctxs() have had no in-tree users since commit 5ec4333b1967 ("mm/damon: remove DAMON debugfs interface") removed their remaining callers. Remove the unused declarations and definitions. This patch (of 2): damon_targets_empty() has had no in-tree users since commit 5ec4333b1967 ("mm/damon: remove DAMON debugfs interface") removed its last caller. Remove the unused declaration and definition. Link: https://lore.kernel.org/20260831145724.82387-1-sj@kernel.org Link: https://lore.kernel.org/20260831145724.82387-2-sj@kernel.org Signed-off-by: Cheng-Han Wu Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park --- include/linux/damon.h | 1 - mm/damon/core.c | 5 ----- 2 files changed, 6 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 4b0d2d2e4ea4..89a41dea1d23 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1055,7 +1055,6 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src); struct damon_target *damon_new_target(void); void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); -bool damon_targets_empty(struct damon_ctx *ctx); void damon_free_target(struct damon_target *t); void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx); unsigned int damon_nr_regions(struct damon_target *t); diff --git a/mm/damon/core.c b/mm/damon/core.c index ab3c4d754964..dc772ceae26f 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -795,11 +795,6 @@ void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) list_add_tail(&t->list, &ctx->adaptive_targets); } -bool damon_targets_empty(struct damon_ctx *ctx) -{ - return list_empty(&ctx->adaptive_targets); -} - static void damon_del_target(struct damon_target *t) { list_del(&t->list); From 0e728fe53606ea6a725f7f27c4623b64dfa7dce8 Mon Sep 17 00:00:00 2001 From: Cheng-Han Wu Date: Mon, 31 Aug 2026 07:57:22 -0700 Subject: [PATCH 116/539] mm/damon/core: remove unused damon_nr_running_ctxs() damon_nr_running_ctxs() has had no in-tree users since commit 5ec4333b1967 ("mm/damon: remove DAMON debugfs interface") removed all of its callers. Remove the unused declaration and definition. Link: https://lore.kernel.org/20260831145724.82387-3-sj@kernel.org Signed-off-by: Cheng-Han Wu Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park --- include/linux/damon.h | 1 - mm/damon/core.c | 14 -------------- 2 files changed, 15 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 89a41dea1d23..6993dca0f355 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1065,7 +1065,6 @@ int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs); void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, ssize_t nr_schemes); int damon_commit_ctx(struct damon_ctx *old_ctx, struct damon_ctx *new_ctx); -int damon_nr_running_ctxs(void); bool damon_is_registered_ops(enum damon_ops_id id); int damon_register_ops(struct damon_operations *ops); int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id); diff --git a/mm/damon/core.c b/mm/damon/core.c index dc772ceae26f..d832a527bcf6 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1847,20 +1847,6 @@ out: return err; } -/** - * damon_nr_running_ctxs() - Return number of currently running contexts. - */ -int damon_nr_running_ctxs(void) -{ - int nr_ctxs; - - mutex_lock(&damon_lock); - nr_ctxs = nr_running_ctxs; - mutex_unlock(&damon_lock); - - return nr_ctxs; -} - /* Returns the size upper limit for each monitoring region */ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) { From f465dc71aba00c541c23e06e2f5d579c470cf80a Mon Sep 17 00:00:00 2001 From: Asier Gutierrez Date: Mon, 31 Aug 2026 07:47:28 -0700 Subject: [PATCH 117/539] mm/damon: introduce DAMOS_QUOTA_HUGEPAGE auto tuning Patch series "mm/damon: Introduce a huge page collapsing mechanism using auto tuning", v4. Overview ======== This patchset introduces a new autotuning which allows to collapse hot regions into hugepages. Motivation ========== Since TLB is a bottleneck for many systems[1], a way to optimize TLB misses (or hits) is to use huge pages. Unfortunately, using "always" in THP leads to memory fragmentation and memory waste. For this reason, most application guides and system administrators suggest to disable THP. Selective huge page collapse per process is possible using prctl and a launcher. However, this does not solve the issue with hot region detection. Additionally, it the sysadmin should create a launcher that uses PRCTL to enable THP for a particular process. We can use the DAMON support for DAMOS_HUGEPAGE and DAMOS_COLLAPSE, to target a certain process. DAMOS_COLLAPSE can also target the hot regions in that process. Still, there is an issue with the amount of huge page consumption. Since huge pages can lead to memory fragmentation and waste, there should be a way to limit the amount of huge page consumption. There is hugetlbfs, but it requires changes to the application code or the use of libhugetlbfs. DAMON has now a way to autotune some of the variables and adjust quotas automatically, so that DAMON is fired only under the right circumstances. It would be nice to have something similar, but for huge pages. Solution ======== A new autotuning quota goal[2], damos_hugepage_mem_bp, is introduced, which checks the huge page consumption to total memory consumption. This new quota mechanism reuses current autotuning architecture. In order to test this new mechanism, a sample module[3] was created, but not included in this patch series. To demonstrate the tool, damo user space tool was modified[4], which sets up huge pages collapse autotuning. Benchmarks ========== Setup: physical server with arm64 processor with 4 NUMA nodes, 1 TB RAM and running mariaDB 10.5.29. Sysbench was used for the benchmark, with 20 tables and 3 million rows per table. The database was pinned to one of the nodes, and the benchmark framework to a different node. No network traffic involved in the benchmark. Damo user space tool was forked and hugepage_mem_bp support added[4]. DAMON was lauched using this command line: sudo ./damo start $(pidof mariadbd) \ --monitoring_nr_regions_range 10 1000 \ --monitoring_intervals 5000 100000 60000000 \ --damos_quota_time 0 --damos_quota_space 128000000 \ --damos_quota_interval 1000 \ --damos_quota_weights 0 1 1 \ --damos_quota_goal hugepage_mem_bp \ --damos_quota_goal_tuner temporal \ --damos_apply_interval 50000 \ --damos_access_rate 0 max --damos_age 50 max \ --damos_action collapse --debug_damon was 1000 to taget 10% hugepage to total memory ratio, or 2500 to target 25%. Tuner was also tested with consistent and temporal. Results ======= After the last timestamp, there was no change in huge page use, and the total huge page to memory consumption ratio barely moved. hugepage_mem_bp: 1000 goal tuner: temporal +-----------+----------------+----------------+----------------------+ | timestamp | total mem used | huge page used | percentage hugepage | +-----------+----------------+----------------+----------------------+ | 0 | 16945.04297 | 0 | 0 | | 7 | 17008.69531 | 74 | 0.435071583 | | 8 | 17036.40234 | 194 | 1.138738074 | | 9 | 17017.01563 | 314 | 1.845211916 | | 10 | 17029.67969 | 434 | 2.548491856 | | 61 | 17111.30859 | 584 | 3.412947623 | | 120 | 17071.05859 | 694 | 4.065360072 | | 180 | 17133.88281 | 804 | 4.692456513 | | 203 | 17088.16406 | 916 | 5.360435426 | | 204 | 17126.34766 | 1046 | 6.107548562 | | 205 | 17093.84375 | 1176 | 6.879669764 | | 206 | 17142.77734 | 1298 | 7.571701913 | | 209 | 17149.17969 | 1686 | 9.831374041 | | 210 | 17097.30859 | 1754 | 10.25892462 | +-----------+----------------+----------------+----------------------+ hugepage_mem_bp: 1000 goal tuner: consistent +-----------+----------------+----------------+----------------------+ | timestamp | total mem used | huge page used | percentage hugepage | +-----------+----------------+----------------+----------------------+ | 0 | 16955.24609 | 0 | 0 | | 34 | 17039.71875 | 106 | 0.622075995 | | 78 | 17009.47656 | 554 | 3.257007927 | | 90 | 17048.92188 | 596 | 3.495822225 | | 150 | 17092.90625 | 706 | 4.130368409 | | 180 | 17053.08984 | 764 | 4.480126517 | | 233 | 17100.50391 | 1496 | 8.748280216 | | 239 | 17098.89063 | 2216 | 12.95990511 | | 240 | 17135.44531 | 2334 | 13.62088908 | | 245 | 17132.55078 | 2932 | 17.11362212 | | 246 | 17117.95313 | 3052 | 17.82923448 | | 250 | 17163.12109 | 3532 | 20.57900763 | +-----------+----------------+----------------+----------------------+ hugepage_mem_bp: 2500 goal tuner: temporal +-----------+----------------+----------------+----------------------+ | timestamp | total mem used | huge page used | percentage hugepage | +-----------+----------------+----------------+----------------------+ | 0 | 17010.31641 | 0 | 0 | | 9 | 17063.6875 | 50 | 0.2930199 | | 10 | 17051.75781 | 170 | 0.996964664 | | 60 | 17133.85547 | 572 | 3.338419663 | | 90 | 17192.07813 | 626 | 3.641211932 | | 120 | 17221.44531 | 682 | 3.960178647 | | 181 | 17199.76172 | 790 | 4.593086886 | | 208 | 17222.77734 | 1206 | 7.002354939 | | 214 | 17245.17969 | 1904 | 11.04076637 | | 215 | 17240.45703 | 2024 | 11.73982799 | | 220 | 17234.79688 | 2624 | 15.22501262 | | 228 | 17222.83594 | 3584 | 20.80958103 | | 231 | 17247.55469 | 3944 | 22.86700968 | | 235 | 17229.37109 | 4424 | 25.67708349 | +-----------+----------------+----------------+----------------------+ hugepage_mem_bp: 1000 goal tuner: consist +-----------+----------------+----------------+----------------------+ | timestamp | total mem used | huge page used | percentage hugepage | +-----------+----------------+----------------+----------------------+ | 0 | 17125.85156 | 0 | 0 | | 38 | 17081.23438 | 76 | 0.444932716 | | 39 | 17133.11719 | 196 | 1.143983304 | | 40 | 17119.83984 | 316 | 1.84581166 | | 60 | 17109.72656 | 554 | 3.237924335 | | 90 | 17164.11328 | 628 | 3.65879664 | | 180 | 17177.66016 | 792 | 4.610639591 | | 220 | 17180.86719 | 1378 | 8.020549749 | | 226 | 17187.82031 | 1980 | 11.51978531 | | 233 | 17143.48438 | 2818 | 16.4377319 | | 240 | 17137.38281 | 3656 | 21.33347921 | | 250 | 17175.5 | 4856 | 28.27283049 | | 260 | 17199.66406 | 6056 | 35.20999002 | | 270 | 17203.98438 | 7254 | 42.16465118 | | 275 | 17207.21875 | 7762 | 45.10897498 | +-----------+----------------+----------------+----------------------+ More detailed tables are provided here[5] From this, we can conclude that the huge page autotuner works fine, achieving the target. When using consistent autotuner, it actually over-achieves the target, which is expected, since quota esz_bp is not set to 0 to cap the DAMOS policy. Patches Sequence ================ Patch 1 -> Introduce DAMOS_QUOTA_HUGEPAGE_MEM_BP and autotuning Patch 2 -> sysfs support for the new quota goal Patch 3 -> Document hugepage_mem_bp parameter This patch (of 3): Introduce DAMOS_QUOTA_HUGEPAGE_MEM_BP auto tuning. Add a new DAMOS quota goal metric to measure the amount of huge page consumption to total memory consumption ratio. Vmstat may lag, which in some cases may lead to NR_FREE_PAGES being greater than or equal to the amount of RAM in the system. A guard is added to avoid the extremely unlikely case [6]. In the case, return 100% (10000 bp). Link: https://lore.kernel.org/20260831144732.80910-1-sj@kernel.org Link: https://lore.kernel.org/20260831144732.80910-2-sj@kernel.org Link: https://dl.acm.org/doi/pdf/10.1145/3307650.3322227 [1] Link: https://lore.kernel.org/e67f05ad-dbb9-45e6-ba30-b167a99ac67d@huawei-partners.com [2] Link: https://lore.kernel.org/20260616150316.580819-3-gutierrez.asier@huawei-partners.com [3] Link: https://github.com/asierHuawei/damo/commit/79ae1a4ab1c012a7161db85a000d14f08fa36736 [4] Link: https://lore.kernel.org/all/03f678dd-9ef3-4b97-b753-c2e4554c5159@huawei-partners.com/ [5] Link: https://lore.kernel.org/all/20260715151615.99767-1-sj@kernel.org/ [6] Signed-off-by: Asier Gutierrez Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 2 ++ mm/damon/core.c | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 6993dca0f355..955b9f614e5b 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -153,6 +153,7 @@ enum damos_action { * @DAMOS_QUOTA_INACTIVE_MEM_BP: Inactive to total LRU memory ratio. * @DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP: Scheme-eligible memory ratio of a * node in basis points (0-10000). + * @DAMOS_QUOTA_HUGEPAGE_MEM_BP: Huge page to total used memory ratio. * @NR_DAMOS_QUOTA_GOAL_METRICS: Number of DAMOS quota goal metrics. * * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported. @@ -167,6 +168,7 @@ enum damos_quota_goal_metric { DAMOS_QUOTA_ACTIVE_MEM_BP, DAMOS_QUOTA_INACTIVE_MEM_BP, DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP, + DAMOS_QUOTA_HUGEPAGE_MEM_BP, NR_DAMOS_QUOTA_GOAL_METRICS, }; diff --git a/mm/damon/core.c b/mm/damon/core.c index d832a527bcf6..153425e416f5 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2991,6 +2991,22 @@ static unsigned int damos_get_in_active_mem_bp(bool active_ratio) return mult_frac(inactive, 10000, total); } +static unsigned int damos_hugepage_mem_bp(void) +{ + unsigned long thp, total_pages, free_pages; + + total_pages = totalram_pages(); + free_pages = global_zone_page_state(NR_FREE_PAGES); + + if (total_pages <= free_pages) + return 10000; + + thp = global_node_page_state(NR_ANON_THPS) + + global_node_page_state(NR_SHMEM_THPS) + + global_node_page_state(NR_FILE_THPS); + return mult_frac(thp, 10000, total_pages - free_pages); +} + static void damos_set_quota_goal_current_value(struct damon_ctx *c, struct damos *s, struct damos_quota_goal *goal) { @@ -3022,6 +3038,9 @@ static void damos_set_quota_goal_current_value(struct damon_ctx *c, goal->current_value = damos_get_node_eligible_mem_bp(c, s, goal->nid); break; + case DAMOS_QUOTA_HUGEPAGE_MEM_BP: + goal->current_value = damos_hugepage_mem_bp(); + break; default: break; } From b165a7a42feb0516f163a2d75c67ada361f5bb5a Mon Sep 17 00:00:00 2001 From: Asier Gutierrez Date: Mon, 31 Aug 2026 07:47:29 -0700 Subject: [PATCH 118/539] mm/damon/sysfs: support hugepage_mem_bp quota goal metric DAMOS has a new autotune policy metric: DAMOS_QUOTA_HUGEPAGE_MEM_BP. This patch exposes DAMOS_QUOTA_HUGEPAGE_MEM_BP through sysfs. Add the "hugepage_mem_bp" to the sysfs-schemes interface. Link: https://lore.kernel.org/20260831144732.80910-3-sj@kernel.org Signed-off-by: Asier Gutierrez Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs-schemes.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 32f495a96b17..d9b81d7b5910 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -1269,6 +1269,10 @@ struct damos_sysfs_qgoal_metric_name damos_sysfs_qgoal_metric_names[] = { .metric = DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP, .name = "node_eligible_mem_bp", }, + { + .metric = DAMOS_QUOTA_HUGEPAGE_MEM_BP, + .name = "hugepage_mem_bp", + }, }; static ssize_t target_metric_show(struct kobject *kobj, From 57d89b940a01cdc96668049903a981fcd59f5a0e Mon Sep 17 00:00:00 2001 From: Asier Gutierrez Date: Mon, 31 Aug 2026 07:47:30 -0700 Subject: [PATCH 119/539] Docs/mm/damon/design: cocument hugepage_mem_bp target metric Document hugepage_mem_bp metric exposed by sysfs. Link: https://lore.kernel.org/20260831144732.80910-4-sj@kernel.org Signed-off-by: Asier Gutierrez Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index d1dd9050ebf4..63cbb7b536da 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -713,6 +713,8 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to bp (1/10,000). - ``node_eligible_mem_bp``: Scheme target access pattern-eligible memory ratio of a node in bp (1/10,000). +- ``hugepage_mem_bp``: Total huge page to total used memory ratio in bp + (1/10,000). ``nid`` is optionally required for ``node_mem_used_bp``, ``node_mem_free_bp``, ``node_memcg_used_bp``, ``node_memcg_free_bp`` and ``node_eligible_mem_bp`` to From e287e45ab84f5f95859ee8810bebbea9914b73e5 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Mon, 31 Aug 2026 07:26:03 -0700 Subject: [PATCH 120/539] mm/damon/core: remove declaration of __damon_commit_ctx() Patch series "mm/damon: misc cleanups". Cleanup the code, tests and samples for clarifications and readability. The patches are individually sent by the authors. I'm reposting those as one series for convenience of handling. For this reason, changelog is on each patch's commentary section. This patch (of 7): __damon_commit_ctx() was added by commit b1471afe4d10 ("mm/damon/core: do parameter testing commit on damon_start()") but is actually not needed. Remove it. Link: https://lore.kernel.org/20260831142611.77572-1-sj@kernel.org Link: https://lore.kernel.org/20260831142611.77572-2-sj@kernel.org Signed-off-by: Zenghui Yu (Huawei) Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Greg Kroah-Hartman Cc: Shuah Khan Cc: Enze Li Cc: Hari Mishal Cc: Jaeyeon Lee Cc: Li Youhong Cc: zhaozhengzhuo --- mm/damon/core.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 153425e416f5..fb52d99661bb 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1935,8 +1935,6 @@ static int __damon_start(struct damon_ctx *ctx) return err; } -static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src); - /** * damon_start() - Starts the monitorings for a given group of contexts. * @ctxs: an array of the pointers for contexts to start monitoring From ee5a5b0a281339cb590d657ec69971e2627802eb Mon Sep 17 00:00:00 2001 From: Enze Li Date: Mon, 31 Aug 2026 07:26:04 -0700 Subject: [PATCH 121/539] mm/damon/core: introduce damon_set_target_pid() The logic that finds the struct pid for a given pid number and assigns it to a damon_target is duplicated in multiple places. Including damon_sysfs_add_target() of mm/damon/sysfs.c and the start functions of the two sample modules, samples/damon/wsse.c and samples/damon/prcl.c. Add a function that does the work, and replace the duplicated code in the places with calls to the function. Link: https://lore.kernel.org/20260831142611.77572-3-sj@kernel.org Signed-off-by: Enze Li Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Greg Kroah-Hartman Cc: Hari Mishal Cc: Jaeyeon Lee Cc: Li Youhong Cc: Shuah Khan Cc: "Zenghui Yu (Huawei)" Cc: zhaozhengzhuo --- include/linux/damon.h | 1 + mm/damon/core.c | 12 ++++++++++++ mm/damon/sysfs.c | 6 ++---- samples/damon/prcl.c | 5 +---- samples/damon/wsse.c | 5 +---- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 955b9f614e5b..7b1b6050a828 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1057,6 +1057,7 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src); struct damon_target *damon_new_target(void); void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); +int damon_set_target_pid(struct damon_target *t, int pid); void damon_free_target(struct damon_target *t); void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx); unsigned int damon_nr_regions(struct damon_target *t); diff --git a/mm/damon/core.c b/mm/damon/core.c index fb52d99661bb..d63d4c6fd3ff 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -795,6 +796,17 @@ void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) list_add_tail(&t->list, &ctx->adaptive_targets); } +/* + * Assign the struct pid of the given pid number to the given target. + */ +int damon_set_target_pid(struct damon_target *t, int pid) +{ + t->pid = find_get_pid(pid); + if (!t->pid) + return -EINVAL; + return 0; +} + static void damon_del_target(struct damon_target *t) { list_del(&t->list); diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index e3858ffab4b2..3c81b4c91ac0 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -3,7 +3,6 @@ * DAMON sysfs Interface */ -#include #include #include @@ -2035,9 +2034,8 @@ static int damon_sysfs_add_target(struct damon_sysfs_target *sys_target, return -ENOMEM; damon_add_target(ctx, t); if (damon_target_has_pid(ctx)) { - t->pid = find_get_pid(sys_target->pid); - if (!t->pid) - /* caller will destroy targets */ + /* caller will destroy targets */ + if (damon_set_target_pid(t, sys_target->pid)) return -EINVAL; } t->obsolete = sys_target->obsolete; diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index 842099bd6228..83ddf12811d5 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -32,7 +32,6 @@ module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_PRCL"); static struct damon_ctx *ctx; -static struct pid *target_pidp; static int damon_sample_prcl_repeat_call_fn(void *data) { @@ -79,12 +78,10 @@ static int damon_sample_prcl_start(void) return -ENOMEM; } damon_add_target(ctx, target); - target_pidp = find_get_pid(target_pid); - if (!target_pidp) { + if (damon_set_target_pid(target, target_pid)) { damon_destroy_ctx(ctx); return -EINVAL; } - target->pid = target_pidp; scheme = damon_new_scheme( &(struct damos_access_pattern) { diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index 37fd5da20158..53944aea8428 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -33,7 +33,6 @@ module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_WSSE"); static struct damon_ctx *ctx; -static struct pid *target_pidp; static int damon_sample_wsse_repeat_call_fn(void *data) { @@ -79,12 +78,10 @@ static int damon_sample_wsse_start(void) return -ENOMEM; } damon_add_target(ctx, target); - target_pidp = find_get_pid(target_pid); - if (!target_pidp) { + if (damon_set_target_pid(target, target_pid)) { damon_destroy_ctx(ctx); return -EINVAL; } - target->pid = target_pidp; err = damon_start(&ctx, 1, true); if (err) { From 61e9d12ca2150574b7ca0196bc9708a117df3e8a Mon Sep 17 00:00:00 2001 From: Li Youhong Date: Mon, 31 Aug 2026 07:26:05 -0700 Subject: [PATCH 122/539] mm/damon/ops-common: factor out damon_putback_folio_list() The putback loop is duplicated in damon_migrate_folio_list() and on the invalid-nid path of damon_migrate_pages(). Factor it into a small helper for readability. No functional change. Link: https://lore.kernel.org/20260831142611.77572-4-sj@kernel.org Signed-off-by: Li Youhong Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Enze Li Cc: Greg Kroah-Hartman Cc: Hari Mishal Cc: Jaeyeon Lee Cc: Shuah Khan Cc: "Zenghui Yu (Huawei)" Cc: zhaozhengzhuo --- mm/damon/ops-common.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 8fc61d06d358..7219c608b195 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -335,6 +335,19 @@ static unsigned int __damon_migrate_folio_list( return nr_succeeded; } +static void damon_putback_folio_list(struct list_head *folio_list) +{ + struct folio *folio; + + while (!list_empty(folio_list)) { + folio = lru_to_folio(folio_list); + list_del(&folio->lru); + node_stat_sub_folio(folio, NR_ISOLATED_ANON + + folio_is_file_lru(folio)); + folio_putback_lru(folio); + } +} + static unsigned int damon_migrate_folio_list(struct list_head *folio_list, struct pglist_data *pgdat, int target_nid) @@ -376,13 +389,7 @@ keep: list_splice(&ret_folios, folio_list); - while (!list_empty(folio_list)) { - folio = lru_to_folio(folio_list); - list_del(&folio->lru); - node_stat_sub_folio(folio, NR_ISOLATED_ANON + - folio_is_file_lru(folio)); - folio_putback_lru(folio); - } + damon_putback_folio_list(folio_list); return nr_migrated; } @@ -399,14 +406,7 @@ unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid) if (target_nid < 0 || target_nid >= MAX_NUMNODES || !node_state(target_nid, N_MEMORY)) { - while (!list_empty(folio_list)) { - struct folio *folio = lru_to_folio(folio_list); - - list_del(&folio->lru); - node_stat_sub_folio(folio, NR_ISOLATED_ANON + - folio_is_file_lru(folio)); - folio_putback_lru(folio); - } + damon_putback_folio_list(folio_list); return nr_migrated; } From 3ec4e93d870fd9df09dc9a466298f909e7aaacc5 Mon Sep 17 00:00:00 2001 From: Hari Mishal Date: Mon, 31 Aug 2026 07:26:06 -0700 Subject: [PATCH 123/539] selftests/damon/sysfs.py: clean up sh processes used for obsolete_target test The obsolete_target test spawns three sh processes and uses their pids as DAMON monitoring targets. These processes are never terminated or waited on, so they are left running (or become zombies) as orphaned children after the test program exits. Terminate each process and communicate() with it after the targets are no longer needed, so it exits and gets reaped instead of being leaked. Link: https://lore.kernel.org/20260831142611.77572-5-sj@kernel.org Signed-off-by: Hari Mishal Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Greg Kroah-Hartman Cc: Enze Li Cc: Jaeyeon Lee Cc: Li Youhong Cc: Shuah Khan Cc: "Zenghui Yu (Huawei)" Cc: zhaozhengzhuo --- tools/testing/selftests/damon/sysfs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index 3ffa054b6386..88a26422ff44 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -385,6 +385,10 @@ def main(): assert_ctxs_committed(kdamonds) kdamonds.stop() + for proc in (proc1, proc2, proc3): + proc.terminate() + proc.communicate() + test_memcg_filter_memcg_path_staging() if __name__ == '__main__': From a861a1dbfa35cade6078972df11831fd84c0dea2 Mon Sep 17 00:00:00 2001 From: Jaeyeon Lee Date: Mon, 31 Aug 2026 07:26:07 -0700 Subject: [PATCH 124/539] mm/damon/tests: use scoped_guard() for damon_test_ops_registration Replace manual mutex_lock() and mutex_unlock() calls with the scoped_guard() macro. This simplifies the code, improves readability, and ensures that the lock is automatically released when the scope ends, preventing potential lock leaks in the future. Link: https://lore.kernel.org/20260831142611.77572-6-sj@kernel.org Signed-off-by: Jaeyeon Lee Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Enze Li Cc: Greg Kroah-Hartman Cc: Hari Mishal Cc: Li Youhong Cc: Shuah Khan Cc: "Zenghui Yu (Huawei)" Cc: zhaozhengzhuo --- mm/damon/tests/core-kunit.h | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index b1ca4c8e03f0..7071ec277b00 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -441,17 +441,17 @@ static void damon_test_ops_registration(struct kunit *test) KUNIT_EXPECT_EQ(test, damon_select_ops(c, NR_DAMON_OPS), -EINVAL); /* Registration should success after unregistration */ - mutex_lock(&damon_ops_lock); - bak = damon_registered_ops[DAMON_OPS_VADDR]; - damon_registered_ops[DAMON_OPS_VADDR] = (struct damon_operations){}; - mutex_unlock(&damon_ops_lock); + scoped_guard(mutex, &damon_ops_lock) { + bak = damon_registered_ops[DAMON_OPS_VADDR]; + damon_registered_ops[DAMON_OPS_VADDR] = + (struct damon_operations){}; + } ops.id = DAMON_OPS_VADDR; KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), 0); - mutex_lock(&damon_ops_lock); - damon_registered_ops[DAMON_OPS_VADDR] = bak; - mutex_unlock(&damon_ops_lock); + scoped_guard(mutex, &damon_ops_lock) + damon_registered_ops[DAMON_OPS_VADDR] = bak; /* Check double-registration failure again */ KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), -EINVAL); @@ -459,10 +459,9 @@ static void damon_test_ops_registration(struct kunit *test) damon_destroy_ctx(c); if (need_cleanup) { - mutex_lock(&damon_ops_lock); - damon_registered_ops[DAMON_OPS_VADDR] = - (struct damon_operations){}; - mutex_unlock(&damon_ops_lock); + scoped_guard(mutex, &damon_ops_lock) + damon_registered_ops[DAMON_OPS_VADDR] = + (struct damon_operations){}; } } From 3009b0a76cd19d63c45da6b11ce910b00e9f630b Mon Sep 17 00:00:00 2001 From: zhaozhengzhuo Date: Mon, 31 Aug 2026 07:26:08 -0700 Subject: [PATCH 125/539] selftests/damon: prevent remaining cross-object state pollution _damon_sysfs.py defines constructors with mutable default arguments, including DamosAccessPattern(), DamosQuota(), DamosWatermarks(), DamosDests(), IntervalsGoal(), and empty lists. Default arguments are evaluated once at function definition time. Damos() instances created without explicit arguments therefore share the same DamosQuota(), and the other default-constructed sub-objects and lists are shared in the same way. The sub-objects keep back-pointers to their owner scheme, so constructing the second Damos() rebinds the shared quota's scheme pointer to the second object. An item appended to one object's default contexts or filters list is also visible from other default-constructed objects. The shared state can corrupt test configurations. DamosQuota.sysfs_dir() derives the sysfs directory from its scheme pointer, so operating on the first scheme's default quota may write to the second scheme's directory. The wrong values often match the defaults, so tests still pass, but the behavior depends on object creation order. Commit 8319dadcbd81 ("selftests/damon: prevent cross-context state pollution in DamonCtx") fixed the same pattern in DamonCtx only. Fix the remaining constructors by defaulting to None and creating fresh objects or lists inside each constructor. Explicit arguments keep their previous behavior. Link: https://lore.kernel.org/20260831142611.77572-7-sj@kernel.org Signed-off-by: zhaozhengzhuo Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Enze Li Cc: Greg Kroah-Hartman Cc: Hari Mishal Cc: Jaeyeon Lee Cc: Li Youhong Cc: Shuah Khan Cc: "Zenghui Yu (Huawei)" --- tools/testing/selftests/damon/_damon_sysfs.py | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index e6a2265d721e..f604b7d6530b 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -321,8 +321,10 @@ class DamosFilters: filters = None scheme = None # owner scheme - def __init__(self, name, filters=[]): + def __init__(self, name, filters=None): self.name = name + if filters is None: + filters = [] self.filters = filters for idx, filter_ in enumerate(self.filters): filter_.idx = idx @@ -368,7 +370,9 @@ class DamosDests: dests = None scheme = None # owner scheme - def __init__(self, dests=[]): + def __init__(self, dests=None): + if dests is None: + dests = [] self.dests = dests for idx, dest in enumerate(self.dests): dest.idx = idx @@ -426,15 +430,21 @@ class Damos: stats = None tried_regions = None - def __init__(self, action='stat', access_pattern=DamosAccessPattern(), - quota=DamosQuota(), watermarks=DamosWatermarks(), - core_filters=[], ops_filters=[], filters=[], target_nid=0, - dests=DamosDests(), apply_interval_us=0): + def __init__(self, action='stat', access_pattern=None, quota=None, + watermarks=None, core_filters=None, ops_filters=None, + filters=None, target_nid=0, dests=None, + apply_interval_us=0): self.action = action + if access_pattern is None: + access_pattern = DamosAccessPattern() self.access_pattern = access_pattern self.access_pattern.scheme = self + if quota is None: + quota = DamosQuota() self.quota = quota self.quota.scheme = self + if watermarks is None: + watermarks = DamosWatermarks() self.watermarks = watermarks self.watermarks.scheme = self @@ -448,6 +458,8 @@ class Damos: self.filters.scheme = self self.target_nid = target_nid + if dests is None: + dests = DamosDests() self.dests = dests self.dests.scheme = self @@ -568,10 +580,12 @@ class DamonAttrs: context = None def __init__(self, sample_us=5000, aggr_us=100000, - intervals_goal=IntervalsGoal(), update_us=1000000, - min_nr_regions=10, max_nr_regions=1000): + intervals_goal=None, update_us=1000000, min_nr_regions=10, + max_nr_regions=1000): self.sample_us = sample_us self.aggr_us = aggr_us + if intervals_goal is None: + intervals_goal = IntervalsGoal() self.intervals_goal = intervals_goal self.intervals_goal.attrs = self self.update_us = update_us @@ -703,7 +717,9 @@ class Kdamond: idx = None # index of this kdamond between siblings kdamonds = None # parent - def __init__(self, contexts=[], refresh_ms=None): + def __init__(self, contexts=None, refresh_ms=None): + if contexts is None: + contexts = [] self.contexts = contexts self.refresh_ms = refresh_ms for idx, context in enumerate(self.contexts): @@ -853,7 +869,9 @@ class Kdamond: class Kdamonds: kdamonds = [] - def __init__(self, kdamonds=[]): + def __init__(self, kdamonds=None): + if kdamonds is None: + kdamonds = [] self.kdamonds = kdamonds for idx, kdamond in enumerate(self.kdamonds): kdamond.idx = idx From fb14504f2e8fdbf5efe4faf0051397d180bff4af Mon Sep 17 00:00:00 2001 From: Enze Li Date: Mon, 31 Aug 2026 07:26:09 -0700 Subject: [PATCH 126/539] samples/damon/mtier: add comment for struct region_range The mtier sample defines a local struct region_range using phys_addr_t instead of damon_addr_range which uses unsigned long. Add a comment explaining the rationale: on 32-bit systems with more than 4GiB memory, phys_addr_t will be 64-bit while unsigned long is 32-bit. Link: https://lore.kernel.org/20260831142611.77572-8-sj@kernel.org Signed-off-by: Enze Li Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Greg Kroah-Hartman Cc: Hari Mishal Cc: Jaeyeon Lee Cc: Li Youhong Cc: Shuah Khan Cc: "Zenghui Yu (Huawei)" Cc: zhaozhengzhuo --- samples/damon/mtier.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index d1123ebbfab9..bea45c87cc9b 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -52,6 +52,11 @@ module_param(detect_node_addresses, bool, 0600); static struct damon_ctx *ctxs[2]; +/* + * Use phys_addr_t instead of damon_addr_range (unsigned long) for physical + * addresses. On 32-bit systems with more than 4GB memory, phys_addr_t will + * be 64-bit while unsigned long is 32-bit. + */ struct region_range { phys_addr_t start; phys_addr_t end; From acf561badfadea98525a2ce296612083595e1ba3 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 31 Aug 2026 19:16:32 +0800 Subject: [PATCH 127/539] mm: fix stale ZONE_DEVICE refcount comment Patch series "mm: optimize zone-device memmap initialization", v11. memmap_init_zone_device() can take a noticeable amount of time when large pmem namespaces are bound or rebound, because it initializes nearly identical struct page descriptors one PFN at a time. This series reduces that ZONE_DEVICE memmap initialization overhead by reusing prepared struct page templates and, on x86, using memcpy_nontemporal() for the template copy path. The main target is large fsdax/devdax pmem configurations, where the cost of initializing the memmap shows up directly in nd_pmem/dax_pmem bind and rebind latency. This matters because the cost is paid in the synchronous probe/bind path for large DAX/PMEM ZONE_DEVICE mappings. Userspace workflows such as provisioning or reconfiguring nd_pmem/dax_pmem namespaces, bringing hot-added PMEM-backed capacity online, and recovering or rebinding a device after driver or device changes all wait for this initialization to finish. Reducing this cost will yield benefits as lower user-visible provisioning, hot-add, recovery, and rebind latency for large DAX/PMEM devices. Patches 1-2 are preparatory cleanups and helper extraction. Patches 3-4 add the template-copy path for head pages and compound tails. Patch 5 introduces memcpy_nontemporal(). Patch 6 switches the ZONE_DEVICE template-copy path over to memcpy_nontemporal(). Patch 7 extends the x86 fixed-size memcpy_flushcache() inline cases used by the x86 memcpy_nontemporal() backend for struct page sized copies. Architectures without a specialized memcpy_nontemporal() backend fall back to memcpy(), so the generic template-copy optimization remains available without arch-specific support. On x86, memcpy_nontemporal() maps to the existing memcpy_flushcache() backend and can use the fixed-size MOVNTI paths added by this series for struct page sized copies. memcpy_nontemporal() is only a copy primitive. It does not imply a drain or a publication barrier. Callers that use it before a producer-consumer or device-visible handoff must provide the required ordering. The ZONE_DEVICE template-copy path uses it only while initializing struct page metadata, so the copy primitive itself does not grow a separate drain contract. The numbers below measure the time spent in memmap_init_zone_device() during driver bind/rebind. They are not measurements of the full nd_pmem or dax_pmem bind/rebind operation. Tested in an x86_64 QEMU/KVM VM with a 100 GB fsdax namespace device configured with map=dev and a 100 GB devdax namespace (align=2097152) on Intel Ice Lake server. Test procedure: Rebind the nd_pmem and dax_pmem drivers 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.3-rc1): Average of nd_pmem rebinds: 221.07 ms Average of dax_pmem rebinds: 191.20 ms With this series applied: Average of nd_pmem rebinds: 71.93 ms Average of dax_pmem rebinds: 87.37 ms This reduces the average memmap initialization time measured during rebind by about 67.5% for nd_pmem and 54.3% for dax_pmem. As an additional x86_64 data point, I also ran measurements on the same physical host with a 100 GB PMEM region created via the memmap= kernel command line, configured as fsdax and devdax namespaces with map=dev and 2 MiB alignment. For brevity, the individual patches keep only the VM results rather than including a second set of physical-host measurements throughout the series. The physical-host numbers below are included only as supplemental evidence that the same optimization also provides a similar benefit on a non-virtualized system. Test procedure: Reconfigure the namespace mode, rebind the nd_pmem or dax_pmem driver 30 times, and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base (v7.3-rc1): nd_pmem / fsdax: 205.90 ms dax_pmem / devdax: 225.43 ms With this series applied: nd_pmem / fsdax: 69.13 ms dax_pmem / devdax: 90.67 ms This reduces the measured memmap initialization time during rebind by about 66.4% for nd_pmem and 59.8% for dax_pmem on that setup, which is broadly consistent with the VM results above. As another supplemental data point, I measured the test_hmm.ko module on the same physical x86_64 host, using the test_hmm.ko setup from the previous discussion that times ten 64 GB memremap_pages()/memunmap_pages() iterations during module insertion[1]. By default, module insertion initializes two DEVICE_PRIVATE dmirror devices, so two avg memremap values are reported; each value is the average for one 64 GB chunk. This is not the primary target workload of the series, but it exercises the same large ZONE_DEVICE memmap initialization path and shows the same direction of improvement. Base (v7.3-rc1): avg memremap reported during module insertion: 116500596 ns, 116438028 ns With this series applied: avg memremap reported during module insertion: 46953088 ns, 46428399 ns This corresponds to about a 59.9% reduction based on the mean of the reported values, which is again consistent with the pmem bind/rebind results above. I also include an arm64 data point for the generic template-copy part. It was measured on an arm64 QEMU virt VM with 64 KB pages and a 100 GB ACPI NVDIMM sparse backend. This setup does not use the x86 MOVNTI fast paths, so it exercises the architecture-independent part of the optimization. For devdax, 2 MiB alignment is rejected in this 64 KB page setup, so the devdax namespace was tested with the supported default 512 MiB alignment. Base (v7.3-rc1): Average of rebinds for nd_pmem driver: 27.93 ms Average of rebinds for dax_pmem driver: 27.87 ms With this series applied: Average of rebinds for nd_pmem driver: 14.53 ms Average of rebinds for dax_pmem driver: 16.27 ms This reduces the average memmap initialization time measured during rebind by about 48.0% for nd_pmem and 41.6% for dax_pmem on that arm64 VM setup. Since this arm64 setup does not use the x86 MOVNTI fast paths, the result also suggests that the generic template-copy optimization can benefit architectures without an architecture-specific memcpy_nontemporal() backend. This patch (of 7): The comment in __init_zone_device_page() still uses the old MEMORY_TYPE_* names and implies that FS_DAX pages regain a refcount of 1 in the free path. That no longer matches the code. Update the comment to describe the current policy correctly: MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free path, while the remaining ZONE_DEVICE types start from 0 here and raise the count again when the allocator or driver hands the page out. No functional change intended. Link: https://lore.kernel.org/20260831111638.76012-1-lizhe.67@bytedance.com Link: https://lore.kernel.org/20260831111638.76012-2-lizhe.67@bytedance.com Link: https://lore.kernel.org/all/aiEoByaQdRR3xtM5@nvdebian.thelocal/ [1] Signed-off-by: Li Zhe Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Alistair Popple Reviewed-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Cc: Arnd Bergmann Cc: Balbir Singh Cc: "Borislav Petkov (AMD)" Cc: Dave Hansen Cc: Ingo Molnar Cc: Kees Cook --- mm/mm_init.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 304c88da5cce..951e6fc17f58 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1012,13 +1012,9 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, page->zone_device_data = NULL; /* - * ZONE_DEVICE pages other than MEMORY_TYPE_GENERIC are released - * directly to the driver page allocator which will set the page count - * to 1 when allocating the page. - * - * MEMORY_TYPE_GENERIC and MEMORY_TYPE_FS_DAX pages automatically have - * their refcount reset to one whenever they are freed (ie. after - * their refcount drops to 0). + * MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free + * path. The remaining ZONE_DEVICE types start from 0 here and raise + * the count again when the allocator or driver hands the page out. */ switch (pgmap->type) { case MEMORY_DEVICE_FS_DAX: From a5d43b7cd64235cc12d8563889f393894589f12f Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 31 Aug 2026 19:16:33 +0800 Subject: [PATCH 128/539] mm: add a set_page_section_from_pfn() helper Callers that want to update section bits from a PFN currently need to open-code: set_page_section(page, pfn_to_section_nr(pfn)); and guard that sequence with #ifdef SECTION_IN_PAGE_FLAGS. Add set_page_section_from_pfn() to wrap that update in one place. When section bits are stored in page flags, the helper derives the section number from the PFN and updates the page flags. Otherwise keep it as a no-op so callers can use one helper without open-coding SECTION_IN_PAGE_FLAGS. Convert set_page_links() to use the new helper so later ZONE_DEVICE fast-path patches can also update section bits without open-coding SECTION_IN_PAGE_FLAGS at each callsite. This keeps the PFN-to-section translation local to the configurations that actually store section bits in struct page flags, and avoids exposing that detail to generic callers. No functional change intended. Link: https://lore.kernel.org/20260831111638.76012-3-lizhe.67@bytedance.com Signed-off-by: Li Zhe Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport (Microsoft) Acked-by: Muchun Song Reviewed-by: Balbir Singh Cc: Alistair Popple Cc: Arnd Bergmann Cc: "Borislav Petkov (AMD)" Cc: Dave Hansen Cc: David Hildenbrand (Arm) Cc: Ingo Molnar Cc: Kees Cook --- include/linux/mm.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index a9fbe26536f4..1b28e6fc8d5d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2632,12 +2632,23 @@ static inline void set_page_section(struct page *page, unsigned long section) page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ + set_page_section(page, pfn_to_section_nr(pfn)); +} + static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ +} + static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; @@ -2860,9 +2871,7 @@ static inline void set_page_links(struct page *page, enum zone_type zone, { set_page_zone(page, zone); set_page_node(page, node); -#ifdef SECTION_IN_PAGE_FLAGS - set_page_section(page, pfn_to_section_nr(pfn)); -#endif + set_page_section_from_pfn(page, pfn); } /** From 72dcfdc2ce47c84d248ab43490ec91a0d5098253 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 31 Aug 2026 19:16:34 +0800 Subject: [PATCH 129/539] mm: add a template-based fast path for zone-device page init memmap_init_zone_device() repeats nearly identical head-page initialization for each PFN. Initialize the first real ZONE_DEVICE head page through the existing path, copy that final state into a reusable template, refresh the PFN-dependent fields in that template before each copy, and copy it into the remaining destination pages. Use the template path unconditionally. The page_ref_set tracepoint is primarily a debugging aid, while this code is still initializing struct pages before they are handed out. From the perspective of users of those pages, the initialization-time refcount transitions are not part of the observable page lifetime. This means page_ref_set will no longer observe every initialization-time refcount assignment for copied ZONE_DEVICE head pages. The impact is controlled because the final initialized struct page state is unchanged, and keeping a separate non-template path only for this local tracepoint observability would add complexity to the common path. This patch accelerates head-page initialization. The pfns_per_compound == 1 case gets the full benefit here, compound tails are handled in the next patch. Tested in a VM with a 100 GB fsdax namespace device configured with map=dev on Intel Ice Lake server. This test exercises the nd_pmem rebind path (pfns_per_compound == 1). Test procedure: Rebind the nd_pmem driver 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.3-rc1): Average of rebinds for nd_pmem driver: 221.07 ms With this patch and its prerequisites applied: Average of rebinds for nd_pmem driver: 155.00 ms This reduces the average memmap initialization time measured during rebind from 221.07 ms to 155.00 ms, or about 29.9%. [lizhe.67@bytedance.com: whitespace fix, per Mike] Link: https://lore.kernel.org/20260903025806.70825-1-lizhe.67@bytedance.com Link: https://lore.kernel.org/20260831111638.76012-4-lizhe.67@bytedance.com Signed-off-by: Li Zhe Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport (Microsoft) Cc: Alistair Popple Cc: Arnd Bergmann Cc: Balbir Singh Cc: "Borislav Petkov (AMD)" Cc: Dave Hansen Cc: David Hildenbrand (Arm) Cc: Ingo Molnar Cc: Kees Cook Cc: Muchun Song --- mm/mm_init.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 951e6fc17f58..af11885f17ef 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1029,6 +1029,17 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, } } +static void zone_device_page_init_from_template(struct page *page, + unsigned long pfn, struct page *template) +{ + set_page_section_from_pfn(template, pfn); +#ifdef WANT_PAGE_VIRTUAL + if (!is_highmem_idx(ZONE_DEVICE)) + set_page_address(template, __va(pfn << PAGE_SHIFT)); +#endif + memcpy(page, template, sizeof(*page)); +} + /* * With compound page geometry and when struct pages are stored in ram most * tail pages are reused. Consequently, the amount of unique struct pages to @@ -1091,6 +1102,8 @@ void __ref memmap_init_zone_device(struct zone *zone, unsigned long zone_idx = zone_idx(zone); unsigned long start = jiffies; int nid = pgdat->node_id; + struct page template; + struct page *page; if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) return; @@ -1105,10 +1118,28 @@ void __ref memmap_init_zone_device(struct zone *zone, nr_pages = end_pfn - start_pfn; } - for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { - struct page *page = pfn_to_page(pfn); + if (!nr_pages) + return; - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + /* + * Seed the reusable head-page template from the first real struct + * page. The normal page-init and refcount helpers must operate on + * a real memmap entry rather than a stack object. + */ + pfn = start_pfn; + page = pfn_to_page(pfn); + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + memcpy(&template, page, sizeof(*page)); + if (pfns_per_compound != 1) + memmap_init_compound(page, pfn, zone_idx, nid, pgmap, + compound_nr_pages(pfn, altmap, pgmap)); + pfn += pfns_per_compound; + + /* Initialize the remaining head pages from template. */ + for (; pfn < end_pfn; pfn += pfns_per_compound) { + page = pfn_to_page(pfn); + + zone_device_page_init_from_template(page, pfn, &template); if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); From a43a0ceff082382d943d8de13cec3ef0ff133a88 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 31 Aug 2026 19:16:35 +0800 Subject: [PATCH 130/539] mm: extend the template fast path to zone-device compound tails The template fast path from the previous patch only accelerates head pages. Compound tails in memmap_init_compound() still go through the normal initialization path one by one. Build separate head and tail templates and reuse one prepared tail template across the tail pages in a compound range. Head pages preserve the existing refcount policy, while compound tails always start with a refcount of 0 after prep_compound_tail(). This extends the template-copy fast path to pfns_per_compound > 1. Tail-page PFN-dependent fields are refreshed in the reusable tail template before each copy. Do not keep a separate non-template fallback for compound tails either. These pages are still under memmap initialization, and the initialization-time refcount updates are not part of the observable lifetime of pages handed out later. The impact is controlled for the same reason as for head pages. The first tail page still seeds the reusable tail template through the normal tail initialization sequence, and the copied tail pages have the same final initialized state except for the PFN-dependent fields refreshed before each copy. Tested in a VM with a 100 GB devdax namespace (align=2097152) on Intel Ice Lake server. This test exercises the dax_pmem rebind path and measures memmap initialization latency. Test procedure: Unbind and rebind the dax_pmem driver 30 times, collect memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.3-rc1): Average of rebinds for dax_pmem driver: 191.20 ms With this patch and its prerequisites applied: Average of rebinds for dax_pmem driver: 176.87 ms This reduces the average memmap initialization time measured during rebind from 191.20 ms to 176.87 ms, or about 7.5%. Link: https://lore.kernel.org/20260831111638.76012-5-lizhe.67@bytedance.com Signed-off-by: Li Zhe Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport (Microsoft) Cc: Alistair Popple Cc: Arnd Bergmann Cc: Balbir Singh Cc: "Borislav Petkov (AMD)" Cc: Dave Hansen Cc: David Hildenbrand (Arm) Cc: Ingo Molnar Cc: Kees Cook Cc: Muchun Song --- mm/mm_init.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index af11885f17ef..e2b0952023b3 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1072,6 +1072,8 @@ static void __ref memmap_init_compound(struct page *head, { unsigned long pfn, end_pfn = head_pfn + nr_pages; unsigned int order = pgmap->vmemmap_shift; + struct page template; + struct page *page; /* * We have to initialize the pages, including setting up page links. @@ -1080,13 +1082,23 @@ static void __ref memmap_init_compound(struct page *head, * the pages in the same go. */ __SetPageHead(head); - for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { - struct page *page = pfn_to_page(pfn); - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); - prep_compound_tail(page, head, order); - set_page_count(page, 0); - } + /* + * All tails of the same compound page share the state established by + * prep_compound_tail(). Reuse one tail template for the whole range and + * refresh only the PFN-dependent fields in that template before each copy. + */ + pfn = head_pfn + 1; + page = pfn_to_page(pfn); + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + prep_compound_tail(page, head, order); + set_page_count(page, 0); + memcpy(&template, page, sizeof(*page)); + + /* Initialize the remaining tail pages from template. */ + for (pfn = head_pfn + 2; pfn < end_pfn; pfn++) + zone_device_page_init_from_template(pfn_to_page(pfn), pfn, + &template); prep_compound_head(head, order); } From 1f0d15093bea9c5d760c95997d5e99f302bf635a Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 31 Aug 2026 19:16:36 +0800 Subject: [PATCH 131/539] string: introduce memcpy_nontemporal() Introduce memcpy_nontemporal() for write-once copy sites that want a named non-temporal copy primitive. On x86_64, override the helper in arch/x86/include/asm/string_64.h using the usual self-macro pattern, next to the existing memcpy_flushcache() backend that memcpy_nontemporal() wraps. include/linux/string.h provides the generic memcpy_nontemporal() fallback as #define memcpy_nontemporal(dst, src, len) \ ((void)memcpy(dst, src, len)) instead of an inline wrapper, so architectures without a specialized backend keep the usual memcpy() FORTIFY coverage when the compiler can still see object sizes at the original call site. It also makes the memcpy_nontemporal() API uniformly void, matching memcpy_flushcache() and the x86 backend, so callers cannot accidentally depend on a return value on fallback architectures. memcpy_nontemporal() is only a copy primitive. It does not imply a drain or a publication barrier. Callers that use it before a producer-consumer or device-visible handoff must provide the required ordering at that handoff point. The immediate user is the ZONE_DEVICE template-copy path. It populates struct page descriptors in a write-once pattern, so a regular cached memcpy() can incur avoidable write-allocate traffic and cache pollution for data with little near-term reuse. Link: https://lore.kernel.org/20260831111638.76012-6-lizhe.67@bytedance.com Signed-off-by: Li Zhe Signed-off-by: Andrew Morton Cc: Alistair Popple Cc: Arnd Bergmann Cc: Balbir Singh Cc: "Borislav Petkov (AMD)" Cc: Dave Hansen Cc: David Hildenbrand (Arm) Cc: Ingo Molnar Cc: Kees Cook Cc: Mike Rapoport (Microsoft) Cc: Muchun Song --- arch/x86/include/asm/string_64.h | 12 ++++++++++++ include/linux/string.h | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 4635616863f5..21ae515ae35a 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -100,6 +100,18 @@ static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t } __memcpy_flushcache(dst, src, cnt); } + +#define memcpy_nontemporal memcpy_nontemporal +/* + * Reuse the existing x86 flushcache backend as the non-temporal copy + * primitive. + */ +static __always_inline void memcpy_nontemporal(void *dst, const void *src, + size_t cnt) +{ + memcpy_flushcache(dst, src, cnt); +} + #endif #endif /* __KERNEL__ */ diff --git a/include/linux/string.h b/include/linux/string.h index 5702daca4326..6cb5cdd01158 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -278,6 +278,19 @@ static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) } #endif +#ifndef memcpy_nontemporal +/* + * memcpy_nontemporal() requests a non-temporal copy when the + * architecture has a suitable backend. Architectures without a + * specialized backend fall back to memcpy(). Keep this as a + * function-like macro so the compiler can still see the original + * memcpy() call site and preserve the usual FORTIFY coverage when + * object sizes remain visible there, while keeping the API void. + */ +#define memcpy_nontemporal(dst, src, len) \ + ((void)memcpy(dst, src, len)) +#endif + void *memchr_inv(const void *s, int c, size_t n); char *strreplace(char *str, char old, char new); From e52dcd69fbdb50976405d85a1d315a232283b6b5 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 31 Aug 2026 19:16:37 +0800 Subject: [PATCH 132/539] mm: use memcpy_nontemporal() in zone-device template copies The template fast path currently uses memcpy() for the actual struct page copy. Switch zone_device_page_init_from_template() to memcpy_nontemporal(). ZONE_DEVICE memmap initialization is largely write-once: each struct page is populated once, and most destination cachelines are not expected to be reused immediately afterwards. On x86, a regular cached memcpy() can therefore incur write-allocate traffic by pulling destination cachelines into the cache before writeback, and can populate the cache with data that has little near-term reuse. Using memcpy_nontemporal() lets this path request nontemporal stores for that copy pattern, which can reduce cache pollution and avoid part of the associated write-allocate overhead, while architectures without a specialized backend still fall back to memcpy(). Do not add a KASAN/KMSAN-specific fallback around this call site. As Muchun pointed out, special KASAN handling for memcpy_flushcache() or memcpy_nontemporal(), if needed, belongs in the low-level helper rather than in this ZONE_DEVICE caller. No separate drain is added here. memcpy_nontemporal() is used only as the copy primitive while memmap_init_zone_device() is still initializing the struct page array. The ordinary stores that follow in this path, such as compound-page setup, are part of the same CPU's initialization sequence; they are not used as a publication store that tells another CPU or device to consume data written by the non-temporal copy. Therefore this call site does not need a helper-level drain for correctness. Callers that use memcpy_nontemporal() as part of a producer-consumer or device-visible handoff must add the required ordering themselves. Tested in a VM with a 100 GB fsdax namespace device configured with map=dev and a 100 GB devdax namespace (align=2097152) on Intel Ice Lake server. Test procedure: Rebind the nd_pmem and dax_pmem driver 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.3-rc1): Average of rebinds for nd_pmem driver: 221.07 ms Average of rebinds for dax_pmem driver: 191.20 ms With this patch and its prerequisites applied: Average of rebinds for nd_pmem driver: 150.40 ms Average of rebinds for dax_pmem driver: 161.83 ms This reduces the average memmap initialization time measured during rebind by about 32.0% for nd_pmem and 15.4% for dax_pmem. Link: https://lore.kernel.org/20260831111638.76012-7-lizhe.67@bytedance.com Signed-off-by: Li Zhe Signed-off-by: Andrew Morton Cc: Alistair Popple Cc: Arnd Bergmann Cc: Balbir Singh Cc: "Borislav Petkov (AMD)" Cc: Dave Hansen Cc: David Hildenbrand (Arm) Cc: Ingo Molnar Cc: Kees Cook Cc: Mike Rapoport (Microsoft) Cc: Muchun Song --- mm/mm_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index e2b0952023b3..97e0158d2aca 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1037,7 +1037,7 @@ static void zone_device_page_init_from_template(struct page *page, if (!is_highmem_idx(ZONE_DEVICE)) set_page_address(template, __va(pfn << PAGE_SHIFT)); #endif - memcpy(page, template, sizeof(*page)); + memcpy_nontemporal(page, template, sizeof(*page)); } /* From 11bb36dbdf71577c1ba6519a0e4b10247ef92652 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 31 Aug 2026 19:16:38 +0800 Subject: [PATCH 133/539] x86/string: extend memcpy_flushcache() fixed-size fastpaths The x86 memcpy_nontemporal() helper maps to memcpy_flushcache(), and the ZONE_DEVICE template-copy path uses it to copy one struct page at a time. The relevant copy size is sizeof(struct page). On x86_64, the base struct page layout is 64 bytes. Adding either the KMSAN metadata pointers or an out-of-flags last_cpupid field can make it 80 bytes after alignment, and enabling both can make it 96 bytes. memcpy_flushcache() currently only has inline fixed-size cases for 4, 8, and 16 bytes. As a result, these constant-sized struct page copies fall through to __memcpy_flushcache() even though the compiler knows the copy size at the call site. Add fixed-size MOVNTI cases up to 96 bytes so the ZONE_DEVICE template-copy path can keep these struct page copies in the inline memcpy_flushcache() path. This matters for ZONE_DEVICE memmap initialization because the copy happens once per initialized struct page. For a 100 GB fsdax namespace with map=dev, this is about 25 million struct page copies during nd_pmem binding or rebinding. Tested in a VM with a 100 GB fsdax namespace device configured with map=dev and a 100 GB devdax namespace (align=2097152) on Intel Ice Lake server. Test procedure: Rebind the nd_pmem and dax_pmem drivers 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). With memcpy_nontemporal() used by the ZONE_DEVICE template-copy path: Average of rebinds for nd_pmem driver: 150.40 ms Average of rebinds for dax_pmem driver: 161.83 ms With this x86 fixed-size fastpath patch applied: Average of rebinds for nd_pmem driver: 71.93 ms Average of rebinds for dax_pmem driver: 87.37 ms This further reduces the average memmap initialization time measured during rebind by about 52.2% for nd_pmem and 46.0% for dax_pmem. Link: https://lore.kernel.org/20260831111638.76012-8-lizhe.67@bytedance.com Signed-off-by: Li Zhe Signed-off-by: Andrew Morton Suggested-by: Borislav Petkov Acked-by: Borislav Petkov (AMD) Acked-by: Dave Hansen Cc: Alistair Popple Cc: Arnd Bergmann Cc: Balbir Singh Cc: David Hildenbrand (Arm) Cc: Ingo Molnar Cc: Kees Cook Cc: Mike Rapoport (Microsoft) Cc: Muchun Song --- arch/x86/include/asm/string_64.h | 71 +++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 21ae515ae35a..831d3dda3b38 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -82,23 +82,64 @@ int strcmp(const char *cs, const char *ct); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1 void __memcpy_flushcache(void *dst, const void *src, size_t cnt); -static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) + +static __always_inline void movnti_4(void *dst, const void *src) { - if (__builtin_constant_p(cnt)) { - switch (cnt) { - case 4: - asm ("movntil %1, %0" : "=m"(*(u32 *)dst) : "r"(*(u32 *)src)); - return; - case 8: - asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src)); - return; - case 16: - asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src)); - asm ("movntiq %1, %0" : "=m"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src + 8))); - return; - } + asm volatile("movntil %1, %0" + : "=m"(*(u32 *)dst) + : "r"(*(const u32 *)src) + : "memory"); +} + +static __always_inline void movnti_8(void *dst, const void *src) +{ + asm volatile("movntiq %1, %0" + : "=m"(*(u64 *)dst) + : "r"(*(const u64 *)src) + : "memory"); +} + +static __always_inline void movnti_16(void *dst, const void *src) +{ + movnti_8(dst, src); + movnti_8(dst + 8, src + 8); +} + +static __always_inline void movnti_32(void *dst, const void *src) +{ + movnti_16(dst, src); + movnti_16(dst + 16, src + 16); +} + +static __always_inline void movnti_64(void *dst, const void *src) +{ + movnti_32(dst, src); + movnti_32(dst + 32, src + 32); +} + +static __always_inline void memcpy_flushcache(void *dst, const void *src, + size_t cnt) +{ + if (!__builtin_constant_p(cnt)) + return __memcpy_flushcache(dst, src, cnt); + + /* + * The relevant fixed-size copies here are the x86_64 struct page sizes: + * 64, 80, and 96 bytes. Keep 32-byte and 48-byte copies inline as well + * instead of sending those nearby fixed-size cases back to + * __memcpy_flushcache(). + */ + switch (cnt) { + case 4: movnti_4(dst, src); break; + case 8: movnti_8(dst, src); break; + case 16: movnti_16(dst, src); break; + case 32: movnti_32(dst, src); break; + case 48: movnti_32(dst, src); movnti_16(dst + 32, src + 32); break; + case 64: movnti_64(dst, src); break; + case 80: movnti_64(dst, src); movnti_16(dst + 64, src + 64); break; + case 96: movnti_64(dst, src); movnti_32(dst + 64, src + 64); break; + default: __memcpy_flushcache(dst, src, cnt); break; } - __memcpy_flushcache(dst, src, cnt); } #define memcpy_nontemporal memcpy_nontemporal From 39580da0aa8dcfc033c5dd724eb3debdc542d17a Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Mon, 31 Aug 2026 08:28:47 +0000 Subject: [PATCH 134/539] mm/rmap: remove stale hugetlb check in try_to_unmap_one Post commit d4ec5572825a ("mm/rmap: add try_to_unmap_poisoned_hugetlb_one") try_to_unmap_one() cannot be called with a hugetlb folio. Therefore remove the folio_test_hugetlb() check. Link: https://lore.kernel.org/20260831082849.3573957-1-dev.jain@arm.com Signed-off-by: Dev Jain Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Lance Yang Reviewed-by: Kunwu Chan Acked-by: David Hildenbrand (Arm) Cc: Harry Yoo Cc: Jann Horn Cc: Liam R. Howlett Cc: Rik van Riel Cc: Vlastimil Babka --- mm/rmap.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 3c67ad0e9562..0a3952706faf 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2301,11 +2301,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, VM_BUG_ON_FOLIO(!pvmw.pte, folio); address = pvmw.address; - if (folio_test_hugetlb(folio)) { - pteval = huge_ptep_get(mm, address, pvmw.pte); - } else { - pteval = ptep_get(pvmw.pte); - } + pteval = ptep_get(pvmw.pte); + if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { From 4f7d722673475bac9672fa9d1e734e8b784c75e9 Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Mon, 31 Aug 2026 15:43:04 +0530 Subject: [PATCH 135/539] mm/gup_test: report actual pinned bytes __gup_test_ioctl() advances addr to the end of the current batch before checking if GUP pinned the entire requested batch. If GUP pins more than 0 pages but less than the requested batch size, addr still advances by the requested batch size. The next iteration detects the partial pinning and breaks out of the loop. Again gup->size is calculated using addr - gup->addr, so it also includes the unpinned pages of the requested batch. Calculate gup->size using the actual number of pages pinned multiplied by PAGE_SIZE. Link: https://lore.kernel.org/20260831101304.162867-1-sarthak.sharma@arm.com Fixes: 64c349f4ae78 ("mm: add infrastructure for get_user_pages_fast() benchmarking") Signed-off-by: Sarthak Sharma Signed-off-by: Andrew Morton Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu --- mm/gup_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/gup_test.c b/mm/gup_test.c index 44c1cdfb9c37..185ba3bb8ed1 100644 --- a/mm/gup_test.c +++ b/mm/gup_test.c @@ -188,7 +188,7 @@ static int __gup_test_ioctl(unsigned int cmd, nr_pages = i; gup->get_delta_usec = ktime_us_delta(end_time, start_time); - gup->size = addr - gup->addr; + gup->size = nr_pages * PAGE_SIZE; /* * Take an un-benchmark-timed moment to verify DMA pinned From 5da8ec3262bbb40f057f3b0dacc18f37bda06f95 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Mon, 31 Aug 2026 10:15:13 +0100 Subject: [PATCH 136/539] mm/huge_memory: do not touch frozen folios in deferred_split_isolate() Patch series "Fix deferred_split_isolate() and drop the split workaround", v2. deferred_split_isolate() probes each queued folio with folio_try_get(). folio_try_get() failure is treated as a lost race with folio_put(). It leads to wrong results when !folio_try_get() was not caused by folio_put(): for a frozen folio, PG_partially_mapped gets wrongfully cleared and the folio dropped from the queue. It came up in the review of my collapse RFC series: https://lore.kernel.org/all/20260824131224.73344-1-lance.yang@linux.dev/ The bug is inert in upstream code: - __folio_split() works around it; - __folio_migrate_mapping() freezes a folio it is about to replace; - reclaim freezes only what try_to_unmap() already unmapped. No cc:stable needed. But my collapse rework steps on it, so it is worth fixing. The branch the first patch removes also hid an inert, pre-existing bug in the zone device path: https://lore.kernel.org/all/20260827163838.1813081-1-usama.arif@linux.dev/ The first patch fixes deferred_split_isolate(). The second patch removes the workaround for this deferred_split_isolate() behaviour from __folio_freeze_and_split_unmapped(). Tested in a VM: split_huge_page_test, folio_split_race_test and cow pass. Also ran a test that leaves 16 partially mapped THPs on the deferred split queue and drives thp-deferred_split through debugfs, checking nr_anon_partially_mapped. This patch (of 2): deferred_split_isolate() probes each queued folio with folio_try_get(). folio_try_get() failure is treated as a lost race with folio_put(): clear PG_partially_mapped, correct MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, take the folio off the queue. The folio_put() race is the most common case for !folio_try_get(), but it is not the only option. Another scenario is folio_ref_freeze(). A zero refcount in such cases does not mean the folio is going away. It means "don't touch me" and current deferred_split_isolate() doesn't respect it. It can lead to unqueueing folios from the deferred list for no reason: CPU 0 CPU 1 --------------------------- ------------------------------ freeze a mapped folio deferred_split_scan() folio_ref_freeze() folio_try_get() fails folio_clear_partially_mapped() NR_ANON_PARTIALLY_MAPPED-- folio off the queue give up, put it back folio_ref_unfreeze() The folio is still partially mapped, but it is no longer a split candidate. Nothing queues it again until part of it is unmapped once more. Skip the folio instead: whoever freezes the folio, owns it and owner is responsible for its fate. It also covers the folio_put() case: __folio_put() unqueues the folio via folio_unqueue_deferred_split(). Nothing is lost by skipping. Everything that frees a queued folio unqueues it first, and folio_unqueue_deferred_split() clears PG_partially_mapped and brings MTHP_STAT_NR_ANON_PARTIALLY_MAPPED down on the way: __folio_put(), folios_put_refs() mm/folio.c __folio_migrate_mapping() mm/migrate.c shrink_folio_list() mm/vmscan.c __folio_freeze_and_split_unmapped() does the same by hand, under the list_lru lock it holds across the freeze. A freeze that ends in folio_ref_unfreeze() leaves a folio that is still partially mapped and still belongs on the queue. Link: https://lore.kernel.org/20260831091514.1879786-1-kirill@shutemov.name Link: https://lore.kernel.org/20260831091514.1879786-2-kirill@shutemov.name Fixes: 8422acdc97ed ("mm: introduce a pageflag for partially mapped folios") Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Reported-by: Lance Yang Closes: https://lore.kernel.org/all/20260824131224.73344-1-lance.yang@linux.dev/ Reviewed-by: Zi Yan Reviewed-by: Johannes Weiner Reviewed-by: Lance Yang Acked-by: Usama Arif Reviewed-by: Baolin Wang Acked-by: David Hildenbrand (Arm) Assisted-by: Claude-Code:claude-opus-5 Cc: Balbir Singh Cc: Barry Song Cc: Dev Jain Cc: Hugh Dickins Cc: Kairui Song Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Ryan Roberts --- mm/huge_memory.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 54494c3fa983..779c02e0bf6c 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4638,22 +4638,11 @@ static enum lru_status deferred_split_isolate(struct list_head *item, struct folio *folio = container_of(item, struct folio, _deferred_list); struct list_head *freeable = cb_arg; - if (folio_try_get(folio)) { - list_lru_isolate_move(lru, item, freeable); - return LRU_REMOVED; - } + /* Lost race to folio_put() or the folio is under folio_ref_freeze() */ + if (!folio_try_get(folio)) + return LRU_SKIP; - /* - * We lost race with folio_put(). Read folio state before the - * isolate: folio_unqueue_deferred_split() checks list_empty() - * locklessly, so once removed the folio can be freed any time. - */ - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(folio_order(folio), - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } - list_lru_isolate(lru, item); + list_lru_isolate_move(lru, item, freeable); return LRU_REMOVED; } From 81ad3e1d15bcd444687f2814fd1b6cb92fd5cf79 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Mon, 31 Aug 2026 10:15:14 +0100 Subject: [PATCH 137/539] mm/huge_memory: dequeue the deferred split after the split freeze __folio_freeze_and_split_unmapped() takes the deferred split list_lru lock across the freeze. It is only there to stop deferred_split_scan() from touching the folio under split. With deferred_split_isolate() fixed, the workaround can be dropped. Unqueue the folio after folio_ref_freeze(), the way __folio_migrate_mapping() does: folio_unqueue_deferred_split() needs a zero refcount and a memcg still set, and both hold there. If the split is called from deferred_split_scan(), the unqueue is a no-op -- the folio is already removed from the list. But PG_partially_mapped is still set, so it has to be cleared here or MTHP_STAT_NR_ANON_PARTIALLY_MAPPED never comes back down. Link: https://lore.kernel.org/20260831091514.1879786-3-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Johannes Weiner Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Assisted-by: Claude-Code:claude-opus-5 Cc: Balbir Singh Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Hugh Dickins Cc: Kairui Song Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Ryan Roberts Cc: Usama Arif --- mm/huge_memory.c | 44 +++++++++++++------------------------------- 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 779c02e0bf6c..c5d11147b69a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3979,41 +3979,27 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n struct folio *end_folio = folio_next(folio); struct folio *new_folio, *next; int old_order = folio_order(folio); - struct list_lru_one *lru; - bool dequeue_deferred; int ret = 0; VM_WARN_ON_ONCE(!mapping && end); - /* - * If this folio can be on the deferred split queue, lock out - * the shrinker before freezing the ref. If the shrinker sees - * a 0-ref folio, it assumes it beat folio_put() to the list - * lock and must clean up the LRU state - the same dequeue we - * will do below as part of the split. - */ - dequeue_deferred = folio_test_anon(folio) && old_order > 1; - if (dequeue_deferred) { - struct mem_cgroup *memcg; - rcu_read_lock(); - memcg = folio_memcg(folio); - lru = list_lru_lock(&deferred_split_lru, - folio_nid(folio), &memcg); - } if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { struct swap_cluster_info *ci = NULL; struct lruvec *lruvec; - if (dequeue_deferred) { - __list_lru_del(&deferred_split_lru, lru, - &folio->_deferred_list, folio_nid(folio)); - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(old_order, - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } - list_lru_unlock(lru); - rcu_read_unlock(); + /* Take off the deferred split queue while frozen and memcg set */ + folio_unqueue_deferred_split(folio); + + /* + * deferred_split_scan() takes the folio off the queue before it + * splits it, so the unqueue above finds an empty list and + * leaves PG_partially_mapped set. + * Clear it here: the flag does not survive the split. + */ + if (folio_test_partially_mapped(folio)) { + folio_clear_partially_mapped(folio); + mod_mthp_stat(old_order, + MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); } if (mapping) { @@ -4115,10 +4101,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n if (ci) swap_cluster_unlock(ci); } else { - if (dequeue_deferred) { - list_lru_unlock(lru); - rcu_read_unlock(); - } return -EAGAIN; } From 7857e0213aad0da2b86f24d6f0698635a585dc30 Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Mon, 31 Aug 2026 21:35:18 +0800 Subject: [PATCH 138/539] mm/hugetlb: preserve source surplus accounting during demotion Patch series "mm/hugetlb: fix surplus accounting and availability checks during demotion", v2. Fix surplus accounting and availability checks in the hugetlb demote path. Patch 1 fixes source hstate accounting when the free folio selected for demotion accounts for a surplus page. Patch 2 prevents demotion from removing free huge pages that back reservations. Both fixes were tested with x86_64 QEMU guests. The commands below use: hstate=/sys/kernel/mm/hugepages/hugepages-1048576kB Patch 1: surplus accounting A vmemmap restoration failure is difficult to trigger deterministically. For this test only, add a one-shot fault injection that makes the first attempt to restore the vmemmap of an optimized 1 GiB folio fail: /* TEST ONLY: fail the first optimized 1G folio restore. */ static atomic_t fail_next_1g_restore = ATOMIC_INIT(1); /* In __hugetlb_vmemmap_restore_folio(). */ if (huge_page_size(h) == SZ_1G && atomic_cmpxchg(&fail_next_1g_restore, 1, 0) == 1) { pr_info("TEST ONLY: forcing one 1G vmemmap " "restore failure\n"); return -ENOMEM; } The injection does not modify the demotion or accounting code. It is one-shot so that the later restore performed during demotion can succeed. 1. Boot QEMU with: hugepagesz=1G hugepages=0 hugetlb_cma=1G hugetlb_free_vmemmap=on 2. Enable overcommit: echo 1 > "$hstate/nr_overcommit_hugepages" 3. Allocate one 1 GiB huge page: nr=1 surplus=1 free=0 resv=0 4. Unmap it. The forced restoration failure leaves the folio on the freelist while it is still accounted as surplus: nr=1 surplus=1 free=1 resv=0 5. Demote one page: echo 1 > "$hstate/demote" Before this fix: nr=0 surplus=1 free=0 resv=0 surplus > nr After this fix: nr=0 surplus=0 free=0 resv=0 Patch 2: cap demotion This reproducer requires no kernel instrumentation. 1. Boot QEMU with: hugepagesz=1G hugepages=2 nr=2 surplus=0 free=2 resv=0 2. Reserve one 1 GiB huge page with an untouched hugetlbfs mapping: nr=2 surplus=0 free=2 resv=1 3. Request demotion of two pages: echo 2 > "$hstate/demote" Before this fix: nr=0 surplus=0 free=0 resv=1 resv > free After this fix: nr=1 surplus=0 free=1 resv=1 resv == free 4. Touch the reserved page and let the process exit. Before this fix, the access fails with SIGBUS and leaves: nr=0 surplus=0 free=0 resv=0 After this fix, the access succeeds and leaves: nr=1 surplus=0 free=1 resv=0 This patch (of 2): demote_pool_huge_page() currently removes every source folio as a persistent folio. A free folio can instead account for one of the source hstate's surplus pages, for example after a vmemmap restoration failure. Removing such a folio without adjusting surplus_huge_pages makes the persistent count underflow, and later subtracting it from max_huge_pages can underflow that counter as well. Classify selected folios against the node's surplus count while holding hugetlb_lock, and preserve that classification on rollback. Track the number of successfully demoted persistent folios separately so only those folios reduce the source max_huge_pages target. All successfully demoted folios still increase the destination target because the new destination folios are added as persistent pages. Testing: Tested on an x86_64 QEMU guest booted with: hugepagesz=1G hugepages=0 hugetlb_cma=1G hugetlb_free_vmemmap=on For testing only, add a one-shot fault injection that makes the first call to __hugetlb_vmemmap_restore_folio() for an optimized 1 GiB folio return -ENOMEM. Set nr_overcommit_hugepages to 1, then allocate one 1 GiB huge page: nr=1 surplus=1 free=0 resv=0 Unmap it. The failed restoration leaves the folio on the freelist while it is still accounted as surplus: nr=1 surplus=1 free=1 resv=0 Demote one page. Before this fix, the result is: nr=0 surplus=1 free=0 resv=0 After this fix, the result is: nr=0 surplus=0 free=0 resv=0 The fault injection is one-shot, so the restore performed during demotion can succeed. Link: https://lore.kernel.org/20260831133519.2505020-2-xialonglong2025@163.com Fixes: 8531fc6f52f5 ("hugetlb: add hugetlb demote page support") Signed-off-by: Longlong Xia Signed-off-by: Andrew Morton Assisted-by: Codex:gpt-5.6-sol Cc: David Hildenbrand Cc: Muchun Song Cc: Oscar Salvador Cc: Yu Zhao --- mm/hugetlb.c | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 76857cdbac1b..71031f9d1587 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4000,6 +4000,7 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, struct hstate *dst; long rc = 0; long nr_demoted = 0; + long nr_persistent = 0; lockdep_assert_held(&hugetlb_lock); @@ -4012,22 +4013,40 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, for_each_node_mask_to_free(src, nr_nodes, node, nodes_allowed) { LIST_HEAD(list); + LIST_HEAD(surplus_list); struct folio *folio, *next; list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node], lru) { + bool adjust_surplus; + if (folio_test_hwpoison(folio)) continue; - remove_hugetlb_folio(src, folio, false); - list_add(&folio->lru, &list); + /* Surplus accounting is maintained per node, not per folio. */ + adjust_surplus = src->surplus_huge_pages_node[node] > 0; + remove_hugetlb_folio(src, folio, adjust_surplus); + list_add(&folio->lru, adjust_surplus ? &surplus_list : &list); + if (!adjust_surplus) + nr_persistent++; if (++nr_demoted == nr_to_demote) break; } + if (list_empty(&list) && list_empty(&surplus_list)) + continue; + spin_unlock_irq(&hugetlb_lock); - rc = demote_free_hugetlb_folios(src, dst, &list); + if (!list_empty(&list)) + rc = demote_free_hugetlb_folios(src, dst, &list); + if (!list_empty(&surplus_list)) { + long tmp_rc; + + tmp_rc = demote_free_hugetlb_folios(src, dst, &surplus_list); + if (rc >= 0) + rc = tmp_rc; + } spin_lock_irq(&hugetlb_lock); @@ -4035,6 +4054,14 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, list_del(&folio->lru); add_hugetlb_folio(src, folio, false); + nr_demoted--; + nr_persistent--; + } + + list_for_each_entry_safe(folio, next, &surplus_list, lru) { + list_del(&folio->lru); + add_hugetlb_folio(src, folio, true); + nr_demoted--; } @@ -4046,7 +4073,7 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, * Not absolutely necessary, but for consistency update max_huge_pages * based on pool changes for the demoted page. */ - src->max_huge_pages -= nr_demoted; + src->max_huge_pages -= nr_persistent; dst->max_huge_pages += nr_demoted << (huge_page_order(src) - huge_page_order(dst)); if (rc < 0) From 3c2ea15046f85c1a006818ebbe8a87b0b3a18b9c Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Mon, 31 Aug 2026 21:35:19 +0800 Subject: [PATCH 139/539] mm/hugetlb: cap demotion at currently available free pages Demotion must not remove free huge pages that back existing reservations. The sysfs path checks whether any page is available, but passes the entire request to demote_pool_huge_page(). For example, with two free pages and one reservation, a request for two pages removes both and leaves the reservation without a backing page. Cap the sysfs request by both global availability and the selected node's free pages. Recheck global availability in demote_pool_huge_page() before each node batch because that function drops hugetlb_lock while restoring vmemmap and reservations can change before the next batch. Testing: Tested on an x86_64 QEMU guest booted with: hugepagesz=1G hugepages=2 Reserve one 1 GiB huge page with an untouched hugetlbfs mapping: nr=2 surplus=0 free=2 resv=1 Request demotion of two pages. Before this fix, both free pages are demoted: nr=0 surplus=0 free=0 resv=1 Touching the reserved mapping then fails with SIGBUS. After this fix, the request is capped at the single available page: nr=1 surplus=0 free=1 resv=1 Touching the reserved mapping succeeds. After the process exits, the counters are: nr=1 surplus=0 free=1 resv=0 Link: https://lore.kernel.org/20260831133519.2505020-3-xialonglong2025@163.com Fixes: c0f398c3b2cf ("mm/hugetlb_vmemmap: batch HVO work when demoting") Signed-off-by: Longlong Xia Signed-off-by: Andrew Morton Assisted-by: Codex:gpt-5.6-sol Cc: David Hildenbrand Cc: Muchun Song Cc: Oscar Salvador Cc: Yu Zhao --- mm/hugetlb.c | 22 +++++++++++++++++++++- mm/hugetlb_sysfs.c | 10 +++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 71031f9d1587..84552c993cbb 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4015,6 +4015,26 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, LIST_HEAD(list); LIST_HEAD(surplus_list); struct folio *folio, *next; + unsigned long nr_available, nr_target; + + /* + * Re-check available each node batch: the previous + * batch released hugetlb_lock for vmemmap restore/split, + * and a new reservation could have been added in that + * window, shrinking the budget. available is global + * (resv is not per-node), so 0 means no node can + * contribute -- stop the whole scan. + */ + nr_available = available_huge_pages(src); + if (!nr_available) + break; + + /* + * Cap this batch at the current budget; expressed as a + * cumulative stop point because nr_demoted is running. + */ + nr_target = nr_demoted + min_t(unsigned long, + nr_to_demote - nr_demoted, nr_available); list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node], lru) { bool adjust_surplus; @@ -4029,7 +4049,7 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, if (!adjust_surplus) nr_persistent++; - if (++nr_demoted == nr_to_demote) + if (++nr_demoted == nr_target) break; } diff --git a/mm/hugetlb_sysfs.c b/mm/hugetlb_sysfs.c index 79ece91406bf..326a54b4d991 100644 --- a/mm/hugetlb_sysfs.c +++ b/mm/hugetlb_sysfs.c @@ -211,15 +211,15 @@ static ssize_t demote_store(struct kobject *kobj, * Check for available pages to demote each time thorough the * loop as demote_pool_huge_page will drop hugetlb_lock. */ + nr_available = h->free_huge_pages - h->resv_huge_pages; if (nid != NUMA_NO_NODE) - nr_available = h->free_huge_pages_node[nid]; - else - nr_available = h->free_huge_pages; - nr_available -= h->resv_huge_pages; + nr_available = min(nr_available, + h->free_huge_pages_node[nid]); if (!nr_available) break; - rc = demote_pool_huge_page(h, n_mask, nr_demote); + rc = demote_pool_huge_page(h, n_mask, + min(nr_demote, nr_available)); if (rc < 0) { err = rc; break; From fbcdf5b52f3c406d8f4e494f3cad3d3f0b0fac62 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 31 Aug 2026 11:13:23 +0530 Subject: [PATCH 140/539] mm: make ptval_to_str() generally available Patch series "mm: Drop pxd_ERROR()". pxd_ERROR() macros have been provided by all platforms, which are very much identical and can be dropped off completely if these pgtable printing could be moved to callers in generic MM aka all pxd_clear_bad(). But first cleanups and re-organizations are required in some platforms that are using these macros internally. Afterwards [pte|pmd|pud|p4d|pgd]_ERROR() macros have been completely dropped from the entire tree. This patch (of 8): Move ptval_to_str() inside a header thus making the helper more generally available for new users which are being added later. While here, also move another related string size macro PTVAL_STR_MAX inside the header as well. Link: https://lore.kernel.org/20260831054331.625505-1-anshuman.khandual@arm.com Link: https://lore.kernel.org/20260831054331.625505-2-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Lorenzo Stoakes Cc: Helge Deller Cc: Huacai Chen Cc: James Bottomley Cc: John Paul Adrian Glaubitz Cc: Lorenzo Stoakes Cc: Rich Felker Cc: Samuel Holland Cc: WANG Xuerui Cc: Yoshinori Sato Cc: Geert Uytterhoeven --- include/linux/pgtable.h | 14 ++++++++++++++ mm/memory.c | 15 +-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 8c093c119e5a..e3c8ab96941c 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -2313,6 +2313,20 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) } } +void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size); + +#define ptval_to_str(buf, val) \ + do { \ + auto __val = (val); \ + \ + ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ + } while (0) + +#if defined(__SIZEOF_INT128__) +#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */ +#else +#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */ +#endif #endif /* !__ASSEMBLER__ */ #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) diff --git a/mm/memory.c b/mm/memory.c index bc14cae3c49d..ec63dd6212ac 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -495,7 +495,7 @@ static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) /* Allow a burst of 60 bad page map reports per minute. */ static DEFINE_RATELIMIT_STATE(bad_page_map_ratelimit, 60 * HZ, 60); -static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size) +void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size) { if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) { snprintf(buf, buf_size, "overflow"); @@ -522,19 +522,6 @@ static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry } } -#define ptval_to_str(buf, val) \ - do { \ - auto __val = (val); \ - \ - ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ - } while (0) - -#if defined(__SIZEOF_INT128__) -#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */ -#else -#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */ -#endif - static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr) { char pgd_str[PTVAL_STR_MAX]; From 0666fc9d44322489dbe03361122164f7701efce3 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 31 Aug 2026 11:13:24 +0530 Subject: [PATCH 141/539] mm: stop using pxd_ERROR() pxd_ERROR() has been used in generic mm just to print the page table entry in pxd_clear_bad() before clearing those out with pxd_clear() later. These pxd_ERROR() macros have been provided by all platforms which basically did the same thing. Make pxd_clear_bad() use recently added ptval_to_str() instead for printing page table entries thus completely dropping dependency on platform provided pxd_ERROR() macros which can then be dropped off later on. Link: https://lore.kernel.org/20260831054331.625505-3-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Lorenzo Stoakes Cc: Geert Uytterhoeven Cc: Helge Deller Cc: Huacai Chen Cc: James Bottomley Cc: John Paul Adrian Glaubitz Cc: Rich Felker Cc: Samuel Holland Cc: WANG Xuerui Cc: Yoshinori Sato --- mm/pgtable-generic.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index b91b1a98029c..224c444cf45d 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -26,14 +26,20 @@ void pgd_clear_bad(pgd_t *pgd) { - pgd_ERROR(*pgd); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, pgd_val(*pgd)); + pr_err("bad pgd %s.\n", str); pgd_clear(pgd); } #ifndef __PAGETABLE_P4D_FOLDED void p4d_clear_bad(p4d_t *p4d) { - p4d_ERROR(*p4d); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, p4d_val(*p4d)); + pr_err("bad p4d %s.\n", str); p4d_clear(p4d); } #endif @@ -41,7 +47,10 @@ void p4d_clear_bad(p4d_t *p4d) #ifndef __PAGETABLE_PUD_FOLDED void pud_clear_bad(pud_t *pud) { - pud_ERROR(*pud); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, pud_val(*pud)); + pr_err("bad pud %s.\n", str); pud_clear(pud); } #endif @@ -53,7 +62,10 @@ void pud_clear_bad(pud_t *pud) */ void pmd_clear_bad(pmd_t *pmd) { - pmd_ERROR(*pmd); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, pmd_val(*pmd)); + pr_err("bad pmd %s.\n", str); pmd_clear(pmd); } From ea8763ac3af75fcf8cb97afff83ecf52757af1dd Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 31 Aug 2026 11:13:25 +0530 Subject: [PATCH 142/539] loongarch/mm: stop using pte_ERROR() Directly use pr_err() in __set_fixmap() and drop pte_ERROR() which helps in eventually dropping pte_ERROR() macro across the tree. In this new printing __FILE__ and __LINE__ has been dropped because they are always the same and don't really add any value. The new ptval_to_str() helper is being used for converting pgtable entry value into a string. The error message itself has been cleaned up as well. Link: https://lore.kernel.org/20260831054331.625505-4-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Huacai Chen Cc: WANG Xuerui Cc: Geert Uytterhoeven Cc: Helge Deller Cc: James Bottomley Cc: John Paul Adrian Glaubitz Cc: Lorenzo Stoakes Cc: Rich Felker Cc: Samuel Holland Cc: Yoshinori Sato --- arch/loongarch/include/asm/pgtable.h | 2 -- arch/loongarch/mm/init.c | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index a05f6a4928dc..eddd8906b77d 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -135,8 +135,6 @@ struct vm_area_struct; #define ptep_get(ptep) READ_ONCE(*(ptep)) #define pmdp_get(pmdp) READ_ONCE(*(pmdp)) -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) #ifndef __PAGETABLE_PMD_FOLDED #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c index 4b46c5d30708..f801f7097f03 100644 --- a/arch/loongarch/mm/init.c +++ b/arch/loongarch/mm/init.c @@ -197,13 +197,15 @@ void __init __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t flags) { unsigned long addr = __fix_to_virt(idx); + char str[PTVAL_STR_MAX]; pte_t *ptep; BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); ptep = populate_kernel_pte(addr); if (!pte_none(ptep_get(ptep))) { - pte_ERROR(*ptep); + ptval_to_str(str, pte_val(*ptep)); + pr_err("unexpected set PTE at %lx in %s: %s\n", addr, __func__, str); return; } From 94d6c5466d1aaef27242c81e75bb70f97184b06e Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 31 Aug 2026 11:13:26 +0530 Subject: [PATCH 143/539] parisc/mm: directly use generic [pmd|pgd]_clear_bad() Drop [pmd|pgd]_ERROR() followed by [pmd|pgd]_clear() instances. But instead directly use semantically equivalent generic helpers [pmd|pgd]_clear_bad() in unmap_uncached_[pte|pmd]() which helps in dropping their corresponding [pmd|pgd]_ERROR() macros across the tree. Link: https://lore.kernel.org/20260831054331.625505-5-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: James E.J. Bottomley Cc: Helge Deller Cc: Geert Uytterhoeven Cc: Huacai Chen Cc: John Paul Adrian Glaubitz Cc: Lorenzo Stoakes Cc: Rich Felker Cc: Samuel Holland Cc: WANG Xuerui Cc: Yoshinori Sato --- arch/parisc/kernel/pci-dma.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index bf9f192c826e..84e7309a8266 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -160,8 +160,7 @@ static inline void unmap_uncached_pte(pmd_t * pmd, unsigned long vaddr, if (pmd_none(*pmd)) return; if (pmd_bad(*pmd)) { - pmd_ERROR(*pmd); - pmd_clear(pmd); + pmd_clear_bad(pmd); return; } pte = pte_offset_kernel(pmd, vaddr); @@ -196,8 +195,7 @@ static inline void unmap_uncached_pmd(pgd_t * dir, unsigned long vaddr, if (pgd_none(*dir)) return; if (pgd_bad(*dir)) { - pgd_ERROR(*dir); - pgd_clear(dir); + pgd_clear_bad(dir); return; } pmd = pmd_offset(pud_offset(p4d_offset(dir, vaddr), vaddr), vaddr); From 6858e42cc2bcc1abf8b926ddabba2000c9408c9d Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 31 Aug 2026 11:13:27 +0530 Subject: [PATCH 144/539] sh/mm: stop using pte_ERROR() Directly use pr_err() in set_pte_phys() and drop pte_ERROR() which helps in eventually dropping pte_ERROR() macro across the tree. In this new printing __FILE__ and __LINE__ has been dropped because they are always the same and don't really add any value. Besides ptrval_to_str() has been able to handle different PTE representation with and without CONFIG_X2TLB, which helped in unifying error message printing. Link: https://lore.kernel.org/20260831054331.625505-6-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Yoshinori Sato Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Geert Uytterhoeven Cc: Helge Deller Cc: Huacai Chen Cc: James Bottomley Cc: Lorenzo Stoakes Cc: Samuel Holland Cc: WANG Xuerui --- arch/sh/include/asm/pgtable_32.h | 5 ----- arch/sh/mm/init.c | 6 +++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h index 5f51af18997b..c8eb9a7a4c4c 100644 --- a/arch/sh/include/asm/pgtable_32.h +++ b/arch/sh/include/asm/pgtable_32.h @@ -401,14 +401,9 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) #define pmd_page(pmd) (virt_to_page(pmd_val(pmd))) #ifdef CONFIG_X2TLB -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %p(%08lx%08lx).\n", __FILE__, __LINE__, \ - &(e), (e).pte_high, (e).pte_low) #define pgd_ERROR(e) \ printk("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) #else -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) #define pgd_ERROR(e) \ printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) #endif diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 110308bdef01..9466ae6f9f16 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -84,7 +84,11 @@ static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot) pte = __get_pte_phys(addr); if (!pte_none(*pte)) { - pte_ERROR(*pte); + char str[PTVAL_STR_MAX]; + + ptval_to_str(str, pte_val(*pte)); + pr_err("unexpected set PTE at %lx in %s: bad pte %p(%s).\n", + addr, __func__, pte, str); return; } From 3b225ce37f627b2e55fcdd6def5b8c5a7c87fc02 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 31 Aug 2026 11:13:28 +0530 Subject: [PATCH 145/539] sh/mm: stop using [p4d|pud|pmd]_ERROR() Stop using [p4d|pud|pmd]_ERROR() in __get_pte_phys() as the pgtable entries are known to be NULL and hence could not really be accessed. Link: https://lore.kernel.org/20260831054331.625505-7-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Yoshinori Sato Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Geert Uytterhoeven Cc: Helge Deller Cc: Huacai Chen Cc: James Bottomley Cc: Lorenzo Stoakes Cc: Samuel Holland Cc: WANG Xuerui --- arch/sh/mm/init.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 9466ae6f9f16..8d65e60688dc 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -59,19 +59,19 @@ static pte_t *__get_pte_phys(unsigned long addr) p4d = p4d_alloc(NULL, pgd, addr); if (unlikely(!p4d)) { - p4d_ERROR(*p4d); + pr_err("allocating p4d table failed\n"); return NULL; } pud = pud_alloc(NULL, p4d, addr); if (unlikely(!pud)) { - pud_ERROR(*pud); + pr_err("allocating pud table failed\n"); return NULL; } pmd = pmd_alloc(NULL, pud, addr); if (unlikely(!pmd)) { - pmd_ERROR(*pmd); + pr_err("allocating pmd table failed\n"); return NULL; } From 66a0441a1791d70ac2dcd1b2deab9a1b9438fffe Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 31 Aug 2026 11:13:29 +0530 Subject: [PATCH 146/539] sh/mm: stop using pgd_ERROR() Stop using pgd_ERROR() in __get_pte_phys() when page table entry is already known to be empty. Link: https://lore.kernel.org/20260831054331.625505-8-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Yoshinori Sato Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Geert Uytterhoeven Cc: Helge Deller Cc: Huacai Chen Cc: James Bottomley Cc: Lorenzo Stoakes Cc: Samuel Holland Cc: WANG Xuerui --- arch/sh/mm/init.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 8d65e60688dc..93921109f4e6 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -52,10 +52,8 @@ static pte_t *__get_pte_phys(unsigned long addr) pmd_t *pmd; pgd = pgd_offset_k(addr); - if (pgd_none(*pgd)) { - pgd_ERROR(*pgd); + if (pgd_none(*pgd)) return NULL; - } p4d = p4d_alloc(NULL, pgd, addr); if (unlikely(!p4d)) { From c970236c300f9c2a32832cfd685f0f4f20a4a683 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 31 Aug 2026 11:13:30 +0530 Subject: [PATCH 147/539] mm: drop pxd_ERROR() There are no more users left for any pxd_ERROR() either in generic MM or in the platform MM. Hence all these platform macros along with their generic fallback could be dropped across the tree. Link: https://lore.kernel.org/20260831054331.625505-9-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Acked-by: Geert Uytterhoeven # m68k Acked-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Helge Deller Cc: Huacai Chen Cc: James Bottomley Cc: John Paul Adrian Glaubitz Cc: Lorenzo Stoakes Cc: Rich Felker Cc: Samuel Holland Cc: WANG Xuerui Cc: Yoshinori Sato --- arch/alpha/include/asm/pgtable.h | 7 ------- arch/arc/include/asm/pgtable-levels.h | 11 ----------- arch/arm/include/asm/pgtable.h | 7 ------- arch/arm/kernel/traps.c | 17 ----------------- arch/arm64/include/asm/pgtable.h | 15 --------------- arch/csky/include/asm/pgtable.h | 4 ---- arch/hexagon/include/asm/pgtable.h | 3 --- arch/loongarch/include/asm/pgtable.h | 11 ----------- arch/m68k/include/asm/mcf_pgtable.h | 6 ------ arch/m68k/include/asm/motorola_pgtable.h | 8 -------- arch/m68k/include/asm/sun3_pgtable.h | 7 ------- arch/microblaze/include/asm/pgtable.h | 7 ------- arch/mips/include/asm/pgtable-32.h | 10 ---------- arch/mips/include/asm/pgtable-64.h | 13 ------------- arch/nios2/include/asm/pgtable.h | 7 ------- arch/openrisc/include/asm/pgtable.h | 7 ------- arch/parisc/include/asm/pgtable.h | 9 --------- arch/powerpc/include/asm/book3s/32/pgtable.h | 2 -- arch/powerpc/include/asm/book3s/64/pgtable.h | 7 ------- arch/powerpc/include/asm/nohash/32/pgtable.h | 2 -- .../powerpc/include/asm/nohash/64/pgtable-4k.h | 3 --- arch/powerpc/include/asm/nohash/64/pgtable.h | 5 ----- arch/riscv/include/asm/page.h | 6 ------ arch/riscv/include/asm/pgtable-64.h | 9 --------- arch/riscv/include/asm/pgtable.h | 4 ---- arch/s390/include/asm/pgtable.h | 11 ----------- arch/sh/include/asm/pgtable-3level.h | 3 --- arch/sh/include/asm/pgtable_32.h | 8 -------- arch/sparc/include/asm/pgtable_32.h | 3 --- arch/sparc/include/asm/pgtable_64.h | 10 ---------- arch/um/include/asm/pgtable-2level.h | 7 ------- arch/um/include/asm/pgtable-4level.h | 13 ------------- arch/x86/include/asm/pgtable-2level.h | 5 ----- arch/x86/include/asm/pgtable-3level.h | 11 ----------- arch/x86/include/asm/pgtable_64.h | 18 ------------------ arch/xtensa/include/asm/pgtable.h | 4 ---- include/asm-generic/pgtable-nop4d.h | 1 - include/asm-generic/pgtable-nopmd.h | 1 - include/asm-generic/pgtable-nopud.h | 1 - 39 files changed, 283 deletions(-) diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 8e00cf9dc39d..7cac8241ee67 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -357,13 +357,6 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return pte; } -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) - extern void paging_init(void); /* We have our own get_unmapped_area */ diff --git a/arch/arc/include/asm/pgtable-levels.h b/arch/arc/include/asm/pgtable-levels.h index c8f9273372c0..167b82fcfafe 100644 --- a/arch/arc/include/asm/pgtable-levels.h +++ b/arch/arc/include/asm/pgtable-levels.h @@ -98,8 +98,6 @@ /* * 1st level paging: pgd */ -#define pgd_ERROR(e) \ - pr_crit("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) #if CONFIG_PGTABLE_LEVELS > 3 @@ -115,9 +113,6 @@ /* * 2nd level paging: pud */ -#define pud_ERROR(e) \ - pr_crit("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) - #endif #if CONFIG_PGTABLE_LEVELS > 2 @@ -137,9 +132,6 @@ /* * 3rd level paging: pmd */ -#define pmd_ERROR(e) \ - pr_crit("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) - #define pmd_pfn(pmd) ((pmd_val(pmd) & PMD_MASK) >> PAGE_SHIFT) #define pfn_pmd(pfn,prot) __pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) @@ -165,9 +157,6 @@ /* * 4th level paging: pte */ -#define pte_ERROR(e) \ - pr_crit("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) - #define PFN_PTE_SHIFT PAGE_SHIFT #define pte_none(x) (!pte_val(x)) #define pte_present(x) (pte_val(x) & _PAGE_PRESENT) diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 982795cf4563..8dd17d20faa3 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -44,13 +44,6 @@ #define LIBRARY_TEXT_START 0x0c000000 #ifndef __ASSEMBLY__ -extern void __pte_error(const char *file, int line, pte_t); -extern void __pmd_error(const char *file, int line, pmd_t); -extern void __pgd_error(const char *file, int line, pgd_t); - -#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte) -#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd) -#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd) /* * This is the lowest virtual address we can permit any user space diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index afbd2ebe5c39..ad04c806cc9d 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -753,23 +753,6 @@ void __readwrite_bug(const char *fn) } EXPORT_SYMBOL(__readwrite_bug); -#ifdef CONFIG_MMU -void __pte_error(const char *file, int line, pte_t pte) -{ - pr_err("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte)); -} - -void __pmd_error(const char *file, int line, pmd_t pmd) -{ - pr_err("%s:%d: bad pmd %08llx.\n", file, line, (long long)pmd_val(pmd)); -} - -void __pgd_error(const char *file, int line, pgd_t pgd) -{ - pr_err("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd)); -} -#endif - asmlinkage void __div0(void) { pr_err("Division by zero in kernel.\n"); diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 6000905a2e86..e89ec5f4787b 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -107,9 +107,6 @@ static inline void arch_leave_lazy_mmu_mode(void) __flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2, \ TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE) -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e)) - #ifdef CONFIG_ARM64_PA_BITS_52 static inline phys_addr_t __pte_to_phys(pte_t pte) { @@ -866,9 +863,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) #if CONFIG_PGTABLE_LEVELS > 2 -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) - #define pud_none(pud) (!pud_val(pud)) #define pud_bad(pud) ((pud_val(pud) & PUD_TYPE_MASK) != \ PUD_TYPE_TABLE) @@ -960,9 +954,6 @@ static inline bool mm_pud_folded(const struct mm_struct *mm) } #define mm_pud_folded mm_pud_folded -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e)) - #define p4d_none(p4d) (pgtable_l4_enabled() && !p4d_val(p4d)) #define p4d_bad(p4d) (pgtable_l4_enabled() && \ ((p4d_val(p4d) & P4D_TYPE_MASK) != \ @@ -1088,9 +1079,6 @@ static inline bool mm_p4d_folded(const struct mm_struct *mm) } #define mm_p4d_folded mm_p4d_folded -#define p4d_ERROR(e) \ - pr_err("%s:%d: bad p4d %016llx.\n", __FILE__, __LINE__, p4d_val(e)) - #define pgd_none(pgd) (pgtable_l5_enabled() && !pgd_val(pgd)) #define pgd_bad(pgd) (pgtable_l5_enabled() && \ ((pgd_val(pgd) & PGD_TYPE_MASK) != \ @@ -1217,9 +1205,6 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) #endif /* CONFIG_PGTABLE_LEVELS > 4 */ -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) - #define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) #define pgd_clear_fixmap() clear_fixmap(FIX_PGD) diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h index bafcd5823531..5ca77ff89ef9 100644 --- a/arch/csky/include/asm/pgtable.h +++ b/arch/csky/include/asm/pgtable.h @@ -23,10 +23,6 @@ #define PTRS_PER_PMD 1 #define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t)) -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, (e).pte_low) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) #define PFN_PTE_SHIFT PAGE_SHIFT #define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT) diff --git a/arch/hexagon/include/asm/pgtable.h b/arch/hexagon/include/asm/pgtable.h index 27b269e2870d..2fdb27afe703 100644 --- a/arch/hexagon/include/asm/pgtable.h +++ b/arch/hexagon/include/asm/pgtable.h @@ -94,9 +94,6 @@ #endif /* Any bigger and the PTE disappears. */ -#define pgd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__,\ - pgd_val(e)) /* * Page Protection Constants. Includes (in this variant) cache attributes. diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index eddd8906b77d..cf29a4c8ac59 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -135,17 +135,6 @@ struct vm_area_struct; #define ptep_get(ptep) READ_ONCE(*(ptep)) #define pmdp_get(pmdp) READ_ONCE(*(pmdp)) -#ifndef __PAGETABLE_PMD_FOLDED -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) -#endif -#ifndef __PAGETABLE_PUD_FOLDED -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) -#endif -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) - extern pte_t invalid_pte_table[PTRS_PER_PTE]; #ifndef __PAGETABLE_PUD_FOLDED diff --git a/arch/m68k/include/asm/mcf_pgtable.h b/arch/m68k/include/asm/mcf_pgtable.h index 189bb7b1e663..f45a882238db 100644 --- a/arch/m68k/include/asm/mcf_pgtable.h +++ b/arch/m68k/include/asm/mcf_pgtable.h @@ -137,12 +137,6 @@ static inline int pmd_bad2(pmd_t *pmd) { return 0; } #define pmd_present(pmd) (!pmd_none2(&(pmd))) static inline void pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = 0; } -#define pte_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pte %08lx.\n", \ - __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", \ - __FILE__, __LINE__, pgd_val(e)) /* * The following only work if pte_present() is true. diff --git a/arch/m68k/include/asm/motorola_pgtable.h b/arch/m68k/include/asm/motorola_pgtable.h index dcf6829b3eab..d9393b310add 100644 --- a/arch/m68k/include/asm/motorola_pgtable.h +++ b/arch/m68k/include/asm/motorola_pgtable.h @@ -131,14 +131,6 @@ static inline void pud_set(pud_t *pudp, pmd_t *pmdp) #define pud_clear(pudp) ({ pud_val(*pudp) = 0; }) #define pud_page(pud) (mem_map + ((unsigned long)(__va(pud_val(pud)) - PAGE_OFFSET) >> PAGE_SHIFT)) -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - - /* * The following only work if pte_present() is true. * Undefined behaviour if not.. diff --git a/arch/m68k/include/asm/sun3_pgtable.h b/arch/m68k/include/asm/sun3_pgtable.h index 80ca185a18a1..704442a391fd 100644 --- a/arch/m68k/include/asm/sun3_pgtable.h +++ b/arch/m68k/include/asm/sun3_pgtable.h @@ -119,13 +119,6 @@ static inline int pmd_present2 (pmd_t *pmd) { return pmd_val (*pmd) & SUN3_PMD_V #define pmd_present(pmd) (!pmd_none2(&(pmd))) static inline void pmd_clear (pmd_t *pmdp) { pmd_val (*pmdp) = 0; } - -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - - /* * The following only work if pte_present() is true. * Undefined behaviour if not... diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h index 7678c040a2fd..72708f9af1c0 100644 --- a/arch/microblaze/include/asm/pgtable.h +++ b/arch/microblaze/include/asm/pgtable.h @@ -103,13 +103,6 @@ extern pte_t *va_to_pte(unsigned long address); #define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT) #define KERNEL_PGD_PTRS (PTRS_PER_PGD-USER_PGD_PTRS) -#define pte_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pte "PTE_FMT".\n", \ - __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", \ - __FILE__, __LINE__, pgd_val(e)) - /* * Bits in a linux-style PTE. These match the bits in the * (hardware-defined) PTE as closely as possible. diff --git a/arch/mips/include/asm/pgtable-32.h b/arch/mips/include/asm/pgtable-32.h index 92b7591aac2a..ef1001ab09c5 100644 --- a/arch/mips/include/asm/pgtable-32.h +++ b/arch/mips/include/asm/pgtable-32.h @@ -104,16 +104,6 @@ extern int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, # define VMALLOC_END (FIXADDR_START-2*PAGE_SIZE) #endif -#ifdef CONFIG_PHYS_ADDR_T_64BIT -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %016Lx.\n", __FILE__, __LINE__, pte_val(e)) -#else -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#endif -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - extern void load_pgd(unsigned long pg_dir); extern pte_t invalid_pte_table[PTRS_PER_PTE]; diff --git a/arch/mips/include/asm/pgtable-64.h b/arch/mips/include/asm/pgtable-64.h index 6e854bb11f37..785fc37bab94 100644 --- a/arch/mips/include/asm/pgtable-64.h +++ b/arch/mips/include/asm/pgtable-64.h @@ -151,19 +151,6 @@ #define MODULES_END (FIXADDR_START-2*PAGE_SIZE) #endif -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) -#ifndef __PAGETABLE_PMD_FOLDED -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) -#endif -#ifndef __PAGETABLE_PUD_FOLDED -#define pud_ERROR(e) \ - printk("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) -#endif -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) - extern pte_t invalid_pte_table[PTRS_PER_PTE]; #ifndef __PAGETABLE_PUD_FOLDED diff --git a/arch/nios2/include/asm/pgtable.h b/arch/nios2/include/asm/pgtable.h index d389aa9ca57c..272707d48f1b 100644 --- a/arch/nios2/include/asm/pgtable.h +++ b/arch/nios2/include/asm/pgtable.h @@ -223,13 +223,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) return pmd_val(pmd); } -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx.\n", \ - __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", \ - __FILE__, __LINE__, pgd_val(e)) - /* * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that * are !pte_none() && !pte_present(). diff --git a/arch/openrisc/include/asm/pgtable.h b/arch/openrisc/include/asm/pgtable.h index 6b89996d0b62..13afcc0bd863 100644 --- a/arch/openrisc/include/asm/pgtable.h +++ b/arch/openrisc/include/asm/pgtable.h @@ -338,13 +338,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) #define pte_pfn(x) ((unsigned long)(((x).pte)) >> PAGE_SHIFT) #define pfn_pte(pfn, prot) __pte((((pfn) << PAGE_SHIFT)) | pgprot_val(prot)) -#define pte_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pte %p(%08lx).\n", \ - __FILE__, __LINE__, &(e), pte_val(e)) -#define pgd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pgd %p(%08lx).\n", \ - __FILE__, __LINE__, &(e), pgd_val(e)) - extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; /* defined in head.S */ struct vm_area_struct; diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index 467b8547ac8b..f6899375cb43 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -75,15 +75,6 @@ extern void __update_cache(pte_t pte); #endif /* !__ASSEMBLER__ */ -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#if CONFIG_PGTABLE_LEVELS == 3 -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, (unsigned long)pmd_val(e)) -#endif -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, (unsigned long)pgd_val(e)) - /* This is the size of the initially mapped kernel memory */ #if defined(CONFIG_64BIT) || defined(CONFIG_KALLSYMS) #define KERNEL_INITIAL_ORDER 26 /* 1<<26 = 64MB */ diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index e18a4fa282a1..835e84caee13 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -203,8 +203,6 @@ void unmap_kernel_page(unsigned long va); /* Bits to mask out from a PGD to get to the PUD page */ #define PGD_MASKED_BITS 0 -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) /* * Bits in a linux-style PTE. These match the bits in the * (hardware-defined) PowerPC PTE as closely as possible. diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index f4db7d7fbd5c..dff8790a047d 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -991,13 +991,6 @@ static inline pmd_t *pud_pgtable(pud_t pud) return (pmd_t *)__va(pud_val(pud) & ~PUD_MASKED_BITS); } -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - static inline int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) { if (radix_enabled()) { diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index 496ecc65ac25..f17afde89fa1 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -51,8 +51,6 @@ #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08llx.\n", __FILE__, __LINE__, (unsigned long long)pgd_val(e)) /* * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary diff --git a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h index fb6fa1d4e074..75cf3c331b92 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h @@ -82,9 +82,6 @@ extern struct page *p4d_page(p4d_t p4d); #endif /* !__ASSEMBLER__ */ -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) - /* * On all 4K setups, remap_4k_pfn() equates to remap_pfn_range() */ #define remap_4k_pfn(vma, addr, pfn, prot) \ diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 661eb3820d12..446dde8b6ead 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -159,11 +159,6 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __young; \ }) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - /* * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that * are !pte_none() && !pte_present(). diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 709a36fb4323..b4bbae55e931 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -76,12 +76,6 @@ typedef struct page *pgtable_t; #define __pgd(x) ((pgd_t) { (x) }) #define __pgprot(x) ((pgprot_t) { (x) }) -#ifdef CONFIG_64BIT -#define PTE_FMT "%016lx" -#else -#define PTE_FMT "%08lx" -#endif - #if defined(CONFIG_64BIT) && defined(CONFIG_MMU) /* * We override this value as its generic definition uses __pa too early in diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 6e789fa58514..ae23182b572c 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -264,15 +264,6 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) return __page_val_to_pfn(pmd_val(pmd)); } -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) - -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) - -#define p4d_ERROR(e) \ - pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) - static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) { if (pgtable_l4_enabled) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 40b1ed4f3ea8..4c8fc6845503 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -556,10 +556,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) return __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val); } -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e)) - - /* Commit new configuration to MMU hardware */ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index e882663a58e7..2d5c2ab06de9 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -68,17 +68,6 @@ extern unsigned long zero_page_mask; /* TODO: s390 cannot support io_remap_pfn_range... */ -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) -#define p4d_ERROR(e) \ - pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) - /* * The vmalloc and module area will always be on the topmost area of the * kernel mapping. 512GB are reserved for vmalloc by default. diff --git a/arch/sh/include/asm/pgtable-3level.h b/arch/sh/include/asm/pgtable-3level.h index d1ce73f3bd85..3f4d747f30f4 100644 --- a/arch/sh/include/asm/pgtable-3level.h +++ b/arch/sh/include/asm/pgtable-3level.h @@ -25,9 +25,6 @@ #define PTRS_PER_PMD ((1 << PGDIR_SHIFT) / PMD_SIZE) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) - typedef union { struct { unsigned long pmd_low; diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h index c8eb9a7a4c4c..cde1bf0c6734 100644 --- a/arch/sh/include/asm/pgtable_32.h +++ b/arch/sh/include/asm/pgtable_32.h @@ -400,14 +400,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) #define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT) #define pmd_page(pmd) (virt_to_page(pmd_val(pmd))) -#ifdef CONFIG_X2TLB -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) -#else -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) -#endif - /* * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that * are !pte_none() && !pte_present(). diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index f89b1250661d..5a5f54a090f5 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -40,9 +40,6 @@ void load_mmu(void); unsigned long calc_highpages(void); unsigned long __init bootmem_init(unsigned long *pages_avail); -#define pte_ERROR(e) __builtin_trap() -#define pmd_ERROR(e) __builtin_trap() -#define pgd_ERROR(e) __builtin_trap() #define PTRS_PER_PTE 64 #define PTRS_PER_PMD 64 diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 0837ebbc5dce..44d1333065a6 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -96,16 +96,6 @@ bool kern_addr_valid(unsigned long addr); #define PTRS_PER_PUD (1UL << PUD_BITS) #define PTRS_PER_PGD (1UL << PGDIR_BITS) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \ - __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0)) -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n", \ - __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \ - __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0)) - #endif /* !(__ASSEMBLER__) */ /* PTE bits which are the same in SUN4U and SUN4V format. */ diff --git a/arch/um/include/asm/pgtable-2level.h b/arch/um/include/asm/pgtable-2level.h index 14ec16f92ce4..fa625f5b5ef7 100644 --- a/arch/um/include/asm/pgtable-2level.h +++ b/arch/um/include/asm/pgtable-2level.h @@ -24,13 +24,6 @@ #define USER_PTRS_PER_PGD ((TASK_SIZE + (PGDIR_SIZE - 1)) / PGDIR_SIZE) #define PTRS_PER_PGD 1024 -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %p(%08lx).\n", __FILE__, __LINE__, &(e), \ - pte_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %p(%08lx).\n", __FILE__, __LINE__, &(e), \ - pgd_val(e)) - static inline int pgd_needsync(pgd_t pgd) { return 0; } static inline void pgd_mkuptodate(pgd_t pgd) { } diff --git a/arch/um/include/asm/pgtable-4level.h b/arch/um/include/asm/pgtable-4level.h index 7a271b7b83d2..ff82f99c80fa 100644 --- a/arch/um/include/asm/pgtable-4level.h +++ b/arch/um/include/asm/pgtable-4level.h @@ -42,19 +42,6 @@ #define USER_PTRS_PER_PGD ((TASK_SIZE + (PGDIR_SIZE - 1)) / PGDIR_SIZE) -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), \ - pte_val(e)) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), \ - pmd_val(e)) -#define pud_ERROR(e) \ - printk("%s:%d: bad pud %p(%016lx).\n", __FILE__, __LINE__, &(e), \ - pud_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), \ - pgd_val(e)) - #define pud_none(x) (!(pud_val(x) & ~_PAGE_NEEDSYNC)) #define pud_bad(x) ((pud_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE) #define pud_present(x) (pud_val(x) & _PAGE_PRESENT) diff --git a/arch/x86/include/asm/pgtable-2level.h b/arch/x86/include/asm/pgtable-2level.h index e9482a11ac52..83427765cfbf 100644 --- a/arch/x86/include/asm/pgtable-2level.h +++ b/arch/x86/include/asm/pgtable-2level.h @@ -2,11 +2,6 @@ #ifndef _ASM_X86_PGTABLE_2LEVEL_H #define _ASM_X86_PGTABLE_2LEVEL_H -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx\n", __FILE__, __LINE__, (e).pte_low) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %08lx\n", __FILE__, __LINE__, pgd_val(e)) - /* * Certain architectures need to do special things when PTEs * within a page table are directly modified. Thus, the following diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h index dabafba957ea..d6729911e09a 100644 --- a/arch/x86/include/asm/pgtable-3level.h +++ b/arch/x86/include/asm/pgtable-3level.h @@ -8,17 +8,6 @@ * * Copyright (C) 1999 Ingo Molnar */ - -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %p(%08lx%08lx)\n", \ - __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %p(%016Lx)\n", \ - __FILE__, __LINE__, &(e), pmd_val(e)) -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %p(%016Lx)\n", \ - __FILE__, __LINE__, &(e), pgd_val(e)) - #define pxx_xchg64(_pxx, _ptr, _val) ({ \ _pxx##val_t *_p = (_pxx##val_t *)_ptr; \ _pxx##val_t _o = *_p; \ diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index ce45882ccd07..c861f3832bed 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -29,24 +29,6 @@ extern pgd_t init_top_pgt[]; extern void paging_init(void); static inline void sync_initial_page_table(void) { } -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pte_val(e)) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pmd_val(e)) -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pud_val(e)) - -#define p4d_ERROR(e) \ - pr_err("%s:%d: bad p4d %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), p4d_val(e)) - -#define pgd_ERROR(e) \ - pr_err("%s:%d: bad pgd %p(%016lx)\n", \ - __FILE__, __LINE__, &(e), pgd_val(e)) - struct mm_struct; #define mm_p4d_folded mm_p4d_folded diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h index f00a879dc298..60fb67a99729 100644 --- a/arch/xtensa/include/asm/pgtable.h +++ b/arch/xtensa/include/asm/pgtable.h @@ -204,10 +204,6 @@ */ #ifndef __ASSEMBLER__ -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd entry %08lx.\n", __FILE__, __LINE__, pgd_val(e)) #ifdef CONFIG_MMU extern pgd_t swapper_pg_dir[PAGE_SIZE/sizeof(pgd_t)]; diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h index 89c21f84cffb..1cf739ee38aa 100644 --- a/include/asm-generic/pgtable-nop4d.h +++ b/include/asm-generic/pgtable-nop4d.h @@ -22,7 +22,6 @@ static inline int pgd_none(pgd_t pgd) { return 0; } static inline int pgd_bad(pgd_t pgd) { return 0; } static inline int pgd_present(pgd_t pgd) { return 1; } static inline void pgd_clear(pgd_t *pgd) { } -#define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd)) #define pgd_populate(mm, pgd, p4d) do { } while (0) #define pgd_populate_safe(mm, pgd, p4d) do { } while (0) diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index 36b6490ed180..ff4235cf84d7 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -33,7 +33,6 @@ static inline int pud_present(pud_t pud) { return 1; } static inline int pud_user(pud_t pud) { return 0; } static inline int pud_leaf(pud_t pud) { return 0; } static inline void pud_clear(pud_t *pud) { } -#define pmd_ERROR(pmd) (pud_ERROR((pmd).pud)) #define pud_populate(mm, pmd, pte) do { } while (0) diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index 356cbfbaab24..eedee8e3ad68 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -29,7 +29,6 @@ static inline int p4d_none(p4d_t p4d) { return 0; } static inline int p4d_bad(p4d_t p4d) { return 0; } static inline int p4d_present(p4d_t p4d) { return 1; } static inline void p4d_clear(p4d_t *p4d) { } -#define pud_ERROR(pud) (p4d_ERROR((pud).p4d)) #define p4d_populate(mm, p4d, pud) do { } while (0) #define p4d_populate_safe(mm, p4d, pud) do { } while (0) From 56b351d00a824598acf7b99fd59e518e78658e6f Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Sun, 30 Aug 2026 12:29:17 +0800 Subject: [PATCH 148/539] mm: add page_counter_margin() Patch series "mm: avoid large folio splits when swap is unavailable", v7. This is v7 of Barry's original RFC patch, "mm: Avoiding split large folios if swap has no space": https://lore.kernel.org/r/20260618221720.71768-1-baohua@kernel.org Barry's RFC showed the no-swap case with MADV_PAGEOUT on 16KB mTHP: the large-folio split counter increased by 1024 even though no swapout progress was possible. Skipping the split in that case kept the counter at 0. This series makes folio_alloc_swap() classify failures according to whether splitting a large folio might allow swapout to make progress. Callers can then avoid destroying the large folio when neither global swap availability nor the folio's memcg swap hierarchy has capacity for even a smaller folio. Patch #1 adds page_counter_margin(), a small helper that computes the minimum remaining chargeable space across a page_counter hierarchy. Patch #2 establishes the folio_alloc_swap() return-value contract: - -E2BIG: splitting may let smaller folios make progress - -ENOSPC: no global swap space is available - -ENOMEM: splitting is not expected to help, including when the folio's memcg swap hierarchy has no remaining capacity Patch #3 makes vmscan split a large folio only when folio_alloc_swap() returns -E2BIG. Other failures keep the existing activation path and avoid destroying the large folio when no smaller part can be backed by swap either. Patch #4 applies the same contract to shmem_writeout(), which currently splits a large folio on every folio_alloc_swap() failure. It now enters the split fallback only on -E2BIG; other failures redirty and reactivate the folio as before. Testing: With a 1GB anonymous mapping backed by 16KB mTHPs and memory.swap.max=0, the patch reduced the median latency of 30 process_madvise(MADV_PAGEOUT) runs from 743.8 ms to 181.7 ms, while the number of large-folio splits per run dropped from 65536 to 0. Neither kernel swapped out any pages. I also ran DaCapo h2 under swap pressure and found no statistically significant change in wall time or CPU time. The overall benefit appears minor and workload-dependent. This patch (of 4): mem_cgroup_get_nr_swap_pages() open-codes the remaining capacity across the memcg swap counter hierarchy. Add page_counter_margin() to return the minimum usable space from a page counter to the root, and use it in mem_cgroup_get_nr_swap_pages(). This is a pure refactoring with no intended behavior change. Link: https://lore.kernel.org/20260830042920.2280454-1-xueyuan.chen21@gmail.com Link: https://lore.kernel.org/20260830042920.2280454-2-xueyuan.chen21@gmail.com Signed-off-by: Johannes Weiner Signed-off-by: Xueyuan Chen Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Barry Song Cc: Baolin Wang Cc: Baoquan He Cc: Chris Li Cc: Hugh Dickins Cc: Kairui Song Cc: Kemeng Shi Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Nhat Pham Cc: Roman Gushchin Cc: Shakeel Butt Cc: Nanzhe Zhao Cc: Youngjun Park --- include/linux/page_counter.h | 1 + mm/memcontrol.c | 9 +++------ mm/page_counter.c | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index d649b6bbbc87..07b7cb12249c 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -68,6 +68,7 @@ static inline unsigned long page_counter_read(struct page_counter *counter) return atomic_long_read(&counter->usage); } +long page_counter_margin(struct page_counter *counter); void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages); void page_counter_charge(struct page_counter *counter, unsigned long nr_pages); bool page_counter_try_charge(struct page_counter *counter, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index aeaa09e01d70..7f63bf9e8ef1 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5832,12 +5832,9 @@ long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) { long nr_swap_pages = get_nr_swap_pages(); - if (mem_cgroup_disabled() || do_memsw_account()) - return nr_swap_pages; - for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) - nr_swap_pages = min_t(long, nr_swap_pages, - READ_ONCE(memcg->swap.max) - - page_counter_read(&memcg->swap)); + if (!mem_cgroup_disabled() && !do_memsw_account()) + nr_swap_pages = min(nr_swap_pages, page_counter_margin(&memcg->swap)); + return nr_swap_pages; } diff --git a/mm/page_counter.c b/mm/page_counter.c index 661e0f2a5127..450543f4b318 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -46,6 +46,26 @@ static void propagate_protected_usage(struct page_counter *c, } } +/** + * page_counter_margin - remaining usable space within hierarchical limits + * @counter: counter + * + * Return: The minimum value of max minus usage across @counter and all of + * its ancestors. The value may be negative during a concurrent charge. + */ +long page_counter_margin(struct page_counter *counter) +{ + long margin = PAGE_COUNTER_MAX; + + do { + long m = READ_ONCE(counter->max) - page_counter_read(counter); + + margin = min(margin, m); + } while ((counter = counter->parent)); + + return margin; +} + /** * page_counter_cancel - take pages out of the local counter * @counter: counter From c50c376e1fd69c9dd9f4908eadc757490fc884a9 Mon Sep 17 00:00:00 2001 From: Xueyuan Chen Date: Sun, 30 Aug 2026 12:29:18 +0800 Subject: [PATCH 149/539] mm: distinguish large folio swap allocation failures folio_alloc_swap() reports most failures with generic negative error codes. Reclaim callers consequently cannot tell whether splitting a large folio could make progress, or whether no swap space is available for even a single page. Classify failures using both the global free swap count and the remaining capacity in the folio's memcg swap hierarchy. Return -ENOSPC when global swap space is exhausted, -ENOMEM when splitting cannot overcome the failure, and -E2BIG for a large folio when allocating or charging a smaller folio might still succeed. Use this classification for all folio_alloc_swap() failure paths, including capability rejection, swap slot allocation failure, and memcg swap charge failure. Callers are updated separately to split large folios only on -E2BIG. Link: https://lore.kernel.org/20260830042920.2280454-3-xueyuan.chen21@gmail.com Signed-off-by: Xueyuan Chen Signed-off-by: Andrew Morton Suggested-by: Kairui Song Suggested-by: Barry Song Suggested-by: Youngjun Park Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Chris Li Cc: Hugh Dickins Cc: Johannes Weiner Cc: Kemeng Shi Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Nanzhe Zhao Cc: Nhat Pham Cc: Roman Gushchin Cc: Shakeel Butt --- include/linux/swap.h | 6 ++++++ mm/memcontrol.c | 23 +++++++++++++++++++++++ mm/swapfile.c | 26 +++++++++++++++++++------- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 5658a1634b85..7a43409879ca 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -508,6 +508,7 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_p __mem_cgroup_uncharge_swap(id, nr_pages); } +long mem_cgroup_get_folio_swap_margin(struct folio *folio); extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg); extern bool mem_cgroup_swap_full(struct folio *folio); #else @@ -521,6 +522,11 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, { } +static inline long mem_cgroup_get_folio_swap_margin(struct folio *folio) +{ + return PAGE_COUNTER_MAX; +} + static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) { return get_nr_swap_pages(); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 7f63bf9e8ef1..bd1e7e154426 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5838,6 +5838,29 @@ long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) return nr_swap_pages; } +/** + * mem_cgroup_get_folio_swap_margin - get a folio's memcg swap margin + * @folio: folio whose memcg margin is queried + * + * Return: Remaining chargeable pages in the folio's memcg hierarchy. + */ +long mem_cgroup_get_folio_swap_margin(struct folio *folio) +{ + struct mem_cgroup *memcg; + long margin; + + if (mem_cgroup_disabled() || do_memsw_account() || + !folio_memcg_charged(folio)) + return PAGE_COUNTER_MAX; + + rcu_read_lock(); + memcg = folio_memcg(folio); + margin = page_counter_margin(&memcg->swap); + rcu_read_unlock(); + + return margin; +} + bool mem_cgroup_swap_full(struct folio *folio) { struct mem_cgroup *memcg; diff --git a/mm/swapfile.c b/mm/swapfile.c index 408f6c72fb5a..01e7b6b046b6 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1735,7 +1735,9 @@ failed: * swap cache. * * Context: Caller needs to hold the folio lock. - * Return: Whether the folio was added to the swap cache. + * Return: %0 on success, %-E2BIG if splitting the folio might allow swapout, + * %-ENOSPC if no global swap space is available, or %-ENOMEM if splitting + * would not help. */ int folio_alloc_swap(struct folio *folio) { @@ -1747,11 +1749,11 @@ int folio_alloc_swap(struct folio *folio) if (order) { /* - * Reject large allocation when THP_SWAP is disabled, - * the caller should split the folio and try again. + * Reject large allocation when THP_SWAP is disabled. Check below + * whether splitting and retrying can make progress. */ if (!IS_ENABLED(CONFIG_THP_SWAP)) - return -EAGAIN; + goto failed; /* * Allocation size should never exceed cluster size @@ -1759,7 +1761,7 @@ int folio_alloc_swap(struct folio *folio) */ if (size > SWAPFILE_CLUSTER) { VM_WARN_ON_ONCE(1); - return -EINVAL; + goto failed; } } @@ -1775,13 +1777,23 @@ again: } /* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */ - if (unlikely(mem_cgroup_try_charge_swap(folio))) + if (unlikely(mem_cgroup_try_charge_swap(folio))) { swap_cache_del_folio(folio); + goto failed; + } if (unlikely(!folio_test_swapcache(folio))) - return -ENOMEM; + goto failed; return 0; + +failed: + if (get_nr_swap_pages() <= 0) + return -ENOSPC; + if (mem_cgroup_get_folio_swap_margin(folio) <= 0) + return -ENOMEM; + + return order ? -E2BIG : -ENOMEM; } /** From 696444530a9be6eacae457f68fe5269d6eb6ce57 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Sun, 30 Aug 2026 12:29:19 +0800 Subject: [PATCH 150/539] mm/vmscan: avoid pointless large folio splits without swap When swap is disabled, exhausted, or unavailable due to memcg swap limits, splitting a large anonymous folio cannot make swapout progress. The fallback only destroys the large folio and inflates split statistics. Use -E2BIG from folio_alloc_swap() as the explicit signal that splitting the folio might allow swapout of smaller pieces. For other allocation failures, keep the existing activation path and avoid the split. This preserves the split fallback for fragmented or partially available swap, while avoiding it when there is no backing space for any part of the folio. Link: https://lore.kernel.org/20260830042920.2280454-4-xueyuan.chen21@gmail.com Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Xueyuan Chen Signed-off-by: Andrew Morton Reported-by: Nanzhe Zhao Acked-by: David Hildenbrand (Arm) Reviewed-by: Baolin Wang Cc: Baoquan He Cc: Chris Li Cc: Hugh Dickins Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Nhat Pham Cc: Roman Gushchin Cc: Shakeel Butt Cc: Youngjun Park --- mm/vmscan.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 42fcdcd3d2e4..ce3bab78af3c 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1259,6 +1259,8 @@ retry: */ if (folio_test_anon(folio) && folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) { + int ret; + if (!(sc->gfp_mask & __GFP_IO)) goto keep_locked; if (folio_maybe_dma_pinned(folio)) @@ -1277,11 +1279,14 @@ retry: split_folio_to_list(folio, folio_list)) goto activate_locked; } - if (folio_alloc_swap(folio)) { + ret = folio_alloc_swap(folio); + if (ret) { int __maybe_unused order = folio_order(folio); if (!folio_test_large(folio)) goto activate_locked_split; + if (ret != -E2BIG) + goto activate_locked; /* Fallback to swap normal pages */ if (split_folio_to_list(folio, folio_list)) goto activate_locked; From 5e3bd0c75c61a1a77c7f04f2d16696db9a245477 Mon Sep 17 00:00:00 2001 From: Xueyuan Chen Date: Sun, 30 Aug 2026 12:29:20 +0800 Subject: [PATCH 151/539] mm/shmem: split large folios only on -E2BIG shmem_writeout() currently splits a large folio on every folio_alloc_swap() failure. With the refined return-value contract, only -E2BIG indicates that splitting might allow smaller folios to be swapped out. Enter the split fallback only for -E2BIG. For -ENOSPC and -ENOMEM, redirty and reactivate the folio as before. Link: https://lore.kernel.org/20260830042920.2280454-5-xueyuan.chen21@gmail.com Signed-off-by: Xueyuan Chen Signed-off-by: Andrew Morton Suggested-by: Baolin Wang Reviewed-by: Baolin Wang Acked-by: David Hildenbrand (Arm) Reviewed-by: Barry Song Cc: Baoquan He Cc: Chris Li Cc: Hugh Dickins Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Nanzhe Zhao Cc: Nhat Pham Cc: Roman Gushchin Cc: Shakeel Butt Cc: Youngjun Park --- mm/shmem.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index d3f24b5977bd..84f0a2eb85fe 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1813,7 +1813,7 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); pgoff_t index; - int nr_pages; + int nr_pages, ret; bool split = false; if ((info->flags & SHMEM_F_LOCKED) || sbinfo->noswap) @@ -1894,7 +1894,8 @@ try_split: folio_mark_uptodate(folio); } - if (!folio_alloc_swap(folio)) { + ret = folio_alloc_swap(folio); + if (!ret) { bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages); int error; @@ -1947,7 +1948,7 @@ try_split: swap_cache_del_folio(folio); goto redirty; } - if (nr_pages > 1) + if (nr_pages > 1 && ret == -E2BIG) goto try_split; redirty: folio_mark_dirty(folio); From fed3c5ac4afbbe57c83854584567523b0e552f78 Mon Sep 17 00:00:00 2001 From: Aristeu Rozanski Date: Thu, 27 Aug 2026 21:55:43 -0400 Subject: [PATCH 152/539] mm: gup: move pmd_protnone() into gup_fast_pmd_leaf() Patch series "mm: gup: cleanup gup_fast call chain", v3. These two patches implement the refactor in gup_fast call chain David Hildenbrand mentioned in [1]. This patch (of 2): Make pmd handling match pud handling by calling pmd_protnone() inside gup_fast_pmd_leaf(). Link: https://lore.kernel.org/20260828015542.125576330@ruivo.org Link: https://lore.kernel.org/20260828015542.245315718@ruivo.org Link: https://lore.kernel.org/all/85e760cf-b994-40db-8d13-221feee55c60@redhat.com/T/#u [1] Signed-off-by: Aristeu Rozanski Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand Link: https://lore.kernel.org/all/85e760cf-b994-40db-8d13-221feee55c60@redhat.com/T/#u Acked-by: David Hildenbrand (Arm) Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu --- mm/gup.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index eb898ea1ee22..bd981ab39751 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2929,6 +2929,10 @@ static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, struct folio *folio; int refs; + /* See gup_fast_pte_range() */ + if (pmd_protnone(orig)) + return 0; + if (!pmd_access_permitted(orig, flags & FOLL_WRITE)) return 0; @@ -3024,10 +3028,6 @@ static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, return 0; if (unlikely(pmd_leaf(pmd))) { - /* See gup_fast_pte_range() */ - if (pmd_protnone(pmd)) - return 0; - if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags, pages, nr)) return 0; From a09bf22530426b9a8cb0b818ff11da97adde1f3d Mon Sep 17 00:00:00 2001 From: Aristeu Rozanski Date: Thu, 27 Aug 2026 21:55:44 -0400 Subject: [PATCH 153/539] mm: gup: cleanup the gup_fast_*() call chain Refactor gup_fast functions so each step of the way returns the number of pages pinned. Because the previous step of the chain knows what the number it should be, less indicates an error. This way there's no need to pass *nr along. Link: https://lore.kernel.org/20260828015542.334186653@ruivo.org Signed-off-by: Aristeu Rozanski Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand Link: https://lore.kernel.org/all/85e760cf-b994-40db-8d13-221feee55c60@redhat.com/T/#u Acked-by: David Hildenbrand (Arm) Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu --- mm/gup.c | 177 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 93 insertions(+), 84 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index bd981ab39751..a4036c02e213 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2826,11 +2826,11 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) * also check pmd here to make sure pmd doesn't change (corresponds to * pmdp_collapse_flush() in the THP collapse code path). */ -static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - int ret = 0; + unsigned long nr_pages = 0; pte_t *ptep, *ptem; ptem = ptep = pte_offset_map(&pmd, addr); @@ -2892,15 +2892,13 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, goto pte_unmap; } folio_set_referenced(folio); - pages[*nr] = page; - (*nr)++; + pages[nr_pages] = page; + nr_pages++; } while (ptep++, addr += PAGE_SIZE, addr != end); - ret = 1; - pte_unmap: pte_unmap(ptem); - return ret; + return nr_pages; } #else @@ -2913,21 +2911,21 @@ pte_unmap: * get_user_pages_fast_only implementation that can pin pages. Thus it's still * useful to have gup_fast_pmd_leaf even if we can't operate on ptes. */ -static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { return 0; } #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ -static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { struct page *page; struct folio *folio; - int refs; + unsigned long nr_pages, i; /* See gup_fast_pte_range() */ if (pmd_protnone(orig)) @@ -2939,42 +2937,40 @@ static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, if (pmd_special(orig)) return 0; - refs = (end - addr) >> PAGE_SHIFT; + nr_pages = (end - addr) >> PAGE_SHIFT; page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - folio = try_grab_folio_fast(page, refs, flags); + folio = try_grab_folio_fast(page, nr_pages, flags); if (!folio) return 0; if (unlikely(pmd_val(orig) != pmd_val(pmdp_get_lockless(pmdp)))) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!gup_fast_folio_allowed(folio, flags)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } - pages += *nr; - *nr += refs; - for (; refs; refs--) + for (i = 0; i < nr_pages; i++) *(pages++) = page++; folio_set_referenced(folio); - return 1; + return nr_pages; } -static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { struct page *page; struct folio *folio; - int refs; + unsigned long nr_pages, i; if (!pud_access_permitted(orig, flags & FOLL_WRITE)) return 0; @@ -2982,41 +2978,39 @@ static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr, if (pud_special(orig)) return 0; - refs = (end - addr) >> PAGE_SHIFT; + nr_pages = (end - addr) >> PAGE_SHIFT; page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); - folio = try_grab_folio_fast(page, refs, flags); + folio = try_grab_folio_fast(page, nr_pages, flags); if (!folio) return 0; if (unlikely(pud_val(orig) != pud_val(pudp_get(pudp)))) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!gup_fast_folio_allowed(folio, flags)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { - gup_put_folio(folio, refs, flags); + gup_put_folio(folio, nr_pages, flags); return 0; } - pages += *nr; - *nr += refs; - for (; refs; refs--) + for (i = 0; i < nr_pages; i++) *(pages++) = page++; folio_set_referenced(folio); - return 1; + return nr_pages; } -static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pmd_range(pud_t *pudp, pud_t pud, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pmd_t *pmdp; pmdp = pmd_offset_lockless(pudp, pud, addr); @@ -3025,26 +3019,30 @@ static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, next = pmd_addr_end(addr, end); if (!pmd_present(pmd)) - return 0; + break; if (unlikely(pmd_leaf(pmd))) { - if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags, - pages, nr)) - return 0; + chunk_nr_pages = gup_fast_pmd_leaf(pmd, pmdp, addr, + next, flags, + &pages[nr_pages]); - } else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags, - pages, nr)) - return 0; + } else + chunk_nr_pages = gup_fast_pte_range(pmd, pmdp, addr, + next, flags, + &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pmdp++, addr = next, addr != end); - return 1; + return nr_pages; } -static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, + unsigned long addr, unsigned long end, + unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pud_t *pudp; pudp = pud_offset_lockless(p4dp, p4d, addr); @@ -3053,24 +3051,27 @@ static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, next = pud_addr_end(addr, end); if (unlikely(!pud_present(pud))) - return 0; - if (unlikely(pud_leaf(pud))) { - if (!gup_fast_pud_leaf(pud, pudp, addr, next, flags, - pages, nr)) - return 0; - } else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags, - pages, nr)) - return 0; + break; + if (unlikely(pud_leaf(pud))) + chunk_nr_pages = gup_fast_pud_leaf(pud, pudp, addr, + next, flags, + &pages[nr_pages]); + else + chunk_nr_pages = gup_fast_pmd_range(pudp, pud, addr, + next, flags, + &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pudp++, addr = next, addr != end); - return 1; + return nr_pages; } -static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +static unsigned long gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; p4d_t *p4dp; p4dp = p4d_offset_lockless(pgdp, pgd, addr); @@ -3079,20 +3080,23 @@ static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, next = p4d_addr_end(addr, end); if (!p4d_present(p4d)) - return 0; + break; BUILD_BUG_ON(p4d_leaf(p4d)); - if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags, - pages, nr)) - return 0; + chunk_nr_pages = gup_fast_pud_range(p4dp, p4d, addr, next, + flags, &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (p4dp++, addr = next, addr != end); - return 1; + return nr_pages; } -static void gup_fast_pgd_range(unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static unsigned long gup_fast_pgd_range(unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages) { - unsigned long next; + unsigned long next, nr_pages = 0, chunk_nr_pages; pgd_t *pgdp; pgdp = pgd_offset(current->mm, addr); @@ -3101,17 +3105,23 @@ static void gup_fast_pgd_range(unsigned long addr, unsigned long end, next = pgd_addr_end(addr, end); if (pgd_none(pgd)) - return; + break; BUILD_BUG_ON(pgd_leaf(pgd)); - if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags, - pages, nr)) - return; + chunk_nr_pages = gup_fast_p4d_range(pgdp, pgd, addr, next, + flags, &pages[nr_pages]); + nr_pages += chunk_nr_pages; + if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT) + break; } while (pgdp++, addr = next, addr != end); + + return nr_pages; } #else -static inline void gup_fast_pgd_range(unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static inline unsigned long gup_fast_pgd_range(unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages) { + return 0; } #endif /* CONFIG_HAVE_GUP_FAST */ @@ -3129,8 +3139,7 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end) static unsigned long gup_fast(unsigned long start, unsigned long end, unsigned int gup_flags, struct page **pages) { - unsigned long flags; - int nr_pinned = 0; + unsigned long flags, nr_pinned; unsigned seq; if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || @@ -3154,7 +3163,7 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, * that come from callers of tlb_remove_table_sync_one(). */ local_irq_save(flags); - gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned); + nr_pinned = gup_fast_pgd_range(start, end, gup_flags, pages); local_irq_restore(flags); /* From 0224133dba0552fbe335770133fe23169fe724f4 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Wed, 19 Aug 2026 10:55:15 +0800 Subject: [PATCH 154/539] mm/page_table_check: add explicit pmd_none check in pte_clear_range In __page_table_check_pte_clear_range(), the condition to determine whether to iterate over PTEs only checked pmd_bad() and pmd_leaf(). This relies on the implicit assumption that pmd_none() is always a subset of pmd_bad() on all architectures supporting PAGE_TABLE_CHECK. While this assumption currently holds for x86_64, arm64, s390, riscv, and powerpc, it is an architecture-dependent behavior that may not hold for future architectures. Add an explicit pmd_none() check to make the intent clear and avoid calling pte_offset_map() on an empty PMD, which could lead to undefined behavior. Link: https://lore.kernel.org/20260819025516.2967199-1-ye.liu@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Cc: Albert Ou Cc: Alexandre Ghiti Cc: Palmer Dabbelt Cc: Pasha Tatashin --- mm/page_table_check.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 6ffc536359cd..ed3e1a76f266 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -278,7 +278,7 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm, if (&init_mm == mm) return; - if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { + if (!pmd_none(pmd) && !pmd_bad(pmd) && !pmd_leaf(pmd)) { pte_t *ptep = pte_offset_map(&pmd, addr); unsigned long i; From 33afd07412a1efe5b750a3fd5eb0272ebdd12761 Mon Sep 17 00:00:00 2001 From: Zhiling Zou Date: Tue, 21 Jul 2026 23:56:38 +0800 Subject: [PATCH 155/539] mm/page_table_check: skip zero pages page_table_check_set() accounts pages by whether they are PageAnon(). The shared zero page is a special page, not an ordinary file-backed page. Read faults on private anonymous mappings can install many read-only PTEs that point at the zero page, but page_table_check currently accounts them in file_map_count. That lets an unprivileged process populate enough zero-page mappings to overflow file_map_count and trip the BUG_ON() in page_table_check_set(). Skip zero pages in page_table_check accounting. They do not need the anonymous/file mapping conflict checks that page_table_check performs for ordinary pages, and this keeps the existing counter size and page_ext layout unchanged. Link: https://lore.kernel.org/1f8848512d2e3ded944f8d595c29faee8fdaeab0.1784645969.git.roxy520tt@gmail.com Fixes: df4e817b7108 ("mm: page table check") Signed-off-by: Zhiling Zou Signed-off-by: Ren Wei Signed-off-by: Andrew Morton Reported-by: Vega Assisted-by: Codex:gpt-5.4 Cc: Ye Liu Cc: --- mm/page_table_check.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index ed3e1a76f266..143a918c0bde 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -67,7 +67,7 @@ static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) struct page *page; bool anon; - if (!pfn_valid(pfn)) + if (!pfn_valid(pfn) || is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return; page = pfn_to_page(pfn); @@ -102,7 +102,7 @@ static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, struct page *page; bool anon; - if (!pfn_valid(pfn)) + if (!pfn_valid(pfn) || is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return; page = pfn_to_page(pfn); From 90c2fa6adfed7abe0935506a43cdff3a05c1fff0 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:18:42 -0700 Subject: [PATCH 156/539] mm/damon/core: skip applying scheme if region split for quota fails Patch series "mm/damon: fix DAMOS bugs in core, paddr and vaddr". Fix misc bugs of DAMOS. Patch 1 makes DAMOS less stress memory allocator under extreme situation. Patches 2 and 3 fix wrong folios walking in DAMON_PADDR. Patches 4 and 5 fix wrong folios walking in DAMON_VADDR. Patches 6-8 handle extreme and unlikely memory situations that can cause divide by zero and underflow. All bugs are discovered by Sashiko. This patch (of 8): damos_apply_scheme() splits a region and apply the action to the subregion if it is needed for not violating the quota. The split operation (damon_split_region_at()) could fail for allocation failure. In the case, the quota could be violated. From the user's perspective, DAMOS becomes more aggressive than expected under the extreme situation. Handle the failure. The user impact is not critical. The failure of damon_split_region_at() is unlikely since it is arguably too small to fail. Also DAMOS being aggressive is limited to the single region. Users can set min_nr_regions to set the maximum size of each region. If it is reasonably set, the transient overhead shouldn't be critical. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260901131850.98037-1-sj@kernel.org Link: https://lore.kernel.org/20260901131850.98037-2-sj@kernel.org Link: https://lore.kernel.org/20260718171523.87547-1-sj@kernel.org [1] Fixes: 2b8a248d5873 ("mm/damon/schemes: implement size quota for schemes application speed control") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Usama Arif Cc: # 5.16.x --- mm/damon/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index d63d4c6fd3ff..53952d16ebd8 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2607,7 +2607,8 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, c->min_region_sz); if (!sz) goto update_stat; - damon_split_region_at(t, r, sz); + if (damon_split_region_at(t, r, sz)) + goto update_stat; } if (damos_core_filter_out(c, t, r, s)) return; From ac733f921d4fc02da70a98b175a6055a206b2a36 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:18:43 -0700 Subject: [PATCH 157/539] mm/damon/paddr: respect folio end for DAMOS_STAT The function for applying DAMOS_STAT in DAMON physical address space operation set (paddr), namely damon_pa_stat(), applies DAMOS filters to folios of the given region. For that, it gets folios of addresses in the region. It starts from the region start address and advances the address by the size of the folio of the address until it goes out of the region. If the start address is in the middle of a large folio, and if the next folios are small, some of the next folios could be skipped. Fix the issue by advancing the address to exactly the start address of the next folio. The user impact is that the DAMOS_STAT-based page level monitoring results become inaccurate. Since the page level monitoring is supposed to provide relatively high precision, this is definitely a problem. It is arguably not critical since it is only monitoring quality degradation. Link: https://lore.kernel.org/20260901131850.98037-3-sj@kernel.org Fixes: bdbe1d7bc325 ("mm/damon/paddr: increment pa_stat damon address range by folio size") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Usama Arif Cc: # 6.14.x --- mm/damon/paddr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5c6c3a597fd0..2ab7b3842701 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -379,7 +379,7 @@ static unsigned long damon_pa_stat(struct damon_region *r, if (!damos_pa_filter_out(s, folio)) *sz_filter_passed += folio_size(folio) / addr_unit; - addr += folio_size(folio); + addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio); folio_put(folio); } s->last_applied = folio; From 1613ac9edf385c35f171a25610e2706a7e6afad6 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:18:44 -0700 Subject: [PATCH 158/539] mm/damon/paddr: respect folio end for DAMOS actions except STAT A few functions for applying DAMOS actions including pageout, lru_[de]prio and migrate_{hot,cold} in DAMON physical address space operation set (paddr) collect folios of the given region by getting the folios of region-internal addresses. Then, those functions apply the action to the collected folios at once. The collection starts from the region start address and advances the address by the size of the folio of the address until it goes out of the region. If the start address is in the middle of a large folio, and if the next folios are small, some of the next folios could be skipped. Fix the issue by advancing the address to exactly the start address of the next folio. The user impact is that DAMOS action is applied to less than expected amount of memory. Given the best effort nature of DAMON, it is no big problem, but it is clearly a bug that is better to be fixed. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260901131850.98037-4-sj@kernel.org Link: https://lore.kernel.org/20260517234112.89245-1-sj@kernel.org [1] Fixes: 3a06696305e7 ("mm/damon/ops: have damon_get_folio return folio even for tail pages") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Usama Arif Cc: # 6.15.x --- mm/damon/paddr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 2ab7b3842701..9ddd1ec8202b 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -264,7 +264,7 @@ static unsigned long damon_pa_pageout(struct damon_region *r, else list_add(&folio->lru, &folio_list); put_folio: - addr += folio_size(folio); + addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio); folio_put(folio); } if (install_young_filter) @@ -302,7 +302,7 @@ static inline unsigned long damon_pa_de_activate( folio_deactivate(folio); applied += folio_nr_pages(folio); put_folio: - addr += folio_size(folio); + addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio); folio_put(folio); } s->last_applied = folio; @@ -350,7 +350,7 @@ static unsigned long damon_pa_migrate(struct damon_region *r, folio_is_file_lru(folio)); list_add(&folio->lru, &folio_list); put_folio: - addr += folio_size(folio); + addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio); folio_put(folio); } applied = damon_migrate_pages(&folio_list, s->target_nid); From 946351a5fc5d4e3afec2bdf735935d17d6c9ceb2 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:18:45 -0700 Subject: [PATCH 159/539] mm/damon/vaddr: respect folio end for DAMOS_STAT For applying DAMOS_STAT action to a region, DAMON virtual address space operation set (vaddr) calls walk_page_range[_vma]() for the region. The pmd walk entry function, namely damon_va_stat_pmd_entry(), applies DAMOS filters to folios of addresses of the region in the pmd. It starts from the walking address and advances the address by the size of the folio of the address until it goes out of the pmd or the region. Let's suppose it is for the first pmd of the region, and the region start address is in the middle of a large folio. Also, the next folios are small. Then, some of the next folios could be skipped. Fix the issue by advancing the address to exactly the start address of the next folio. The user impact is that the DAMOS_STAT-based page level monitoring results become inaccurate. Since the page level monitoring is supposed to provide relatively high precision, this is definitely a problem. It is arguably not critical since it is only monitoring quality degradation. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260901131850.98037-5-sj@kernel.org Link: https://lore.kernel.org/20260514015053.149396-1-sj@kernel.org [1] Fixes: 63f39737d1e3 ("mm/damon/vaddr: support stat-purpose DAMOS filters") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Usama Arif Cc: # 6.18.x --- mm/damon/vaddr.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 04ee2a2c6a4d..b1bed5d19a34 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -838,6 +838,8 @@ huge_unlock: return 0; for (; addr < next; pte += nr, addr += nr * PAGE_SIZE) { + unsigned long page_idx; + nr = 1; ptent = ptep_get(pte); @@ -851,7 +853,8 @@ huge_unlock: if (!damos_va_filter_out(s, folio, vma, addr, pte, NULL)) *sz_filter_passed += folio_size(folio); - nr = folio_nr_pages(folio); + page_idx = folio_page_idx(folio, pte_page(ptent)); + nr = folio_nr_pages(folio) - page_idx; s->last_applied = folio; } pte_unmap_unlock(start_pte, ptl); From 359093bdffbb3bd4d46d2d76d4a4dd273242c6d1 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:18:46 -0700 Subject: [PATCH 160/539] mm/damon/vaddr: respect folio end for DAMOS_MIGRATE_{HOT,COLD} For applying DAMOS_MIGRATE_{HOT,COLD} actions to a region, DAMON virtual address space operation set (vaddr) calls walk_page_range[_vma]() for the region. The pmd walk entry function, namely damon_va_migrate_pmd_entry(), collects folios of addresses of the region in the pmd. It starts from the walking address and advances the address by the size of the folio of the address until it goes out of the pmd or the region. Let's suppose it is for the first pmd of the region, and the region start address is in the middle of a large folio. Also, the next folios are small. Then, some of the next folios could be skipped. Fix the issue by advancing the address to exactly the start address of the next folio. The user impact is that DAMOS action is applied to less than expected amount of memory. Given the best effort nature of DAMON, it is no big problem, but it is clearly a bug that is better to be fixed. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260901131850.98037-6-sj@kernel.org Link: https://lore.kernel.org/20260514015053.149396-1-sj@kernel.org [1] Fixes: 09efc56a3b1c ("mm/damon/vaddr: consistently use only pmd_entry for damos_migrate") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Usama Arif Cc: # 6.19.x --- mm/damon/vaddr.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index b1bed5d19a34..5b4d16c8db62 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -676,6 +676,8 @@ huge_out: return 0; for (; addr < next; pte += nr, addr += nr * PAGE_SIZE) { + unsigned long page_idx; + nr = 1; ptent = ptep_get(pte); @@ -688,7 +690,8 @@ huge_out: continue; damos_va_migrate_dests_add(folio, walk->vma, addr, dests, migration_lists); - nr = folio_nr_pages(folio); + page_idx = folio_page_idx(folio, pte_page(ptent)); + nr = folio_nr_pages(folio) - page_idx; } pte_unmap_unlock(start_pte, ptl); return 0; From 485a9149ad406e00949327a3231b9084d7073706 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:18:47 -0700 Subject: [PATCH 161/539] mm/damon/core: handle extreme memory state in damon_get_node_mem_bp() In an extreme and unlikely situation, si_meminfo_node() might let the caller show zero total ram. That could cause a divide by zero in damon_get_node_mem_bp(). It could also show free memory larger than the total memory. This could cause underflow and make DAMOS temporarily make unexpected behavior. Thanks to safety guards in the auto-tuning feedback loop, that should not be a real problem, though. Fix the problems by respectively returning 100% and 0% for used and free memory queries in the corner cases. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260901131850.98037-7-sj@kernel.org Link: https://lore.kernel.org/20260328133216.9697-1-sj@kernel.org [1] Fixes: 0e1c773b501f ("mm/damon/core: introduce damos quota goal metrics for memory node utilization") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Usama Arif Cc: # 6.16.x --- mm/damon/core.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 53952d16ebd8..2942f23a240b 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2810,6 +2810,13 @@ static __kernel_ulong_t damos_get_node_mem_bp( } si_meminfo_node(&i, goal->nid); + if (!i.totalram || i.totalram < i.freeram) { + if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP) + return 10000; + else /* DAMOS_QUOTA_NODE_MEM_FREE_BP */ + return 0; + } + if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP) numerator = i.totalram - i.freeram; else /* DAMOS_QUOTA_NODE_MEM_FREE_BP */ From c4b1ae3837ff381795f1fcc18d827fce48c4e89b Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:18:48 -0700 Subject: [PATCH 162/539] mm/damon/core: handle extreme memory state in get_node_memcg_used_bp() In extreme unlikely situations, total memory might be zero. In less extreme but still very unlikely situations, lruvec_page_state() calls might let the caller show used memory larger than total memory. In the two cases, damos_get_node_memcg_used_bp() could cause division by zero, or return underflowed value, respectively. Handle the cases by respectively returning 100% and 0% for used and free memory queries in the corner cases. This issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260901131850.98037-8-sj@kernel.org Link: https://lore.kernel.org/20260329154813.47382-1-sj@kernel.org [1] Fixes: b74a120bcf50 ("mm/damon/core: implement DAMOS_QUOTA_NODE_MEMCG_USED_BP") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Usama Arif Cc: # 6.19.x --- mm/damon/core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 2942f23a240b..d00cf1fae239 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2857,6 +2857,12 @@ static unsigned long damos_get_node_memcg_used_bp( mem_cgroup_put(memcg); si_meminfo_node(&i, goal->nid); + if (!i.totalram || i.totalram < used_pages) { + if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP) + return 10000; + else /* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */ + return 0; + } if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP) numerator = used_pages; else /* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */ From d9e43f46fb324e0a311146f937f0d214f023dc63 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:18:49 -0700 Subject: [PATCH 163/539] mm/damon/core: handle extreme memory state in get_in_active_mem_bp() damos_get_in_active_mem_bp() uses the sum of the active and inactive memory amount as a denominator. In an extreme and unlikely environment, active and inactive memory might be zero. In this case, hence, it results in a divide by zero problem. Avoid it by changing the denominator to one if it is zero, before it is being used. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260901131850.98037-9-sj@kernel.org Link: https://lore.kernel.org/20260721034756.147011-1-sj@kernel.org [1] Fixes: 4835e2871321 ("mm/damon/core: introduce [in]active memory ratio damos quota goal metric") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Usama Arif Cc: # 7.0.x --- mm/damon/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index d00cf1fae239..f6a9d2da0cd7 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3009,7 +3009,7 @@ static unsigned int damos_get_in_active_mem_bp(bool active_ratio) global_node_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE); inactive = global_node_page_state(NR_LRU_BASE + LRU_INACTIVE_ANON) + global_node_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE); - total = active + inactive; + total = max(active + inactive, 1); if (active_ratio) return mult_frac(active, 10000, total); return mult_frac(inactive, 10000, total); From f138b7b98ec5fc02c1942ca7b25860005d6383c0 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:48 -0700 Subject: [PATCH 164/539] mm/damon/core: introduce DAMON_FILTER_TYPE_PGIDLE_UNSET Patch series "mm/damon: introduce data access-as-a-data attribute", v1.1. TL;DR: extend DAMON's data attributes monitoring system to support page table accessed bit and PG_idle based access monitoring. DAMON was initially introduced as a data access monitor. Users found data access pattern becomes more useful when it is combined with other data attributes such as belonging cgroups and backing page types. For such cases, DAMON has extended to support such data attributes monitoring in addition to the original data access monitoring. The DAMON probes system was introduced for this purpose. Users set probes for filtering data attributes of their interest. For use cases where the primary interests are the attributes but the access pattern, probe weights system has been introduced. When it is used, DAMON applies its adaptive regions adjustment based on the monitored data attributes. However, DAMON stops access monitoring when the probe weights are used. DAMON cannot optimally help users who have interests in both data attributes and access patterns. Data access can also be thought of as another data attribute, though. Extend the probe system to support data access as a data attribute. Introduce a new probe filter type, pgidle_unset. It shows if the page is not set as idle. Specifically, it shows the page table accessed bit and the PG_idle flag. It can inform if the region is ever accessed. But it cannot say when it is accessed. To answer the second question, introduce a new probe feature, prep actions. Using the features, Users can specify what preparation actions should be made to each region for each probe. DAMON executes the preparation actions for each sampling interval, like it is doing the preparation for access check in the access monitoring mode. To help 'pgidle_unset' probe action use case, 'set_pgidle' preparation action is introduced together. The action does exactly what the access monitoring was doing: clearing the page table accessed bits and setting the PG_idle flags. Tests ===== I compared the access pattern monitoring results from the classic way and the probe based way. As expected, the probe based way shows the results similar to that of the classic way. More detailed test methods and results are below. First, do the access monitoring using the DAMON user-space tool [1] in the classic way. The system is idle. It shows no access as expected. $ sudo ./damo/damo start $ sudo ./damo/damo report access heatmap: 00000000000000000000000000000000000000008999999811111110000000000000000000000000 # min/max temperatures: -640,000,000, -100,000,000, column size: 99.800 MiB intervals: sample 5 ms aggr 100 ms (max access hz 200) 0 addr 4.000 KiB size 3.898 GiB access 0 hz age 6.400 s 1 addr 3.898 GiB size 787.301 MiB access 0 hz age 1 s 2 addr 4.667 GiB size 773.457 MiB access 0 hz age 5.800 s 3 addr 5.423 GiB size 2.374 GiB access 0 hz age 6.400 s memory bw estimate: 0 B per second total size: 7.797 GiB record DAMON intervals: sample 5 ms, aggr 100 ms Start an artificial memory access generator (masim) [2] in another window. $ ./masim/masim.py run --config_file ./masim/configs/zigzag.cfg Show the monitoring results. As expected, it captures accesses. $ sudo ./damo/damo report access heatmap: 00000000000000000000000000000000000000011111111111111118888888833378988888888888 # min/max temperatures: -1,470,000,000, -12,482,536, column size: 99.800 MiB intervals: sample 5 ms aggr 100 ms (max access hz 200) 0 addr 4.000 KiB size 1.542 GiB access 0 hz age 14.700 s 1 addr 1.542 GiB size 788.324 MiB access 0 hz age 14.600 s 2 addr 2.311 GiB size 772.844 MiB access 0 hz age 14.200 s [...] 50 addr 6.839 GiB size 1.742 MiB access 0 hz age 1.300 s 51 addr 6.840 GiB size 8.000 KiB access 100 hz age 0 ns 52 addr 6.840 GiB size 1.496 MiB access 160 hz age 0 ns [...] 97 addr 7.162 GiB size 1.199 MiB access 40 hz age 400 ms 98 addr 7.163 GiB size 1.199 MiB access 20 hz age 400 ms 99 addr 7.164 GiB size 2.004 MiB access 160 hz age 0 ns 100 addr 7.166 GiB size 646.004 MiB access 0 hz age 400 ms memory bw estimate: 26.148 GiB per second total size: 7.797 GiB record DAMON intervals: sample 5 ms, aggr 100 ms After the artificial memory access generator (masim) is terminated, restart DAMON with the probe-based access monitoring. As expected, it shows no access since the system is idle again. $ sudo ./damo/damo stop $ sudo ./damo/damo start --probe_prep set_pgidle \ --probe_filter allow pgidle_unset --probe_weight 1 $ sudo ./damo/damo report attrs heatmap: 00000000000000000000000000000000000000008999999711111100000000000000000000000000 # min/max temperatures: -600,000,000, -430,000,000, column size: 99.800 MiB probe prep: set_pgidle, filter: allow pgidle_unset (weight: 1) intervals: sample 5 ms aggr 100 ms (max probe hits 20) # addr size age probe_hits 0 4.000 KiB 3.898 GiB 6 s 0 1 5.285 GiB 2.512 GiB 6 s 0 2 4.659 GiB 641.816 MiB 5.700 s 0 3 3.898 GiB 778.375 MiB 4.300 s 0 memory bw estimate: 0 B per second total size: 7.797 GiB record DAMON intervals: sample 5 ms, aggr 100 ms Start the artificial memory access generator [2] again. $ ./masim/masim.py run --config_file ./masim/configs/zigzag.cfg Show the monitoring results. As expected, it captures accesses similar to the classic monitoring mode. $ sudo ./damo/damo report attrs heatmap: 00000000000000000000000000000000000000011111110000000177777777777777878798887777 # min/max temperatures: -1,330,000,000, 84,847,514, column size: 99.800 MiB probe prep: set_pgidle, filter: allow pgidle_unset (weight: 1) intervals: sample 5 ms aggr 100 ms (max probe hits 20) # addr size age probe_hits 0 4.000 KiB 1.508 GiB 13.300 s 0 1 1.508 GiB 794.684 MiB 13.200 s 0 2 2.284 GiB 764.555 MiB 13 s 0 [...] 50 6.711 GiB 4.625 MiB 0 ns 17 51 6.716 GiB 2.504 MiB 0 ns 17 52 6.732 GiB 796.000 KiB 0 ns 17 [...] 90 7.162 GiB 512.000 KiB 2.200 s 2 91 7.061 GiB 700.000 KiB 2.300 s 1 92 6.935 GiB 316.000 KiB 2.300 s 4 memory bw estimate: 0 B per second total size: 7.797 GiB record DAMON intervals: sample 5 ms, aggr 100 ms Patches Sequence ================ First four patches (patches 1-4) introduce the new probe filter type for knowing if a region is accessed. Patch 1 defines the new type in the core. Patch 2 implements the execution of the new filter in the physical address space DAMON operation set. Patch 3 implements a user interface on DAMON sysfs interface. Patch 4 updates the documentation for the new filter type. Following 13 patches (patches 5-17) introduce the probe preparation actions feature. Patch 5 defines the data structure for specifying the preparation actions. Patch 6 completes setup of the API parameter for the prep. Patch 7 extends the DAMON operation set callback list to connect the parameter with the underlying operation set. Patch 8 implements the execution of the prep in the physical address space DAMON operation set. Following five patches (patches 9-13) extends DAMON sysfs interface for the new prep feature. Patch 14 adds simple selftest for basic file operations of the new sysfs files. Final three patches (patches 15-17) respectively update design, usage and ABI documents for the new feature and its interface. This patch (of 17): Introduce a new DAMON filter type, pgidle_unset. It will match pages that have their PG_Idle flag unset, or the page table accessed bit set. In other words, it says if the page is accessed. Link: https://lore.kernel.org/20260901132506.99243-1-sj@kernel.org Link: https://lore.kernel.org/20260901132506.99243-2-sj@kernel.org Link: https://github.com/damonitor/damo [1] Link: https://github.com/sjp38/masim [2] Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 7b1b6050a828..7a42cbe79184 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -745,12 +745,14 @@ struct damon_intervals_goal { /** * enum damon_filter_type - Type of &struct damon_filter * - * @DAMON_FILTER_TYPE_ANON: Anonymous pages. - * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. + * @DAMON_FILTER_TYPE_ANON: Anonymous pages. + * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. + * @DAMON_FILTER_TYPE_PGIDLE_UNSET: Pgidle is unset. */ enum damon_filter_type { DAMON_FILTER_TYPE_ANON, DAMON_FILTER_TYPE_MEMCG, + DAMON_FILTER_TYPE_PGIDLE_UNSET, }; /** From ef14c8da645acf156486ab62bf14d70e26cfdd61 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:49 -0700 Subject: [PATCH 165/539] mm/damon/paddr: support PGIDLE_UNSET probe filter type Implement support of DAMON_FILTER_TYPE_PGIDLE_UNSET in the physical address space DAMON operations set. It reuses damon_folio_young(), which was being used for access monitoring. Link: https://lore.kernel.org/20260901132506.99243-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/paddr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 9ddd1ec8202b..6f756f849389 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -132,6 +132,12 @@ static bool damon_pa_filter_match(struct damon_filter *filter, matched = filter->memcg_id == mem_cgroup_id(memcg); rcu_read_unlock(); break; + case DAMON_FILTER_TYPE_PGIDLE_UNSET: + if (!folio) + matched = false; + else + matched = damon_folio_young(folio); + break; default: break; } From 09d247284ed5cbed9da1d52b0252a503997edae0 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:50 -0700 Subject: [PATCH 166/539] mm/damon/sysfs: support pgidle_unset probe filter type Extend DAMON sysfs interface to allow users to set DAMON_FILTER_TYPE_PGIDLE_UNSET by writing 'pgidle_unset' to the probe filter type file. Link: https://lore.kernel.org/20260901132506.99243-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 3c81b4c91ac0..c1ff739dab27 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -781,6 +781,10 @@ damon_sysfs_filter_type_names[] = { .type = DAMON_FILTER_TYPE_MEMCG, .name = "memcg", }, + { + .type = DAMON_FILTER_TYPE_PGIDLE_UNSET, + .name = "pgidle_unset", + }, }; static ssize_t type_show(struct kobject *kobj, From 95ece6b3b352174a56c4a45a886118dc183a7ab7 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:51 -0700 Subject: [PATCH 167/539] Docs/mm/damon/design: document pgidle_unset probe filter type Update DAMON design document for the newly added pgidle_unset probe filter type. Also use a list for the types, as it becomes not very easy to read the whole types in a simple sentence. Link: https://lore.kernel.org/20260901132506.99243-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 63cbb7b536da..947d91ae24a3 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -293,8 +293,12 @@ registration is made by specifying a probe per attribute. Each of the probe specifies a rule to determine if a given memory region has the related attribute. The rule is constructed with multiple filters. The filters work same to :ref:`DAMOS filters ` except the supported -filter types. Currently only ``anon`` and ``memcg`` filter types are supported -for data attributes monitoring. +filter types. Currently below filter types are supported. + +- ``anon``: Same to that for DAMOS filters. +- ``memcg``: Same to that for DAMOS filters. +- ``pgidle_unset``: Matches if the page for the memory is marked as not + access-idle. If such probes are registered, DAMON executes the probes for each region's sampling memory when it does the access :ref:`sampling From 17b8c254757ed0cc58d67eda0f75a2c5a19957ed Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:52 -0700 Subject: [PATCH 168/539] mm/damon/core: introduce damon_prep struct Some DAMON probe filter types require preparatory actions. For example, pgilde_unset probe filter can say if the page was accessed but when. To answer the second question, the PG_Idle flag should be set at a specific time. It can make life much easier if DAMON can do such preparatory actions. Introduce a new data type called damon_prep. It specifies each of the preparation actions for each probe. DAMON will execute the action for each region per sampling interval, like it clears page table accessed bits and unsets PG_Idle flag for access monitoring. Also introduce DAMON_PREP_SET_PGIDLE as the initial prep action. As the name says, it will do exactly what DAMON was doing as the preparation action for the access monitoring. Link: https://lore.kernel.org/20260901132506.99243-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 32 ++++++++++++++++++++++++++++++++ mm/damon/core.c | 26 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 7a42cbe79184..1780c14942e6 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -742,6 +742,27 @@ struct damon_intervals_goal { unsigned long max_sample_us; }; +/** + * enum damon_prep_action - DAMON probing preparation action. + * + * @DAMON_PREP_SET_PGIDLE: Set the probing memory as idle page. + */ +enum damon_prep_action { + DAMON_PREP_SET_PGIDLE, +}; + +/** + * struct damon_prep - DAMON probing preparation request. + * + * @action: Action to do to the probing memory for the preparation. + */ +struct damon_prep { + enum damon_prep_action action; +/* private: */ + /* siblings list. */ + struct list_head list; +}; + /** * enum damon_filter_type - Type of &struct damon_filter * @@ -783,6 +804,8 @@ struct damon_filter { struct damon_probe { unsigned int weight; /* private: */ + /* Preparation actions to apply to each probing memory. */ + struct list_head preps; /* Filters for assessing if a given region is for this probe. */ struct list_head filters; /* Siblings list. */ @@ -962,6 +985,12 @@ static inline unsigned long damon_sz_region(struct damon_region *r) return r->ar.end - r->ar.start; } +#define damon_for_each_prep(p, probe) \ + list_for_each_entry(p, &(probe)->preps, list) + +#define damon_for_each_prep_safe(p, next, probe) \ + list_for_each_entry_safe(p, next, &(probe)->preps, list) + #define damon_for_each_filter(f, p) \ list_for_each_entry(f, &(p)->filters, list) @@ -1015,6 +1044,9 @@ static inline unsigned long damon_sz_region(struct damon_region *r) #ifdef CONFIG_DAMON +struct damon_prep *damon_new_prep(enum damon_prep_action action); +void damon_add_prep(struct damon_probe *p, struct damon_prep *prep); + struct damon_filter *damon_new_filter(enum damon_filter_type type, bool matching, bool allow); void damon_add_filter(struct damon_probe *probe, struct damon_filter *f); diff --git a/mm/damon/core.c b/mm/damon/core.c index f6a9d2da0cd7..a0a61ebdf5bb 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -112,6 +112,28 @@ int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id) return err; } +struct damon_prep *damon_new_prep(enum damon_prep_action action) +{ + struct damon_prep *prep; + + prep = kmalloc_obj(*prep); + if (!prep) + return NULL; + prep->action = action; + INIT_LIST_HEAD(&prep->list); + return prep; +} + +void damon_add_prep(struct damon_probe *p, struct damon_prep *prep) +{ + list_add_tail(&prep->list, &p->preps); +} + +static void damon_free_prep(struct damon_prep *p) +{ + kfree(p); +} + struct damon_filter *damon_new_filter(enum damon_filter_type type, bool matching, bool allow) { @@ -168,6 +190,7 @@ struct damon_probe *damon_new_probe(void) if (!p) return NULL; p->weight = 0; + INIT_LIST_HEAD(&p->preps); INIT_LIST_HEAD(&p->filters); INIT_LIST_HEAD(&p->list); return p; @@ -185,8 +208,11 @@ static void damon_del_probe(struct damon_probe *p) static void damon_free_probe(struct damon_probe *p) { + struct damon_prep *prep, *prep_next; struct damon_filter *f, *next; + damon_for_each_prep_safe(prep, prep_next, p) + damon_free_prep(prep); damon_for_each_filter_safe(f, next, p) damon_free_filter(f); kfree(p); From 9641c80162a1147493d3a5a7d6ca0ec9f5fc7673 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:53 -0700 Subject: [PATCH 169/539] mm/damon/core: commit preps damon_commit_probes() is ignoring damon_prep. Commit the prep actions, too. Link: https://lore.kernel.org/20260901132506.99243-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index a0a61ebdf5bb..c631b5324377 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -129,11 +129,34 @@ void damon_add_prep(struct damon_probe *p, struct damon_prep *prep) list_add_tail(&prep->list, &p->preps); } +static void damon_del_prep(struct damon_prep *p) +{ + list_del(&p->list); +} + static void damon_free_prep(struct damon_prep *p) { kfree(p); } +static void damon_destroy_prep(struct damon_prep *p) +{ + damon_del_prep(p); + damon_free_prep(p); +} + +static struct damon_prep *damon_nth_prep(int n, struct damon_probe *p) +{ + struct damon_prep *prep; + int i = 0; + + damon_for_each_prep(prep, p) { + if (i++ == n) + return prep; + } + return NULL; +} + struct damon_filter *damon_new_filter(enum damon_filter_type type, bool matching, bool allow) { @@ -1699,6 +1722,36 @@ out: return err; } +static void damon_commit_prep(struct damon_prep *dst, struct damon_prep *src) +{ + dst->action = src->action; +} + +static int damon_commit_preps(struct damon_probe *dst, struct damon_probe *src) +{ + struct damon_prep *dst_prep, *next, *src_prep, *new_prep; + int i = 0, j = 0; + + damon_for_each_prep_safe(dst_prep, next, dst) { + src_prep = damon_nth_prep(i++, src); + if (src_prep) + damon_commit_prep(dst_prep, src_prep); + else + damon_destroy_prep(dst_prep); + } + + damon_for_each_prep_safe(src_prep, next, src) { + if (j++ < i) + continue; + + new_prep = damon_new_prep(src_prep->action); + if (!new_prep) + return -ENOMEM; + damon_add_prep(dst, new_prep); + } + return 0; +} + static void damon_commit_filter(struct damon_filter *dst, struct damon_filter *src) { @@ -1757,6 +1810,9 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) src_probe = damon_nth_probe(i++, src); if (src_probe) { dst_probe->weight = src_probe->weight; + err = damon_commit_preps(dst_probe, src_probe); + if (err) + return err; err = damon_commit_filters(dst_probe, src_probe); if (err) return err; @@ -1774,6 +1830,9 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) return -ENOMEM; damon_add_probe(dst, new_probe); new_probe->weight = src_probe->weight; + err = damon_commit_preps(new_probe, src_probe); + if (err) + return err; err = damon_commit_filters(new_probe, src_probe); if (err) return err; From c174156e60f2b234b3370963118db76fa1effd0e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:54 -0700 Subject: [PATCH 170/539] mm/damon/core: introduce damon_operations->prep_probes() damon_prep needs to be executed by the underlying DAMON operation set. Extend the operation set callback list for the execution of damon_prep actions. If the underlying operation set implements the callback, DAMON core executes it in the monitoring preparation time. Link: https://lore.kernel.org/20260901132506.99243-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 5 +++++ mm/damon/core.c | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 1780c14942e6..871d26adf6ae 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -630,6 +630,7 @@ enum damon_ops_id { * @update: Update operations-related data structures. * @prepare_access_checks: Prepare next access check of target regions. * @check_accesses: Check the accesses to target regions. + * @prep_probes: Prepare applying probes for each region. * @apply_probes: Apply probes for each region. * @get_scheme_score: Get the score of a region for a scheme. * @apply_scheme: Apply a DAMON-based operation scheme. @@ -657,6 +658,9 @@ enum damon_ops_id { * last preparation and update the number of observed accesses of each region. * It should also return max number of observed accesses that made as a result * of its update. The value will be used for regions adjustment threshold. + * @prep_probes should execute required &struct damon_prep for next &struct + * damon_probe applications to each region. It should also set + * &damon_region->sampling_addr of each region if ``set_samples`` is true. * @apply_probes should apply the data attribute probes to each region and * accordingly update the probe hits counter of the region. It should also * set &damon_region->sampling_addr of each region if ``set_samples`` is true. @@ -679,6 +683,7 @@ struct damon_operations { void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); + void (*prep_probes)(struct damon_ctx *context, bool set_samples); unsigned int (*apply_probes)(struct damon_ctx *context, bool set_samples, bool return_max_wsum); int (*get_scheme_score)(struct damon_ctx *context, diff --git a/mm/damon/core.c b/mm/damon/core.c index c631b5324377..5ee2d0448a80 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -157,6 +157,18 @@ static struct damon_prep *damon_nth_prep(int n, struct damon_probe *p) return NULL; } +static bool damon_has_prep(struct damon_ctx *c) +{ + struct damon_prep *prep; + struct damon_probe *probe; + + damon_for_each_probe(probe, c) { + damon_for_each_prep(prep, probe) + return true; + } + return false; +} + struct damon_filter *damon_new_filter(enum damon_filter_type type, bool matching, bool allow) { @@ -3905,14 +3917,19 @@ static int kdamond_fn(void *data) unsigned long next_ops_update_sis = ctx->next_ops_update_sis; unsigned long sample_interval = ctx->attrs.sample_interval; bool access_check_disabled = damon_has_probe_weights(ctx); + bool do_prep; unsigned int max_merge_score = 0, max_wsum; bool get_max_wsum; if (kdamond_wait_activation(ctx)) break; + do_prep = ctx->ops.prep_probes && damon_has_prep(ctx); + if (!access_check_disabled && ctx->ops.prepare_access_checks) ctx->ops.prepare_access_checks(ctx); + if (do_prep) + ctx->ops.prep_probes(ctx, access_check_disabled); kdamond_usleep(sample_interval); ctx->passed_sample_intervals++; @@ -3927,7 +3944,8 @@ static int kdamond_fn(void *data) else get_max_wsum = false; max_wsum = ctx->ops.apply_probes(ctx, - access_check_disabled, get_max_wsum); + access_check_disabled && !do_prep, + get_max_wsum); if (get_max_wsum) max_merge_score = max_wsum; } From 81659cef89313e89f1d117a2ef0f9ac1cc23413e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:55 -0700 Subject: [PATCH 171/539] mm/damon/paddr: support damon_prep Implement damon_operations->prep_probes() callback. Support the only existing prep action, DAMON_PREP_SET_PGIDLE in a way similar to what it was doing for the access check preparation: unset page table accessed bits and set PG_Idle flag. Reuse the function for the access check preparation. Link: https://lore.kernel.org/20260901132506.99243-9-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/paddr.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 6f756f849389..c1e7d7a4f40d 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -105,6 +105,40 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) return max_nr_accesses; } +static void damon_pa_prep_probes_region(struct damon_region *r, + struct damon_probe *probe, struct damon_ctx *ctx) +{ + struct damon_prep *p; + + damon_for_each_prep(p, probe) { + switch (p->action) { + case DAMON_PREP_SET_PGIDLE: + damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr, + ctx->addr_unit)); + break; + default: + break; + } + } +} + +static void damon_pa_prep_probes(struct damon_ctx *ctx, bool set_samples) +{ + struct damon_target *t; + struct damon_region *r; + struct damon_probe *p; + + damon_for_each_target(t, ctx) { + damon_for_each_region(r, t) { + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); + damon_for_each_probe(p, ctx) + damon_pa_prep_probes_region(r, p, ctx); + } + } +} + static bool damon_pa_filter_match(struct damon_filter *filter, struct folio *folio) { @@ -448,6 +482,7 @@ static int __init damon_pa_initcall(void) .update = NULL, .prepare_access_checks = damon_pa_prepare_access_checks, .check_accesses = damon_pa_check_accesses, + .prep_probes = damon_pa_prep_probes, .apply_probes = damon_pa_apply_probes, .target_valid = NULL, .apply_scheme = damon_pa_apply_scheme, From 547e4892dbd9de968d3eab9c2a11daf7897d3a5f Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:56 -0700 Subject: [PATCH 172/539] mm/damon/sysfs: implement preps directory Implement a sysfs directory named 'preps' under the probe directory. It will be evolved to be used for specifying probe preps. Link: https://lore.kernel.org/20260901132506.99243-10-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 65 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index c1ff739dab27..35fb308039b6 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -749,6 +749,35 @@ static const struct kobj_type damon_sysfs_intervals_ktype = { .default_groups = damon_sysfs_intervals_groups, }; +/* + * preps directory + */ + +struct damon_sysfs_preps { + struct kobject kobj; +}; + +static struct damon_sysfs_preps *damon_sysfs_preps_alloc(void) +{ + return kzalloc_obj(struct damon_sysfs_preps); +} + +static void damon_sysfs_preps_release(struct kobject *kobj) +{ + kfree(container_of(kobj, struct damon_sysfs_preps, kobj)); +} + +static struct attribute *damon_sysfs_preps_attrs[] = { + NULL, +}; +ATTRIBUTE_GROUPS(damon_sysfs_preps); + +static const struct kobj_type damon_sysfs_preps_ktype = { + .release = damon_sysfs_preps_release, + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = damon_sysfs_preps_groups, +}; + /* * filter directory */ @@ -1069,6 +1098,7 @@ static const struct kobj_type damon_sysfs_filters_ktype = { struct damon_sysfs_probe { struct kobject kobj; unsigned int weight; + struct damon_sysfs_preps *preps; struct damon_sysfs_filters *filters; }; @@ -1079,25 +1109,48 @@ static struct damon_sysfs_probe *damon_sysfs_probe_alloc(void) static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *probe) { + struct damon_sysfs_preps *preps; struct damon_sysfs_filters *filters; int err; - filters = damon_sysfs_filters_alloc(); - if (!filters) + preps = damon_sysfs_preps_alloc(); + if (!preps) return -ENOMEM; + probe->preps = preps; + + err = kobject_init_and_add(&preps->kobj, &damon_sysfs_preps_ktype, + &probe->kobj, "preps"); + if (err) + goto put_preps_out; + + filters = damon_sysfs_filters_alloc(); + if (!filters) { + err = -ENOMEM; + goto del_preps_out; + } probe->filters = filters; err = kobject_init_and_add(&filters->kobj, &damon_sysfs_filters_ktype, &probe->kobj, "filters"); - if (err) { - kobject_put(&filters->kobj); - probe->filters = NULL; - } + if (err) + goto put_filters_out; + return err; + +put_filters_out: + kobject_put(&filters->kobj); + probe->filters = NULL; +del_preps_out: + kobject_del(&preps->kobj); +put_preps_out: + kobject_put(&preps->kobj); + probe->preps = NULL; return err; } static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe) { + if (probe->preps) + kobject_put(&probe->preps->kobj); if (probe->filters) { damon_sysfs_filters_rm_dirs(probe->filters); kobject_put(&probe->filters->kobj); From 3d634389c7fbf4b80cd398c984d6054d60bf5226 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:57 -0700 Subject: [PATCH 173/539] mm/damon/sysfs: implement preps/nr_preps file Implement nr_preps file under the preps directory. It will be evolved to be used for generating sub directories that will represent each probe preparation action. Link: https://lore.kernel.org/20260901132506.99243-11-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 35fb308039b6..b9722ebffd6f 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -755,6 +755,7 @@ static const struct kobj_type damon_sysfs_intervals_ktype = { struct damon_sysfs_preps { struct kobject kobj; + int nr; }; static struct damon_sysfs_preps *damon_sysfs_preps_alloc(void) @@ -762,12 +763,60 @@ static struct damon_sysfs_preps *damon_sysfs_preps_alloc(void) return kzalloc_obj(struct damon_sysfs_preps); } +static void damon_sysfs_preps_rm_dirs(struct damon_sysfs_preps *preps) +{ + preps->nr = 0; +} + +static int damon_sysfs_preps_add_dirs(struct damon_sysfs_preps *preps, + int nr_preps) +{ + preps->nr = nr_preps; + return 0; +} + +static ssize_t nr_preps_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct damon_sysfs_preps *preps = container_of(kobj, + struct damon_sysfs_preps, kobj); + + return sysfs_emit(buf, "%d\n", preps->nr); +} + +static ssize_t nr_preps_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_preps *preps; + int nr, err = kstrtoint(buf, 0, &nr); + + if (err) + return err; + if (nr < 0) + return -EINVAL; + + preps = container_of(kobj, struct damon_sysfs_preps, kobj); + + if (!mutex_trylock(&damon_sysfs_lock)) + return -EBUSY; + err = damon_sysfs_preps_add_dirs(preps, nr); + mutex_unlock(&damon_sysfs_lock); + if (err) + return err; + + return count; +} + static void damon_sysfs_preps_release(struct kobject *kobj) { kfree(container_of(kobj, struct damon_sysfs_preps, kobj)); } +static struct kobj_attribute damon_sysfs_preps_nr_attr = + __ATTR_RW_MODE(nr_preps, 0600); + static struct attribute *damon_sysfs_preps_attrs[] = { + &damon_sysfs_preps_nr_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_preps); @@ -1149,8 +1198,10 @@ put_preps_out: static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe) { - if (probe->preps) + if (probe->preps) { + damon_sysfs_preps_rm_dirs(probe->preps); kobject_put(&probe->preps->kobj); + } if (probe->filters) { damon_sysfs_filters_rm_dirs(probe->filters); kobject_put(&probe->filters->kobj); From df1da9073d78fe7d55f1b3463b74484932528c57 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:58 -0700 Subject: [PATCH 174/539] mm/damon/sysfs: create directories for nr_preps writes Implement nr_preps write action to actually create subdirectories of the number. Each of the directory will be evolved to represent each probe preparation action. Link: https://lore.kernel.org/20260901132506.99243-12-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index b9722ebffd6f..4ff473b68953 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -749,12 +749,50 @@ static const struct kobj_type damon_sysfs_intervals_ktype = { .default_groups = damon_sysfs_intervals_groups, }; +/* + * prep directory + */ + +struct damon_sysfs_prep { + struct kobject kobj; +}; + +static struct damon_sysfs_prep *damon_sysfs_prep_alloc(void) +{ + struct damon_sysfs_prep *prep; + + prep = kzalloc_obj(struct damon_sysfs_prep); + if (!prep) + return prep; + return prep; +} + +static void damon_sysfs_prep_release(struct kobject *kobj) +{ + struct damon_sysfs_prep *prep = container_of(kobj, + struct damon_sysfs_prep, kobj); + + kfree(prep); +} + +static struct attribute *damon_sysfs_prep_attrs[] = { + NULL, +}; +ATTRIBUTE_GROUPS(damon_sysfs_prep); + +static const struct kobj_type damon_sysfs_prep_ktype = { + .release = damon_sysfs_prep_release, + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = damon_sysfs_prep_groups, +}; + /* * preps directory */ struct damon_sysfs_preps { struct kobject kobj; + struct damon_sysfs_prep **preps_arr; int nr; }; @@ -765,13 +803,53 @@ static struct damon_sysfs_preps *damon_sysfs_preps_alloc(void) static void damon_sysfs_preps_rm_dirs(struct damon_sysfs_preps *preps) { + struct damon_sysfs_prep **preps_arr = preps->preps_arr; + int i; + + for (i = 0; i < preps->nr; i++) { + kobject_del(&preps_arr[i]->kobj); + kobject_put(&preps_arr[i]->kobj); + } preps->nr = 0; + kfree(preps_arr); + preps->preps_arr = NULL; } static int damon_sysfs_preps_add_dirs(struct damon_sysfs_preps *preps, int nr_preps) { - preps->nr = nr_preps; + struct damon_sysfs_prep **preps_arr, *prep; + int err, i; + + damon_sysfs_preps_rm_dirs(preps); + if (!nr_preps) + return 0; + + preps_arr = kmalloc_objs(*preps_arr, nr_preps, + GFP_KERNEL | __GFP_NOWARN); + if (!preps_arr) + return -ENOMEM; + preps->preps_arr = preps_arr; + + for (i = 0; i < nr_preps; i++) { + prep = damon_sysfs_prep_alloc(); + if (!prep) { + damon_sysfs_preps_rm_dirs(preps); + return -ENOMEM; + } + + err = kobject_init_and_add(&prep->kobj, + &damon_sysfs_prep_ktype, &preps->kobj, "%d", + i); + if (err) { + kobject_put(&prep->kobj); + damon_sysfs_preps_rm_dirs(preps); + return err; + } + + preps_arr[i] = prep; + preps->nr++; + } return 0; } From 3eaa9c5b4b2a59cb15b2380b566614dfefcf0408 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:24:59 -0700 Subject: [PATCH 175/539] mm/damon/sysfs: implement prep_action file Add a file named prep_action under the prep directory. It represents the corresponding probe preparation action. Link: https://lore.kernel.org/20260901132506.99243-13-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 4ff473b68953..2118089dd9d8 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -755,6 +755,7 @@ static const struct kobj_type damon_sysfs_intervals_ktype = { struct damon_sysfs_prep { struct kobject kobj; + enum damon_prep_action action; }; static struct damon_sysfs_prep *damon_sysfs_prep_alloc(void) @@ -764,9 +765,61 @@ static struct damon_sysfs_prep *damon_sysfs_prep_alloc(void) prep = kzalloc_obj(struct damon_sysfs_prep); if (!prep) return prep; + prep->action = DAMON_PREP_SET_PGIDLE; return prep; } +struct damon_sysfs_prep_action_name { + const enum damon_prep_action action; + const char *name; +}; + +static const struct damon_sysfs_prep_action_name +damon_sysfs_prep_action_names[] = { + { + .action = DAMON_PREP_SET_PGIDLE, + .name = "set_pgidle", + }, +}; + +static ssize_t prep_action_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_prep *prep = container_of(kobj, + struct damon_sysfs_prep, kobj); + int i; + + for (i = 0; i < ARRAY_SIZE(damon_sysfs_prep_action_names); i++) { + const struct damon_sysfs_prep_action_name *action_name; + + action_name = &damon_sysfs_prep_action_names[i]; + if (action_name->action == prep->action) + return sysfs_emit(buf, "%s\n", action_name->name); + } + return -EINVAL; +} + +static ssize_t prep_action_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_prep *prep = container_of(kobj, + struct damon_sysfs_prep, kobj); + ssize_t ret = -EINVAL; + int i; + + for (i = 0; i < ARRAY_SIZE(damon_sysfs_prep_action_names); i++) { + const struct damon_sysfs_prep_action_name *action_name; + + action_name = &damon_sysfs_prep_action_names[i]; + if (sysfs_streq(buf, action_name->name)) { + prep->action = action_name->action; + ret = count; + break; + } + } + return ret; +} + static void damon_sysfs_prep_release(struct kobject *kobj) { struct damon_sysfs_prep *prep = container_of(kobj, @@ -775,7 +828,11 @@ static void damon_sysfs_prep_release(struct kobject *kobj) kfree(prep); } +static struct kobj_attribute damon_sysfs_prep_prep_action_attr = + __ATTR_RW_MODE(prep_action, 0600); + static struct attribute *damon_sysfs_prep_attrs[] = { + &damon_sysfs_prep_prep_action_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_prep); From c3b3a240534022824aae97f6ea61cb13751389fe Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:25:00 -0700 Subject: [PATCH 176/539] mm/damon/sysfs: pass preps to DAMON core DAMON sysfs interface provides the files for setting DAMON probe preps. But the underlying code is not really passing the user-set values to DAMON core. Pass those. Link: https://lore.kernel.org/20260901132506.99243-14-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 2118089dd9d8..7f340b6f1921 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -812,7 +812,10 @@ static ssize_t prep_action_store(struct kobject *kobj, action_name = &damon_sysfs_prep_action_names[i]; if (sysfs_streq(buf, action_name->name)) { + if (!mutex_trylock(&damon_sysfs_lock)) + return -EBUSY; prep->action = action_name->action; + mutex_unlock(&damon_sysfs_lock); ret = count; break; } @@ -2175,6 +2178,23 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, return damon_set_attrs(ctx, &attrs); } +static int damon_sysfs_set_preps(struct damon_probe *probe, + struct damon_sysfs_preps *sys_preps) +{ + int i; + + for (i = 0; i < sys_preps->nr; i++) { + struct damon_sysfs_prep *sys_prep = sys_preps->preps_arr[i]; + struct damon_prep *prep; + + prep = damon_new_prep(sys_prep->action); + if (!prep) + return -ENOMEM; + damon_add_prep(probe, prep); + } + return 0; +} + static int damon_sysfs_set_filters(struct damon_probe *probe, struct damon_sysfs_filters *sys_filters) { @@ -2210,7 +2230,15 @@ static int damon_sysfs_set_probe(struct damon_probe *probe, struct damon_sysfs_probe *sys_probe) { struct damon_sysfs_filters *sys_filters; + struct damon_sysfs_preps *sys_preps; + int err; + sys_preps = sys_probe->preps; + if (sys_preps) { + err = damon_sysfs_set_preps(probe, sys_preps); + if (err) + return err; + } sys_filters = sys_probe->filters; if (!sys_filters) return 0; From 9a7c8a261a438288b7ac43016c2beffc32ecab08 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:25:01 -0700 Subject: [PATCH 177/539] selftests/damon/sysfs.sh: test probe prep sysfs files Add basic file operations test for newly introduced DAMON probe prep sysfs directories and files. Link: https://lore.kernel.org/20260901132506.99243-15-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/damon/sysfs.sh | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index f7fb94b84e71..ddebde6edabe 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -361,6 +361,32 @@ test_intervals() test_intervals_goal "$intervals_dir/intervals_goal" } +test_damon_prep() +{ + damon_prep_dir=$1 + ensure_file "$damon_prep_dir/prep_action" "exist" "600" + ensure_write_succ "$damon_prep_dir/prep_action" "set_pgidle" \ + "valid input" + ensure_write_fail "$damon_prep_dir/prep_action" "foo" "invalid input" +} + +test_damon_preps() +{ + preps_dir=$1 + ensure_dir "$preps_dir" "exist" + ensure_file "$preps_dir/nr_preps" "exist" "600" + ensure_write_succ "$preps_dir/nr_preps" "1" "valid input" + test_damon_prep "$preps_dir/0" + + ensure_write_succ "$preps_dir/nr_preps" "2" "valid input" + test_damon_prep "$preps_dir/0" + test_damon_prep "$preps_dir/1" + + ensure_write_succ "$preps_dir/nr_preps" "0" "valid input" + ensure_dir "$preps_dir/0" "not_exist" + ensure_dir "$preps_dir/1" "not_exist" +} + test_damon_filter() { damon_filter_dir=$1 @@ -392,6 +418,7 @@ test_probe() { probe_dir=$1 ensure_dir "$probe_dir" "exist" + test_damon_preps "$probe_dir/preps" test_damon_filters "$probe_dir/filters" } From fac5fa3517b1bd457a500bc289dbc7997f85b6b3 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:25:02 -0700 Subject: [PATCH 178/539] Docs/mm/damon/design: document probe preps Update DAMON design document for the newly added DAMON probe preps feature. Link: https://lore.kernel.org/20260901132506.99243-16-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 947d91ae24a3..d036340dae8a 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -309,6 +309,13 @@ Users can therefore know how much of a given DAMON region has a specific data attribute by reading the per-region per-probe probe hits counter after each aggregation interval. +Users can optionally register probing preparation actions per probe. If such +actions are registered, DAMON applies the actions to each region's sampling +memory before starting the next sampling interval. Currently only one action, +``set_pgidle`` is supported. The action marks the page for the probing target +memory as access-idle. This can be useful to be used together with +``pgidle_unset`` probe filter. + This is a sampling based mechanism. Hence, it is lightweight but the output may include some measurement errors. The output should be used with good understanding of statistics. From 75db6c73c52b97a881339c836a1cf3d4b250ef6c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:25:03 -0700 Subject: [PATCH 179/539] Docs/admin-guide/mm/damon/usage: document probe preps sysfs files Update DAMON usage document for the newly added DAMON probe preps sysfs files. Link: https://lore.kernel.org/20260901132506.99243-17-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/admin-guide/mm/damon/usage.rst | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index da5f9afd08ae..023c6334024f 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -74,6 +74,9 @@ comma (","). │ │ │ │ │ │ nr_regions/min,max │ │ │ │ │ │ :ref:`probes `/nr_probes │ │ │ │ │ │ │ 0/weight + │ │ │ │ │ │ │ │ preps/nr_preps + │ │ │ │ │ │ │ │ │ 0/prep_action + │ │ │ │ │ │ │ │ │ ... │ │ │ │ │ │ │ │ filters/nr_filters │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path │ │ │ │ │ │ │ │ │ ... @@ -283,9 +286,18 @@ In the beginning, this directory has only one file, ``nr_probes``. Writing a number (``N``) to the file creates the number of child directories named ``0`` to ``N-1``. Each directory represents each monitoring probe. -In each probe directory, one directory, ``filters`` exists. The directory -contains files for installing filters for the probe, that is used to determine -the data attribute for the probe. +In each probe directory, two directories, ``preps`` and ``filters`` exist. The +directories contain files for installing probing preparation actions and +filters for the probe, that are used to determine the data attribute for the +probe. + +In the beginning, ``preps`` directory has only one file, ``nr_preps``. +Writing a number (``N``) to the file creates the number of child directories +named ``0`` to ``N-1``. Each directory represents each preparation action. +Each directory has one file, ``prep_action``. The preparation action can be +selected by writing the name of the action to the ``prep_action`` file. Refer +to the :ref:`design doc ` for the list of +supported actions. Each probe directory also contains ``weight`` file. Reading from and writing to the file gets and sets the :ref:`attributes-only monitoring From 6770d8b103b0a583f1164c8ad5a058bc5e650102 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 06:25:04 -0700 Subject: [PATCH 180/539] Docs/ABI/damon: document probe prep sysfs files Update DAMON ABI document for the newly added DAMON probe prep sysfs files. Link: https://lore.kernel.org/20260901132506.99243-18-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index e675a57145e3..f8d2601e8290 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -173,6 +173,19 @@ Contact: SJ Park Description: Writing to and reading from this file sets and gets the per-probe attribute weight. +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/preps/nr_preps +Date: Jun 2026 +Contact: SJ Park +Description: Writing a number 'N' to this file creates the number of + directories for each DAMON probing preparation action named '0' + to 'N-1' under the preps/ directory. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/preps//prep_action +Date: Jun 2026 +Contact: SJ Park +Description: Writing to and reading from this file sets and gets the probing + preparation action. + What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters/nr_filters Date: May 2026 Contact: SJ Park From 6a100811c7c5fa39d452ff2e380bd14efbc0208e Mon Sep 17 00:00:00 2001 From: Qinyun Tan Date: Tue, 1 Sep 2026 19:51:04 +0800 Subject: [PATCH 181/539] mm/list_lru: don't copy stale shrinker id from non-memcg-aware shrinkers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With cgroup.memory=nokmem, shrinker_memcg_alloc() fails with -ENOSYS for shrinkers without SHRINKER_NONSLAB, and shrinker_alloc() falls back to a non-memcg-aware shrinker. On this fallback path, shrinker->id is never assigned and keeps 0 from kzalloc(), which is a valid id belonging to whichever memcg-aware shrinker registers first. __list_lru_init() copies shrinker->id unconditionally, so every list_lru backed by such a fallback shrinker (thp-deferred_split, zswap-shrinker, workingset shadow nodes, superblock lrus, ...) ends up with lru->shrinker_id == 0 instead of -1. Under nokmem the list_lru collapses to the shared per-node lists, but __list_lru_add() still calls set_shrinker_bit() against the memcg of the added object. Most list_lru users are unaffected because their objects resolve to a NULL memcg without kmem accounting, but the THP deferred split queue holds user folios, which are charged regardless of nokmem. On a system where no SHRINKER_NONSLAB shrinker registers, shrinker_nr_max stays 0 and every memcg's shrinker_info has map_nr_max == 0, so the first folio added by khugepaged triggers on every boot: WARNING: mm/shrinker.c:212 at set_shrinker_bit+0x99/0xa0 On systems where a SHRINKER_NONSLAB shrinker (btrfs, xfs) did register and expand the maps, there is no warning; instead bit 0 is set spuriously for an unrelated shrinker. shrinker->id is only meaningful while SHRINKER_MEMCG_AWARE is set, and all readers inside mm/shrinker.c already check the flag before using the id. Make __list_lru_init() do the same and fall back to -1, so set_shrinker_bit() is never reached with a bogus id. The stale shrinker->id itself is left as is; cleaning that up is a separate topic. Verified on a machine booting with cgroup.memory=nokmem and CONFIG_TRANSPARENT_HUGEPAGE=y: the warning fires once per boot from khugepaged, disappears when nokmem is removed from the command line, and no longer triggers with this fix applied and nokmem set. Link: https://lore.kernel.org/20260901115104.2944996-1-qinyuntan@linux.alibaba.com Fixes: fafaeceb89a5e ("mm: switch deferred split shrinker to list_lru") Signed-off-by: Qinyun Tan Signed-off-by: Andrew Morton Acked-by: Muchun Song Reviewed-by: Baolin Wang Reported-by: Wentao Guan Closes: https://lore.kernel.org/linux-mm/20260904190028.21542-1-guanwentao@uniontech.com/ Tested-by: Wentao Guan Cc: Michal Hocko Cc: Roman Gushchin Cc: Johannes Weiner Cc: Shakeel Butt Cc: Dave Chinner Cc: David Hildenbrand Cc: Lance Yang Cc: Michal Koutný Cc: Xunlei Pang Cc: --- mm/list_lru.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mm/list_lru.c b/mm/list_lru.c index a4522ca93ebc..8a6dd0a489e1 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -666,7 +666,12 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware, struct shrinker *shr int i; #ifdef CONFIG_MEMCG - if (shrinker) + /* + * If the shrinker fell back to being non-memcg-aware (e.g. with + * cgroup.memory=nokmem), its id was never assigned and holds a + * stale 0. Don't let set_shrinker_bit() act on it. + */ + if (shrinker && (shrinker->flags & SHRINKER_MEMCG_AWARE)) lru->shrinker_id = shrinker->id; else lru->shrinker_id = -1; From 73c77f98e234a493b61314e539860aaba4c590e5 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 1 Sep 2026 15:38:04 -0400 Subject: [PATCH 182/539] mm: remove unused mark_page_reserved() Patch series "mm: remove three unused helpers from mm.h", v2. I happened to notice these were unused. Two of them are relatively recently unused, and one has been unused for a few years. Remove them. This patch (of 2): mark_page_reserved() lost its last caller in commit 6215d9f4470f ("arch, mm: consolidate empty_zero_page"). Remove it. Link: https://lore.kernel.org/20260901-mm-remove-unused-helpers-v2-0-f6474e169c23@columbia.edu Link: https://lore.kernel.org/20260901-mm-remove-unused-helpers-v2-1-f6474e169c23@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/mm.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 1b28e6fc8d5d..e3736c42c4db 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4080,12 +4080,6 @@ static inline void free_reserved_page(struct page *page) free_reserved_pages(page, 0); } -static inline void mark_page_reserved(struct page *page) -{ - SetPageReserved(page); - adjust_managed_page_count(page, -1); -} - static inline void free_reserved_ptdesc(struct ptdesc *pt) { free_reserved_page(ptdesc_page(pt)); From 54c4c04553ce8ff49109ec2a1a50cf6114b76674 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 1 Sep 2026 15:38:05 -0400 Subject: [PATCH 183/539] mm: remove unused totalram_pages_inc() and totalram_pages_dec() totalram_pages_inc() and totalram_pages_dec() have had no callers since commit 7fbc5e26123e ("memblock: extract page freeing from free_reserved_area() into a helper") and commit 287b89773d81 ("powerpc/pseries/cmm: Use adjust_managed_page_count() insted of totalram_pages_*"), respectively. Remove them. Drop the totalram_pages_inc() stub from tools mm.h too. Link: https://lore.kernel.org/20260901-mm-remove-unused-helpers-v2-2-f6474e169c23@columbia.edu Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Mike Rapoport (Microsoft) Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/mm.h | 10 ---------- tools/include/linux/mm.h | 4 ---- 2 files changed, 14 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index e3736c42c4db..c105a3758915 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -57,16 +57,6 @@ static inline unsigned long totalram_pages(void) return (unsigned long)atomic_long_read(&_totalram_pages); } -static inline void totalram_pages_inc(void) -{ - atomic_long_inc(&_totalram_pages); -} - -static inline void totalram_pages_dec(void) -{ - atomic_long_dec(&_totalram_pages); -} - static inline void totalram_pages_add(long count) { atomic_long_add(count, &_totalram_pages); diff --git a/tools/include/linux/mm.h b/tools/include/linux/mm.h index 84b5954f66c3..d586a510e6e1 100644 --- a/tools/include/linux/mm.h +++ b/tools/include/linux/mm.h @@ -33,10 +33,6 @@ static inline phys_addr_t virt_to_phys(volatile void *address) return (phys_addr_t)address; } -static inline void totalram_pages_inc(void) -{ -} - static inline void totalram_pages_add(long count) { } From 7d5ae6e5d43946c8a88f8b8011d79b0cbbf614a7 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Wed, 2 Sep 2026 01:53:01 +0900 Subject: [PATCH 184/539] percpu: remove redundant assignments to bits Patch series "percpu: remove code with no effect". While reading the percpu initialization path, I found some minor cleanups for unnecessary initializations and an obsolete return statement. No functional change. This patch (of 4): pcpu_chunk_refresh_hint() and pcpu_find_block_fit() set bits to 0 and later call pcpu_next_md_free_region() or pcpu_next_fit_region(), which unconditionally set *bits to 0. Nothing uses it in between, so the assignments have no effect. Remove redundant assignments. No functional change. Link: https://lore.kernel.org/20260901165307.1026248-1-ekffu200098@gmail.com Link: https://lore.kernel.org/20260901165307.1026248-2-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Signed-off-by: Andrew Morton Cc: Dennis Zhou Cc: Tejun Heo Cc: Christoph Lameter --- mm/percpu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/percpu.c b/mm/percpu.c index 47a903fe3b51..b094c617147c 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -758,7 +758,6 @@ static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan) chunk_md->contig_hint = 0; } - bits = 0; pcpu_for_each_md_free_region(chunk, bit_off, bits) pcpu_block_update(chunk_md, bit_off, bit_off + bits); } @@ -1122,7 +1121,6 @@ static int pcpu_find_block_fit(struct pcpu_chunk *chunk, int alloc_bits, return -1; bit_off = pcpu_next_hint(chunk_md, alloc_bits); - bits = 0; pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) { if (!pop_only || pcpu_is_populated(chunk, bit_off, bits, &next_off)) From 229ca08b9428389245ee1b2b7e2a153462d836c6 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Wed, 2 Sep 2026 01:53:02 +0900 Subject: [PATCH 185/539] percpu: remove unnecessary initialization in pcpu_build_alloc_info() pcpu_build_alloc_info() initializes nr_groups to 1 and unconditionally sets it to the number of groups. Nothing uses it in between, so the initialization has no effect. Remove unnecessary initialization. No functional change. Link: https://lore.kernel.org/20260901165307.1026248-3-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Signed-off-by: Andrew Morton Cc: Christoph Lameter Cc: Dennis Zhou Cc: Tejun Heo --- mm/percpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/percpu.c b/mm/percpu.c index b094c617147c..0ec9b2adfc20 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -2817,7 +2817,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( static int group_cnt[NR_CPUS] __initdata; static struct cpumask mask __initdata; const size_t static_size = __per_cpu_end - __per_cpu_start; - int nr_groups = 1, nr_units = 0; + int nr_groups, nr_units = 0; size_t size_sum, min_unit_size, alloc_size; int upa, max_upa, best_upa; /* units_per_alloc */ int last_allocs, group, unit; From ce1e31297a5c2e2008cac4b03e07ee88f5704ce6 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Wed, 2 Sep 2026 01:53:03 +0900 Subject: [PATCH 186/539] percpu: remove unnecessary cpumask_clear() in pcpu_build_alloc_info() pcpu_build_alloc_info() clears mask and unconditionally sets it to cpu_possible_mask. Nothing uses it in between, so cpumask_clear() has no effect. Remove unnecessary cpumask_clear(). No functional change. Link: https://lore.kernel.org/20260901165307.1026248-4-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Signed-off-by: Andrew Morton Cc: Christoph Lameter Cc: Dennis Zhou Cc: Tejun Heo --- mm/percpu.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/percpu.c b/mm/percpu.c index 0ec9b2adfc20..fe9fe5a0c3d5 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -2828,7 +2828,6 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( /* this function may be called multiple times */ memset(group_map, 0, sizeof(group_map)); memset(group_cnt, 0, sizeof(group_cnt)); - cpumask_clear(&mask); /* calculate size_sum and ensure dyn_size is enough for early alloc */ size_sum = PFN_ALIGN(static_size + reserved_size + From f2a50fd3127c9cab7d84a774fcbbfc8b0cfc1f9e Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Wed, 2 Sep 2026 01:53:04 +0900 Subject: [PATCH 187/539] percpu: remove unnecessary return in pcpu_populate_pte() Since commit c6f239796b55 ("mm/memblock: add memblock_alloc_or_panic interface"), pcpu_populate_pte() no longer has an error label after the return statement, so the return statement has no effect. Remove unnecessary return. No functional change. Link: https://lore.kernel.org/20260901165307.1026248-5-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Signed-off-by: Andrew Morton Cc: Christoph Lameter Cc: Dennis Zhou Cc: Tejun Heo --- mm/percpu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/percpu.c b/mm/percpu.c index fe9fe5a0c3d5..3eff382e565a 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -3178,8 +3178,6 @@ void __init __weak pcpu_populate_pte(unsigned long addr) new = memblock_alloc_or_panic(PTE_TABLE_SIZE, PTE_TABLE_SIZE); pmd_populate_kernel(&init_mm, pmd, new); } - - return; } /** From eb2c18a5f1173716cc5679cd4f35680590b286d2 Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Tue, 1 Sep 2026 14:13:05 +0900 Subject: [PATCH 188/539] zram: remove unreachable kernel_read_file_from_path() return check Sashiko reported that: kernel_read_file_from_path() returns negative error for zero-sized files, so we cannot have "sz == 0" return, remove it and use a generic error message instead. Link: https://lore.kernel.org/20260901051335.2202390-1-senozhatsky@chromium.org Signed-off-by: Sergey Senozhatsky Signed-off-by: Andrew Morton Cc: Haoqin Huang --- drivers/block/zram/zram_drv.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 4ba0f77b2abd..2359eaa6f531 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1722,11 +1722,6 @@ static int comp_params_store(struct zram *zram, u32 prio, s32 level, dict_path, sz); return sz; } - if (sz == 0) { - pr_err("failed to load dictionary %s (empty file)\n", - dict_path); - return -EINVAL; - } } zram->params[prio].dict_sz = sz; From 938d5db367967e7dec5db0a6fd189112deafb8b6 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 17:27:20 -0700 Subject: [PATCH 189/539] mm/damon/tests/core-kunit: test committing psi goal to psi goal Patch series "mm/damon: fix misc bugs in kunit, quota goals and sysfs refresh_ms". DAMOS quota goals commit unit test is mistakenly not testing a test case that was designed to test. DAMOS quota goals and DAMON sysfs refresh_ms file have bugs that can produce non critical but still unexpected behaviors. Fix the bugs. Patch 1 fixes the DAMOS quota goals commit unit test to cover a mistakenly uncovered case. Patches 2 and 3 fix the bugs in DAMOS PSI goal initialization and eligible_mem_bp online commit, respectively. Patch 4 fixes the bug in DAMON sysfs refresh_ms file handling. This patch (of 4): damon_test_commit_quota_goal_for() is set to test committing a new psi goal on an existing psi goal. However, damon_test_commit_quota_goal() is mistakenly not covering the test case. Add the test case. Link: https://lore.kernel.org/20260902002725.108635-1-sj@kernel.org Link: https://lore.kernel.org/20260902002725.108635-2-sj@kernel.org Fixes: 99f89debafc5 ("mm/damon/tests/core-kunit: add damos_commit_quota_goal() test") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Lian Wang Tested-by: Lian Wang Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: Quanmin Yan Cc: # 6.19.x --- mm/damon/tests/core-kunit.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 7071ec277b00..f1e11548c771 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -793,6 +793,13 @@ static void damos_test_commit_quota_goal(struct kunit *test) .last_psi_total = 456, }; + damos_test_commit_quota_goal_for(test, &dst, + &(struct damos_quota_goal) { + .metric = DAMOS_QUOTA_SOME_MEM_PSI_US, + .target_value = 234, + .current_value = 345, + .last_psi_total = 567, + }); damos_test_commit_quota_goal_for(test, &dst, &(struct damos_quota_goal){ .metric = DAMOS_QUOTA_USER_INPUT, From 29c6084658af41b5b1fefd873d26d441830ced34 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 17:27:21 -0700 Subject: [PATCH 190/539] mm/damon/core: handle uninitialized damos_quota_goal->last_psi_total When DAMOS_QUOTA_SOME_MEM_PSI_US metric damos quota goal is set, the PSI delta for the feedback loop is calculated using damos_quota_goal->last_psi_total. However, it is initialized only after the first feedback loop. The first iteration of the loop uses the uninitialized value. As a result, the feedback loop can change the effective quota in an unexpected way at the first iteration. The user impact of the issue is not big, because the issue impacts only the first iteration of the feedback loop. The feedback loop also has an internal cap of the quota adjustment. The wrong adjustment will soon be corrected over a few iterations. For this reason, doing no initialization at commit time was intentional. It is also explicitly commented. That said, nobody likes behaviors that are unexpected or difficult to be expected. Check last_psi_total initialization and skip the tuning round when it is not initialized. For this, initialize last_psi_total with U64_MAX in the goal creation and the goal commit time. U64_MAX means the field is not initialized. The tuning round shows the value and adjusts it to guarantee the current quota is maintained for the round, and last_psi_total is correctly initialized on the next round. Before this change, committing a new PSI goal on an existing PSI goal with goal-only DAMON sysfs command (commit_schemes_quota_goals) just worked. After this change, the tuning round right after the commit will be unnecessarily skipped, because last_psi_total is unconditionally marked as not initialized in the damos_commit_quota_goal_union(). This is an intended tradeoff for simplicity. Skipping just one round of tuning is no problem. Meanwhile it makes both the code and the behavior simple to understand. Also update the quota goal commit unit test for changed last_psi_total setup behavior. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260902002725.108635-3-sj@kernel.org Link: https://lore.kernel.org/20260718005316.89585-1-sj@kernel.org [1] Fixes: 2dbb60f789cb ("mm/damon/core: implement PSI metric DAMOS quota goal") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Lian Wang Tested-by: Lian Wang Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: Quanmin Yan Cc: # 6.9.x --- mm/damon/core.c | 13 +++++++++++-- mm/damon/tests/core-kunit.h | 9 +++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 5ee2d0448a80..f950a3b9fcb9 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -697,6 +697,8 @@ struct damos_quota_goal *damos_new_quota_goal( return NULL; goal->metric = metric; goal->target_value = target_value; + if (metric == DAMOS_QUOTA_SOME_MEM_PSI_US) + goal->last_psi_total = U64_MAX; INIT_LIST_HEAD(&goal->list); return goal; } @@ -1190,6 +1192,9 @@ static void damos_commit_quota_goal_union( struct damos_quota_goal *dst, struct damos_quota_goal *src) { switch (dst->metric) { + case DAMOS_QUOTA_SOME_MEM_PSI_US: + dst->last_psi_total = U64_MAX; + break; case DAMOS_QUOTA_NODE_MEM_USED_BP: case DAMOS_QUOTA_NODE_MEM_FREE_BP: dst->nid = src->nid; @@ -1211,7 +1216,6 @@ static void damos_commit_quota_goal( dst->target_value = src->target_value; if (dst->metric == DAMOS_QUOTA_USER_INPUT) dst->current_value = src->current_value; - /* keep last_psi_total as is, since it will be updated in next cycle */ damos_commit_quota_goal_union(dst, src); } @@ -3139,7 +3143,12 @@ static void damos_set_quota_goal_current_value(struct damon_ctx *c, break; case DAMOS_QUOTA_SOME_MEM_PSI_US: now_psi_total = damos_get_some_mem_psi_total(); - goal->current_value = now_psi_total - goal->last_psi_total; + /* uninitialized last_psi_total; make no effect this round */ + if (goal->last_psi_total == U64_MAX) + goal->current_value = goal->target_value; + else + goal->current_value = now_psi_total - + goal->last_psi_total; goal->last_psi_total = now_psi_total; break; case DAMOS_QUOTA_NODE_MEM_USED_BP: diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index f1e11548c771..af26b3d60957 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -757,19 +757,16 @@ static void damos_test_commit_quota_goal_for(struct kunit *test, struct damos_quota_goal *dst, struct damos_quota_goal *src) { - u64 dst_last_psi_total = 0; - - if (dst->metric == DAMOS_QUOTA_SOME_MEM_PSI_US) - dst_last_psi_total = dst->last_psi_total; damos_commit_quota_goal(dst, src); KUNIT_EXPECT_EQ(test, dst->metric, src->metric); KUNIT_EXPECT_EQ(test, dst->target_value, src->target_value); if (src->metric == DAMOS_QUOTA_USER_INPUT) KUNIT_EXPECT_EQ(test, dst->current_value, src->current_value); - if (dst_last_psi_total && src->metric == DAMOS_QUOTA_SOME_MEM_PSI_US) - KUNIT_EXPECT_EQ(test, dst->last_psi_total, dst_last_psi_total); switch (dst->metric) { + case DAMOS_QUOTA_SOME_MEM_PSI_US: + KUNIT_EXPECT_EQ(test, dst->last_psi_total, U64_MAX); + break; case DAMOS_QUOTA_NODE_MEM_USED_BP: case DAMOS_QUOTA_NODE_MEM_FREE_BP: KUNIT_EXPECT_EQ(test, dst->nid, src->nid); From df7d9a0f8d916eeea375430ae3360724afecde15 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Tue, 22 Sep 2026 22:51:37 -0700 Subject: [PATCH 191/539] mm/damon/core: keep the temporal tuner quota over an unmeasured PSI round The patch in mm-unstable of subject "mm/damon/core: handle uninitialized damos_quota_goal->last_psi_total" scores a PSI quota goal without a previous sample as achieved. This leaves the consist tuner's input unchanged, but the temporal tuner sets its quota to zero for an achieved goal. A running scheme with a nonzero temporal quota therefore loses a charge window after a quota-goal commit. Use the effective quota to preserve the temporal tuner's previous goal-achievement state during an unmeasured round, as SJ suggested [1]. Score the goal as achieved when the effective quota is zero and as not achieved otherwise. A new scheme's initially zero quota stays zero, and the consist tuner's behavior is unchanged. Move the PSI current-value calculation and last_psi_total update into a helper that takes the current PSI total. This lets a unit test cover the unmeasured and measured rounds without depending on system memory pressure. Link: https://lore.kernel.org/20260923055139.2982-1-sj@kernel.org Link: https://lore.kernel.org/damon/20260916001311.101024-1-sj@kernel.org/ [1] Fixes: 2dbb60f789cb ("mm/damon/core: implement PSI metric DAMOS quota goal") Signed-off-by: Karl Mehltretter Signed-off-by: SJ Park Signed-off-by: Andrew Morton Suggested-by: SJ Park Reviewed-by: SJ Park Reviewed-by: Kunwu Chan Assisted-by: LLM Cc: Lian Wang Cc: # 6.9.x --- mm/damon/core.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index f950a3b9fcb9..71e8cedbc643 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2891,6 +2891,28 @@ static inline u64 damos_get_some_mem_psi_total(void) #endif /* CONFIG_PSI */ +static void damos_set_psi_current_val(u64 now_psi_total, + struct damos_quota_goal *goal, struct damos *s) +{ + u64 last_psi_total = goal->last_psi_total; + + goal->last_psi_total = now_psi_total; + if (last_psi_total != U64_MAX) { + goal->current_value = now_psi_total - last_psi_total; + return; + } + /* uninitialized last_psi_total; make no effect this round */ + if (s->quota.goal_tuner == DAMOS_QUOTA_GOAL_TUNER_CONSIST) { + goal->current_value = goal->target_value; + return; + } + /* let temporal tuner show the same achievement as in the last round */ + if (!s->quota.esz) + goal->current_value = goal->target_value; + else + goal->current_value = 0; +} + #ifdef CONFIG_NUMA static bool invalid_mem_node(int nid) { @@ -3143,13 +3165,7 @@ static void damos_set_quota_goal_current_value(struct damon_ctx *c, break; case DAMOS_QUOTA_SOME_MEM_PSI_US: now_psi_total = damos_get_some_mem_psi_total(); - /* uninitialized last_psi_total; make no effect this round */ - if (goal->last_psi_total == U64_MAX) - goal->current_value = goal->target_value; - else - goal->current_value = now_psi_total - - goal->last_psi_total; - goal->last_psi_total = now_psi_total; + damos_set_psi_current_val(now_psi_total, goal, s); break; case DAMOS_QUOTA_NODE_MEM_USED_BP: case DAMOS_QUOTA_NODE_MEM_FREE_BP: From d4a8b5ac2ec28f55bba9fad07b69bf375cd42704 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 17:27:22 -0700 Subject: [PATCH 192/539] mm/damon/core: copy nid for eligible_mem_bp damos quota goal commit damos_commit_quota_goal_union() is not updating the ->nid union field when the goal metric is DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP. Hence, if a DAMOS quota goal of the type is online committed in a way that it will reuse other quota goal's memory space, the new goal will work with a garbage nid value. As a result, the DAMOS scheme can show unexpected aggressiveness. Do the update. The user impact is not catastrophic. No leak or crash happens. Doing the quota goal online commit that can reproduce the issue is expected to be not common. This issue was not found by real users but the AI review. That said, the issue can reliably be reproduced. This issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260902002725.108635-4-sj@kernel.org Link: https://lore.kernel.org/20260827045035.94611-1-sj@kernel.org [1] Fixes: 9138e27a3bc3 ("mm/damon: add node_eligible_mem_bp goal metric") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Lian Wang Tested-by: Lian Wang Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: Quanmin Yan Cc: # 7.2.x --- mm/damon/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 71e8cedbc643..cf4ec122a7f9 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1204,6 +1204,9 @@ static void damos_commit_quota_goal_union( dst->nid = src->nid; dst->memcg_id = src->memcg_id; break; + case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP: + dst->nid = src->nid; + break; default: break; } From b9100ecb5411d2de79175ed534df72ec41e94998 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 17:27:23 -0700 Subject: [PATCH 193/539] mm/damon/sysfs: set next refresh jiffies per sysfs context When 'refresh_ms' is set, DAMON sysfs interface periodically updates auto-tuned parameters and DAMOS stats. The timestamp for the next refresh is initialized when a DAMON context starts, and updated in its damon_call() callback function. That is, each DAMON context updates it. However, the timestamp is a global variable that is shared with all the contexts. When there are multiple DAMON contexts having different refresh_ms, the update frequency will be changed, depending on the order of the contexts. When there are multiple kdamonds, it will be even more chaotic. Fix the problem by having the timestamp per each context. The user impact is not very critical. It does not leak, corrupt or crash. The update will not be faster or slower than the lowest and largest refresh_ms values of the contexts, respectively. The user can also manually ask the updates on demand using kdamond state commands. That said, clearly this is a bug and can easily be reproduced. Link: https://lore.kernel.org/20260902002725.108635-5-sj@kernel.org Fixes: 9fd7bb5083d1 ("mm/damon/sysfs: change next_update_jiffies to a global variable") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Lian Wang Tested-by: Lian Wang Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: Quanmin Yan Cc: # 6.18.x --- mm/damon/sysfs.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 7f340b6f1921..7ec14f48d157 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2034,6 +2034,7 @@ struct damon_sysfs_kdamond { struct damon_sysfs_contexts *contexts; struct damon_ctx *damon_ctx; unsigned int refresh_ms; + unsigned long next_refresh_jiffies; }; static struct damon_sysfs_kdamond *damon_sysfs_kdamond_alloc(void) @@ -2484,17 +2485,15 @@ static struct damon_ctx *damon_sysfs_build_ctx( return ctx; } -static unsigned long damon_sysfs_next_update_jiffies; - static int damon_sysfs_repeat_call_fn(void *data) { struct damon_sysfs_kdamond *sysfs_kdamond = data; if (!sysfs_kdamond->refresh_ms) return 0; - if (time_before(jiffies, damon_sysfs_next_update_jiffies)) + if (time_before(jiffies, sysfs_kdamond->next_refresh_jiffies)) return 0; - damon_sysfs_next_update_jiffies = jiffies + + sysfs_kdamond->next_refresh_jiffies = jiffies + msecs_to_jiffies(sysfs_kdamond->refresh_ms); if (!mutex_trylock(&damon_sysfs_lock)) @@ -2542,8 +2541,8 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond) } kdamond->damon_ctx = ctx; - damon_sysfs_next_update_jiffies = - jiffies + msecs_to_jiffies(kdamond->refresh_ms); + kdamond->next_refresh_jiffies = jiffies + + msecs_to_jiffies(kdamond->refresh_ms); repeat_call_control->fn = damon_sysfs_repeat_call_fn; repeat_call_control->data = kdamond; From 798a6ca31cc7b643ca0b5a676ccb3c5e68afce5b Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 2 Sep 2026 07:24:15 +0800 Subject: [PATCH 194/539] mm/mglru: separate folio generation update from LRU accounting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm/mglru: speed up inc_min_seq() and fix cold/hot inversions", v3. This is an aging speedup series split out from the MGLRU swappiness series [1], with the inc_min_seq changes separated to make them easier to review. Currently, inc_min_seq() has two primary issues that affect both performance and folio hotness assessment: 1. It processes each folio one by one, while many operations can be batched or skipped. For example, a batch of folios can be moved together from the oldest generation to the second-oldest generation, and the associated counting can also be done in batches. 2. It may cause potential cold/hot inversion by placing promoted folios (which have been scanned and found to have young PTEs) behind non-promoted folios. A similar inversion can also occur among non-promoted folios, as tail folios from the oldest generation are placed before head folios when moving them to the second-oldest generation. This series tries to batch operations as much as possible and fix the potential cold/hot inversion by keeping promoted folios ahead of non-promoted folios, while also preserving the order of non-promoted folios when moving them from the oldest generation to the second-oldest generation. Minor issue: inc_min_seq() also counts protected folios improperly, as promoted folios should be skipped, as in sort_folio(). We need a stable workload with a stable number of folios to measure aging and evaluate the speedup in inc_min_seq(). So I asked ChatGPT to generate the microbenchmark below. It ages an LRU vec containing 512 MB of memory 100 times: #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #define SIZE (512UL * 1024 * 1024) #define LRU_GEN "/sys/kernel/debug/lru_gen" #define TARGET_CGROUP "/system.slice/agetest.scope" #define START_GEN 3 #define END_GEN 103 static long long nsec_diff(const struct timespec *start, const struct timespec *end) { return (end->tv_sec - start->tv_sec) * 1000000000LL + (end->tv_nsec - start->tv_nsec); } static int find_memcg_id(void) { FILE *fp; char line[4096]; int memcg_id; fp = fopen(LRU_GEN, "r"); if (!fp) { perror("fopen lru_gen"); return -1; } while (fgets(line, sizeof(line), fp)) { char *p; if (strncmp(line, "memcg ", 6)) continue; p = line + 6; if (sscanf(p, "%d", &memcg_id) != 1) continue; /* * The memcg path follows the numeric ID. */ p = strchr(p, ' '); if (!p) continue; if (strstr(p, TARGET_CGROUP)) { fclose(fp); return memcg_id; } } fclose(fp); fprintf(stderr, "Cannot find %s\n", TARGET_CGROUP); return -1; } int main(void) { void *addr; int memcg_id; int fd; long long total_ns = 0; /* * mmap 512 MB and touch every page. */ addr = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) { perror("mmap"); return 1; } memset(addr, 0x55, SIZE); printf("mmap: %p, size: %lu MB\n", addr, SIZE / 1024 / 1024); /* * Find the memcg ID automatically. */ memcg_id = find_memcg_id(); if (memcg_id < 0) return 1; printf("memcg: %d (%s)\n", memcg_id, TARGET_CGROUP); printf("aging generation %d -> %d\n", START_GEN, END_GEN); fd = open(LRU_GEN, O_WRONLY); if (fd < 0) { perror("open lru_gen"); return 1; } for (int gen = START_GEN; gen <= END_GEN; gen++) { char buf[128]; int len; struct timespec start, end; long long ns; len = snprintf(buf, sizeof(buf), "+ %d 0 %d\n", memcg_id, gen); clock_gettime(CLOCK_MONOTONIC, &start); if (write(fd, buf, len) != len) { perror("write lru_gen"); close(fd); return 1; } clock_gettime(CLOCK_MONOTONIC, &end); ns = nsec_diff(&start, &end); total_ns += ns; printf("gen %3d: %8.3f ms\n", gen, ns / 1000000.0); fflush(stdout); } close(fd); printf("\nTotal: %.3f ms\n", total_ns / 1000000.0); printf("Average: %.3f ms\n", total_ns / (double)(END_GEN - START_GEN + 1) / 1000000.0); while (1) sleep(1); return 0; } Run the above microbenchmark with: systemd-run --scope --unit=agetest -p MemoryMax=1024M ./agetest I’m seeing a significant speedup in inc_min_seq() on my x86 PC: W/o patch: Running scope as unit: agetest.scope mmap: 0x72c1b5a00000, size: 512 MB memcg: 12673 (/system.slice/agetest.scope) aging generation 3 -> 103 gen 3: 7.433 ms gen 4: 0.949 ms gen 5: 2.535 ms gen 6: 5.043 ms gen 7: 5.041 ms gen 8: 5.027 ms ... gen 100: 5.035 ms gen 101: 5.011 ms gen 102: 5.029 ms gen 103: 5.056 ms Total: 503.946 ms Average: 4.990 ms W/ patch: Running scope as unit: agetest.scope mmap: 0x775ec5200000, size: 512 MB memcg: 12717 (/system.slice/agetest.scope) aging generation 3 -> 103 gen 3: 7.558 ms gen 4: 0.916 ms gen 5: 2.277 ms gen 6: 2.328 ms … gen 100: 2.303 ms gen 101: 2.297 ms gen 102: 2.305 ms gen 103: 2.312 ms Total: 236.635 ms Average: 2.343 ms The average aging time drops from 4.990 ms to 2.343 ms! Thanks, Xueyuan, for testing this on ARM[2]. It actually shows an even larger improvement. Xueyuan tested this series on his arm64 machine (24 cores, 4K base pages) and reproduced the improvement: THP=never (PTE): baseline: 7.644 ms patched: 2.964 ms (-61.2%) THP=always (PMD): baseline: 0.0373 ms patched: 0.0292 ms (-21.8%) This patch (of 7): folio_inc_gen() currently updates both the folio's generation and the LRU size accounting. This makes it difficult to batch the LRU size updates when moving multiple folios. Extract the generation update into __folio_inc_gen(), which only updates the folio's generation and reports whether the generation was actually increased. Keep folio_inc_gen() as the wrapper that performs the LRU size accounting when needed. This separates the per-folio generation update from LRU accounting and allows the latter to be batched by subsequent changes. Link: https://lore.kernel.org/20260901232421.40157-1-baohua@kernel.org Link: https://lore.kernel.org/20260901232421.40157-2-baohua@kernel.org Link: https://lore.kernel.org/linux-mm/20260812121658.69965-1-baohua@kernel.org/ [1] Link: https://lore.kernel.org/linux-mm/20260827035416.3012015-1-xueyuan.chen21@gmail.com/ [2] Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Reviewed-by: Baolin Wang Cc: Axel Rasmussen Cc: Baoquan He Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Cc: Kunwu Chan Cc: Ridong Chen --- mm/vmscan.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index ce3bab78af3c..d1a051e7db1e 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3300,21 +3300,21 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; } -/* protect pages accessed multiple times through file descriptors */ -static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) +static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) { - int type = folio_is_file_lru(folio); - struct lru_gen_folio *lrugen = &lruvec->lrugen; - int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); + int new_gen; VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); do { new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; /* folio_update_gen() has promoted this page? */ - if (new_gen >= 0 && new_gen != old_gen) + if (new_gen >= 0 && new_gen != old_gen) { + if (increased) + *increased = false; return new_gen; + } new_gen = (old_gen + 1) % MAX_NR_GENS; @@ -3322,8 +3322,22 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); - lru_gen_update_size(lruvec, folio, old_gen, new_gen); + if (increased) + *increased = true; + return new_gen; +} +/* protect pages accessed multiple times through file descriptors */ +static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) +{ + int type = folio_is_file_lru(folio); + struct lru_gen_folio *lrugen = &lruvec->lrugen; + int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + bool gen_increased; + + new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); + if (gen_increased) + lru_gen_update_size(lruvec, folio, old_gen, new_gen); return new_gen; } From 85dd75276d50bcaee8e4f0eab68561681ae3d46a Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 2 Sep 2026 07:24:16 +0800 Subject: [PATCH 195/539] mm/mglru: batch update lrugen->nr_pages in inc_min_seq() Currently, folio_inc_gen() updates lrugen->nr_pages for every folio as it advances generations. Instead, accumulate the size changes and update lrugen->nr_pages in a batch after scanning the entire oldest generation, or when the scan stops because remaining reaches zero. Since we only move folios from the oldest generation to the second oldest generation, the active/inactive state cannot change. We can therefore skip __lru_update_size(). Link: https://lore.kernel.org/20260901232421.40157-3-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Reviewed-by: Kunwu Chan Reviewed-by: Baolin Wang Cc: Axel Rasmussen Cc: Baoquan He Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Ridong Chen Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index d1a051e7db1e..801345ca4da3 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3923,6 +3923,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) struct lru_gen_folio *lrugen = &lruvec->lrugen; int hist = lru_hist_from_seq(lrugen->min_seq[type]); int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + int target_gen = (old_gen + 1) % MAX_NR_GENS; /* For file type, skip the check if swappiness is anon only */ if (type && (swappiness == SWAPPINESS_ANON_ONLY)) @@ -3932,35 +3933,47 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) if (!type && !swappiness) goto done; + VM_WARN_ON_ONCE(get_nr_gens(lruvec, type) != MAX_NR_GENS); + VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) != + lru_gen_is_active(lruvec, target_gen)); /* prevent cold/hot inversion if the type is evictable */ for (zone = 0; zone < MAX_NR_ZONES; zone++) { struct list_head *head = &lrugen->folios[old_gen][type][zone]; + long delta = 0; while (!list_empty(head)) { struct folio *folio = lru_to_folio(head); + long nr_pages = folio_nr_pages(folio); int refs = folio_lru_refs(folio); bool workingset = folio_test_workingset(folio); + bool gen_increased; VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); - new_gen = folio_inc_gen(lruvec, folio); + new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); - + if (gen_increased) + delta += nr_pages; /* don't count the workingset being lazily promoted */ if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { int tier = lru_tier_from_refs(refs, workingset); - int delta = folio_nr_pages(folio); WRITE_ONCE(lrugen->protected[hist][type][tier], - lrugen->protected[hist][type][tier] + delta); + lrugen->protected[hist][type][tier] + nr_pages); } if (!--remaining) - return false; + break; } + WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], + lrugen->nr_pages[old_gen][type][zone] - delta); + WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone], + lrugen->nr_pages[target_gen][type][zone] + delta); + if (!remaining) + return false; } done: reset_ctrl_pos(lruvec, type, true); From e90113a8e0e8383e8d71ceb4c8926e8aa629f547 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 2 Sep 2026 07:24:17 +0800 Subject: [PATCH 196/539] mm/mglru: enhance cold/hot inversion handling in inc_min_seq() During aging, a folio's generation may already have been updated by folio_update_gen(), even though it has not yet been moved to the corresponding generation list. Such folios are hotter than those already in that generation. It makes sense for inc_min_seq() to increment the generation of folios that were never promoted during aging and move them to the tail of the new oldest generation. However, folios that were already promoted should instead be moved to the head of their updated generation, just as sort_folio() does in scan_folios(). Otherwise, promoted folios could end up behind folios that were never promoted, effectively inverting their hot/cold ordering. Link: https://lore.kernel.org/20260901232421.40157-4-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Reviewed-by: Kairui Song Tested-by: Xueyuan Chen Reviewed-by: Ridong Chen Reviewed-by: Lian Wang Reviewed-by: Baolin Wang Cc: Axel Rasmussen Cc: Baoquan He Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kunwu Chan Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 801345ca4da3..a608483ff9c7 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3954,9 +3954,17 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); - list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); - if (gen_increased) + /* + * If gen_increased is false, this is a promotion. Put folios + * at the head of the promoted gen. Otherwise, put them at + * the tail of the second-oldest gen. + */ + if (gen_increased) { delta += nr_pages; + list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); + } else { + list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]); + } /* don't count the workingset being lazily promoted */ if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { int tier = lru_tier_from_refs(refs, workingset); From 8c1731cfd72b831e0bcaee52d1b7660f28e3602b Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 2 Sep 2026 07:24:18 +0800 Subject: [PATCH 197/539] mm/mglru: exclude folios promoted by aging from protected in inc_min_seq() Some folios may have been promoted during aging, so don't count them as protected, similar to sort_folio(). Link: https://lore.kernel.org/20260901232421.40157-5-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Tested-by: Xueyuan Chen Reviewed-by: Ridong Chen Reviewed-by: Lian Wang Cc: Axel Rasmussen Cc: Baolin Wang Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Kunwu Chan Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index a608483ff9c7..79311f62d3e6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3962,17 +3962,17 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) if (gen_increased) { delta += nr_pages; list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); + + /* don't count the workingset being lazily promoted */ + if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { + int tier = lru_tier_from_refs(refs, workingset); + + WRITE_ONCE(lrugen->protected[hist][type][tier], + lrugen->protected[hist][type][tier] + nr_pages); + } } else { list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]); } - /* don't count the workingset being lazily promoted */ - if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { - int tier = lru_tier_from_refs(refs, workingset); - - WRITE_ONCE(lrugen->protected[hist][type][tier], - lrugen->protected[hist][type][tier] + nr_pages); - } - if (!--remaining) break; } From 6897e879c32c94dec5946fb66ab78c8a0298f87c Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 2 Sep 2026 07:24:19 +0800 Subject: [PATCH 198/539] mm/mglru: make LRU folio prefetch helper an inline function `prefetchw_prev_lru_folio()` is currently implemented as a macro with a potentially unused argument. This makes the helper harder to read and can also trigger checkpatch warnings about unused macro arguments. Make it a `static inline` function and remove the unnecessary `_field` argument. The helper always prefetches the previous folio's `flags`, so there is no need to make the field configurable. Link: https://lore.kernel.org/20260901232421.40157-6-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Reviewed-by: Baolin Wang Cc: Axel Rasmussen Cc: Baoquan He Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Kunwu Chan Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Ridong Chen Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 79311f62d3e6..4b0ed3c9aab2 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -182,17 +182,21 @@ struct scan_control { }; #ifdef ARCH_HAS_PREFETCHW -#define prefetchw_prev_lru_folio(_folio, _base, _field) \ - do { \ - if ((_folio)->lru.prev != _base) { \ - struct folio *prev; \ - \ - prev = lru_to_folio(&(_folio->lru)); \ - prefetchw(&prev->_field); \ - } \ - } while (0) +static inline void prefetchw_prev_lru_folio(struct folio *folio, + struct list_head *base) +{ + if (folio->lru.prev != base) { + struct folio *prev; + + prev = lru_to_folio(&folio->lru); + prefetchw(&prev->flags); + } +} #else -#define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) +static inline void prefetchw_prev_lru_folio(struct folio *folio, + struct list_head *base) +{ +} #endif /* @@ -1700,7 +1704,7 @@ static unsigned long isolate_lru_folios(unsigned long nr_to_scan, struct folio *folio; folio = lru_to_folio(src); - prefetchw_prev_lru_folio(folio, src, flags); + prefetchw_prev_lru_folio(folio, src); nr_pages = folio_nr_pages(folio); total_scan += nr_pages; From 5d3664f070f04262a9be95ff295541993a2f0c39 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 2 Sep 2026 07:24:20 +0800 Subject: [PATCH 199/539] mm/mglru: move folios from oldest gen to second-oldest gen from head to tail For reclamation, it makes sense to reclaim folios from tail to head, as folios near the head are relatively hot. However, when moving folios from the oldest generation to the second-oldest generation, using the tail-to-head order would effectively cause a cold/hot inversion. The impact of the added prefetching might be arch-dependent. Some architectures could benefit more from prefetching, while others might see little to no impact. In my x86 test, it shows a very slight improvement: ***** no-prefetch: agetest: ... gen 100: 2.421 ms gen 101: 2.424 ms gen 102: 2.420 ms gen 103: 2.413 ms Total: 248.418 ms Average: 2.460 ms agetest: ... gen 100: 2.393 ms gen 101: 2.396 ms gen 102: 2.395 ms gen 103: 2.392 ms Total: 245.627 ms Average: 2.432 ms agetest: ... gen 100: 2.433 ms gen 101: 2.427 ms gen 102: 2.432 ms gen 103: 2.450 ms Total: 249.186 ms Average: 2.467 ms **** has-prefetch: agetest: .... gen 100: 2.314 ms gen 101: 2.310 ms gen 102: 2.321 ms gen 103: 2.303 ms Total: 237.619 ms Average: 2.353 ms agetest: gen 100: 2.342 ms gen 101: 2.343 ms gen 102: 2.339 ms gen 103: 2.335 ms Total: 239.929 ms Average: 2.376 ms agetest: gen 100: 2.344 ms gen 101: 2.347 ms gen 102: 2.352 ms gen 103: 2.348 ms Total: 241.188 ms Average: 2.388 ms Basically, its 2.3xx vs. 2.4xx, lower is better. Link: https://lore.kernel.org/20260901232421.40157-7-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Cc: Axel Rasmussen Cc: Baolin Wang Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Kunwu Chan Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Ridong Chen Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 4b0ed3c9aab2..49fab93470ad 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -192,11 +192,27 @@ static inline void prefetchw_prev_lru_folio(struct folio *folio, prefetchw(&prev->flags); } } + +static inline void prefetchw_next_lru_folio(struct folio *folio, + struct list_head *base) +{ + if (folio->lru.next != base) { + struct folio *next; + + next = list_entry(folio->lru.next, struct folio, lru); + prefetchw(&next->flags); + } +} #else static inline void prefetchw_prev_lru_folio(struct folio *folio, struct list_head *base) { } + +static inline void prefetchw_next_lru_folio(struct folio *folio, + struct list_head *base) +{ +} #endif /* @@ -3943,10 +3959,11 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) /* prevent cold/hot inversion if the type is evictable */ for (zone = 0; zone < MAX_NR_ZONES; zone++) { struct list_head *head = &lrugen->folios[old_gen][type][zone]; + struct list_head *pos = head->next; long delta = 0; - while (!list_empty(head)) { - struct folio *folio = lru_to_folio(head); + while (pos != head) { + struct folio *folio = list_entry(pos, struct folio, lru); long nr_pages = folio_nr_pages(folio); int refs = folio_lru_refs(folio); bool workingset = folio_test_workingset(folio); @@ -3957,6 +3974,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + prefetchw_next_lru_folio(folio, head); + pos = pos->next; new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); /* * If gen_increased is false, this is a promotion. Put folios From 3d4bbd98b369c54e36c92873cb029d8826b3ceaa Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 2 Sep 2026 07:24:21 +0800 Subject: [PATCH 200/539] mm/mglru: batch move folios to the second-oldest gen's LRU Detect folios that need to move from the oldest generation to the second-oldest generation, and batch-move them together. This can significantly reduce the sys time of inc_min_seq(), especially when the other type is significantly behind the preferred type. Link: https://lore.kernel.org/20260901232421.40157-8-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Reviewed-by: Baolin Wang Assisted-by: gemini:gemini-3.6-flash Cc: Axel Rasmussen Cc: David Hildenbrand Cc: David Stevens Cc: Johannes Weiner Cc: Kairui Song Cc: Kunwu Chan Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Ridong Chen Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 49fab93470ad..ba7adf36e69f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3936,6 +3936,19 @@ static void clear_mm_walk(void) kfree(walk); } +static inline void flush_lru_batch(struct list_head *head, struct list_head **batch_end, + struct list_head *dst) +{ + LIST_HEAD(movable); + + if (!*batch_end) + return; + + list_cut_position(&movable, head, *batch_end); + list_splice_tail_init(&movable, dst); + *batch_end = NULL; +} + static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) { int zone; @@ -3958,8 +3971,10 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) lru_gen_is_active(lruvec, target_gen)); /* prevent cold/hot inversion if the type is evictable */ for (zone = 0; zone < MAX_NR_ZONES; zone++) { + struct list_head *target_list = &lrugen->folios[target_gen][type][zone]; struct list_head *head = &lrugen->folios[old_gen][type][zone]; struct list_head *pos = head->next; + struct list_head *batch_end = NULL; long delta = 0; while (pos != head) { @@ -3984,7 +3999,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) */ if (gen_increased) { delta += nr_pages; - list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); + batch_end = &folio->lru; /* don't count the workingset being lazily promoted */ if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { @@ -3994,11 +4009,14 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) lrugen->protected[hist][type][tier] + nr_pages); } } else { + flush_lru_batch(head, &batch_end, target_list); list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]); } if (!--remaining) break; } + flush_lru_batch(head, &batch_end, target_list); + WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], lrugen->nr_pages[old_gen][type][zone] - delta); WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone], From b947312b5a1bbfc0f2012757b5f70c7fe3f75e01 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 07:03:07 -0700 Subject: [PATCH 201/539] mm/damon/tests/core-kunit: test damon_commit_filter() Patch series "mm/damon: add kunit and selftests for probes and probe weights". DAMON recently introduced probes and probe weights. Add kunit and selftests for ensuring the parameters for features can be set using the core API and the sysfs ABI, respectively. This patch (of 6): Add kunit test to ensure damon_commit_filter() updates destination filter as expected for valid inputs. Link: https://lore.kernel.org/20260902140313.85983-1-sj@kernel.org Link: https://lore.kernel.org/20260902140313.85983-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Shuah Khan --- mm/damon/tests/core-kunit.h | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index af26b3d60957..3fc5b631b45c 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1316,6 +1316,45 @@ static void damon_test_commit_target_regions(struct kunit *test) (unsigned long[][2]) {{3, 8}, {8, 10}}, 2); } +static void damon_test_commit_filter_for(struct kunit *test, + struct damon_filter *dst, struct damon_filter *src) +{ + damon_commit_filter(dst, src); + KUNIT_EXPECT_EQ(test, dst->type, src->type); + KUNIT_EXPECT_EQ(test, dst->matching, src->matching); + KUNIT_EXPECT_EQ(test, dst->allow, src->allow); + switch (src->type) { + case DAMON_FILTER_TYPE_MEMCG: + KUNIT_EXPECT_EQ(test, dst->memcg_id, src->memcg_id); + break; + default: + break; + } +} + +static void damon_test_commit_filter(struct kunit *test) +{ + struct damon_filter dst = { + .type = DAMON_FILTER_TYPE_ANON, + .matching = false, + .allow = false, + }; + + damon_test_commit_filter_for(test, &dst, + &(struct damon_filter){ + .type = DAMON_FILTER_TYPE_ANON, + .matching = true, + .allow = true, + }); + damon_test_commit_filter_for(test, &dst, + &(struct damon_filter){ + .type = DAMON_FILTER_TYPE_MEMCG, + .matching = false, + .allow = false, + .memcg_id = 123, + }); +} + static void damon_test_commit_ctx(struct kunit *test) { struct damon_ctx *src, *dst; @@ -1722,6 +1761,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damos_test_commit_pageout), KUNIT_CASE(damos_test_commit_migrate_hot), KUNIT_CASE(damon_test_commit_target_regions), + KUNIT_CASE(damon_test_commit_filter), KUNIT_CASE(damon_test_commit_ctx), KUNIT_CASE(damon_test_valid_probe_params), KUNIT_CASE(damos_test_filter_out), From ef72a0c6872a984a8992d97f59e0399193aca007 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 07:03:08 -0700 Subject: [PATCH 202/539] mm/damon/tests/core-kunit: add damon_commit_probes() test Add kunit test to ensure damon_commit_probes() updates destination DAMON context with source probes as expected. Link: https://lore.kernel.org/20260902140313.85983-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Shuah Khan --- mm/damon/tests/core-kunit.h | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 3fc5b631b45c..f2568fba552e 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1355,6 +1355,89 @@ static void damon_test_commit_filter(struct kunit *test) }); } +static struct damon_ctx *damon_test_help_setup_probes(unsigned int weights[], + int nr_weights) +{ + struct damon_ctx *ctx; + struct damon_probe *probe; + int i; + + ctx = damon_new_ctx(); + if (!ctx) + return NULL; + for (i = 0; i < nr_weights; i++) { + probe = damon_new_probe(); + if (!probe) { + damon_destroy_ctx(ctx); + return NULL; + } + probe->weight = weights[i]; + damon_add_probe(ctx, probe); + } + return ctx; +} + +static void damon_test_commit_probes_for(struct kunit *test, + unsigned int dst_weights[], int nr_dst_probes, + unsigned int src_weights[], int nr_src_probes) +{ + struct damon_ctx *dst, *src; + int err; + struct damon_probe *dst_probe, *src_probe; + + dst = damon_test_help_setup_probes(dst_weights, nr_dst_probes); + if (!dst) + kunit_skip(test, "dst alloc fail"); + src = damon_test_help_setup_probes(src_weights, nr_src_probes); + if (!src) { + damon_destroy_ctx(dst); + kunit_skip(test, "src alloc fail"); + } + + err = damon_commit_probes(dst, src); + KUNIT_EXPECT_EQ(test, err, 0); + if (err) + goto out; + nr_dst_probes = 0; + damon_for_each_probe(dst_probe, dst) + nr_dst_probes++; + nr_src_probes = 0; + damon_for_each_probe(src_probe, src) + nr_src_probes++; + KUNIT_EXPECT_EQ(test, nr_dst_probes, nr_src_probes); + if (nr_dst_probes != nr_src_probes) + goto out; + nr_dst_probes = 0; + damon_for_each_probe(dst_probe, dst) { + src_probe = damon_nth_probe(nr_dst_probes, src); + KUNIT_EXPECT_EQ(test, src_probe->weight, dst_probe->weight); + nr_dst_probes++; + } +out: + damon_destroy_ctx(dst); + damon_destroy_ctx(src); +} + +static void damon_test_commit_probes(struct kunit *test) +{ + damon_test_commit_probes_for(test, + (unsigned int[]){}, 0, (unsigned int[]){}, 0); + damon_test_commit_probes_for(test, + (unsigned int[]){}, 0, (unsigned int[]){1}, 1); + damon_test_commit_probes_for(test, + (unsigned int[]){}, 0, (unsigned int[]){1, 2}, 2); + damon_test_commit_probes_for(test, + (unsigned int[]){1}, 1, (unsigned int[]){2}, 1); + damon_test_commit_probes_for(test, + (unsigned int[]){1}, 1, (unsigned int[]){2, 3}, 2); + damon_test_commit_probes_for(test, + (unsigned int[]){2, 3}, 2, (unsigned int[]){1}, 1); + damon_test_commit_probes_for(test, + (unsigned int[]){2, 3}, 2, (unsigned int[]){}, 0); + damon_test_commit_probes_for(test, + (unsigned int[]){2}, 1, (unsigned int[]){}, 0); +} + static void damon_test_commit_ctx(struct kunit *test) { struct damon_ctx *src, *dst; @@ -1762,6 +1845,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damos_test_commit_migrate_hot), KUNIT_CASE(damon_test_commit_target_regions), KUNIT_CASE(damon_test_commit_filter), + KUNIT_CASE(damon_test_commit_probes), KUNIT_CASE(damon_test_commit_ctx), KUNIT_CASE(damon_test_valid_probe_params), KUNIT_CASE(damos_test_filter_out), From d5756953dd5d950940459f963cb7141c4b8d6e64 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 07:03:09 -0700 Subject: [PATCH 203/539] selftests/damon/_damon_sysfs: implement DamonProbes Extend _damon_sysfs.py to support staging and committing DAMON probes. It will be used for setting DAMON probes via sysfs changes for testing purposes. Link: https://lore.kernel.org/20260902140313.85983-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Shuah Khan --- tools/testing/selftests/damon/_damon_sysfs.py | 125 +++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index f604b7d6530b..e7095c336524 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -570,6 +570,117 @@ class IntervalsGoal: return err return None +class DamonFilter: + type_ = None + matching = None + allow = None + filters = None + path = None + idx = None + + def __init__(self, type_='anon', matching=False, allow=False, path=None): + self.type_ = type_ + self.matching = matching + self.allow = allow + self.path = path + + def sysfs_dir(self): + return os.path.join(self.filters.sysfs_dir(), '%d' % self.idx) + + def stage(self): + err = write_file(os.path.join(self.sysfs_dir(), 'type'), self.type_) + if err is not None: + return err + err = write_file(os.path.join(self.sysfs_dir(), 'matching'), + 'Y' if self.matching else 'N') + if err is not None: + return err + err = write_file(os.path.join(self.sysfs_dir(), 'allow'), + 'Y' if self.allow else 'N') + if err is not None: + return err + if self.type_ == 'memcg': + err = write_file(os.path.join(self.sysfs_dir(), 'path'), self.path) + if err is not None: + return err + return None + +class DamonFilters: + filters = None + probe = None + + def __init__(self, filters=None): + if filters is None: + filters = [] + self.filters = filters + for idx, filter in enumerate(self.filters): + filter.filters = self + filter.idx = idx + + def sysfs_dir(self): + return os.path.join(self.probe.sysfs_dir(), 'filters') + + def stage(self): + err = write_file( + os.path.join(self.sysfs_dir(), 'nr_filters'), + len(self.filters)) + if err is not None: + return err + for filter in self.filters: + err = filter.stage() + if err is not None: + return err + return None + +class DamonProbe: + weight = None + filters = None + probes = None + idx = None + + def __init__(self, weight=0, filters=None): + self.weight = weight + if filters is None: + filters = DamonFilters() + self.filters = filters + self.filters.probe = self + + def sysfs_dir(self): + return os.path.join(self.probes.sysfs_dir(), '%d' % self.idx) + + def stage(self): + err = write_file( + os.path.join(self.sysfs_dir(), 'weight'), '%d' % self.weight) + if err is not None: + return err + return self.filters.stage() + +class DamonProbes: + probes = None + attrs = None + + def __init__(self, probes=None): + if probes is None: + probes = [] + self.probes = probes + for idx, probe in enumerate(self.probes): + probe.probes = self + probe.idx = idx + + def sysfs_dir(self): + return os.path.join(self.attrs.sysfs_dir(), 'probes') + + def stage(self): + err = write_file(os.path.join(self.sysfs_dir(), 'nr_probes'), + len(self.probes)) + if err is not None: + return err + for probe in self.probes: + err = probe.stage() + if err is not None: + return err + return None + class DamonAttrs: sample_us = None aggr_us = None @@ -577,11 +688,12 @@ class DamonAttrs: update_us = None min_nr_regions = None max_nr_regions = None + probes = None context = None def __init__(self, sample_us=5000, aggr_us=100000, intervals_goal=None, update_us=1000000, min_nr_regions=10, - max_nr_regions=1000): + max_nr_regions=1000, probes=None): self.sample_us = sample_us self.aggr_us = aggr_us if intervals_goal is None: @@ -591,6 +703,10 @@ class DamonAttrs: self.update_us = update_us self.min_nr_regions = min_nr_regions self.max_nr_regions = max_nr_regions + if probes is None: + probes = DamonProbes() + self.probes = probes + self.probes.attrs = self def interval_sysfs_dir(self): return os.path.join(self.context.sysfs_dir(), 'monitoring_attrs', @@ -600,6 +716,9 @@ class DamonAttrs: return os.path.join(self.context.sysfs_dir(), 'monitoring_attrs', 'nr_regions') + def sysfs_dir(self): + return os.path.join(self.context.sysfs_dir(), 'monitoring_attrs') + def stage(self): err = write_file(os.path.join(self.interval_sysfs_dir(), 'sample_us'), self.sample_us) @@ -629,6 +748,10 @@ class DamonAttrs: if err is not None: return err + err = self.probes.stage() + if err is not None: + return err + class DamonCtx: ops = None monitoring_attrs = None From adc9c5303f22c160841633a20e74c90a9e2acea6 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 07:03:10 -0700 Subject: [PATCH 204/539] selftests/damon/drgn_dump_damon_status: dump probes Extend drgn_dump_damon_status.py to dump damon_ctx->probes. It will be used to see if in-kernel DAMON status are changed as the user sets the probes via sysfs. Link: https://lore.kernel.org/20260902140313.85983-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Shuah Khan --- .../selftests/damon/drgn_dump_damon_status.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tools/testing/selftests/damon/drgn_dump_damon_status.py b/tools/testing/selftests/damon/drgn_dump_damon_status.py index 09552e91bc78..4622046fd011 100755 --- a/tools/testing/selftests/damon/drgn_dump_damon_status.py +++ b/tools/testing/selftests/damon/drgn_dump_damon_status.py @@ -48,6 +48,37 @@ def attrs_to_dict(attrs): ['max_nr_regions', int], ]) +def filter_to_dict(damon_filter): + filter_type_keyword = { + 0: 'anon', + 1: 'memcg', + } + dict_ = { + 'type': filter_type_keyword[int(damon_filter.type)], + 'matching': bool(damon_filter.matching), + 'allow': bool(damon_filter.allow), + } + type_ = dict_['type'] + if type_ == 'memcg': + dict_['memcg_id'] = int(damon_filter.memcg_id) + return dict_ + +def filters_to_list(filters): + return [filter_to_dict(f) + for f in list_for_each_entry( + 'struct damon_filter', filters.address_of_(), 'list')] + +def probe_to_dict(probe): + return to_dict(probe, [ + ['weight', int], + ['filters', filters_to_list], + ]) + +def probes_to_list(probes): + return [probe_to_dict(p) + for p in list_for_each_entry( + 'struct damon_probe', probes.address_of_(), 'list')] + def addr_range_to_dict(addr_range): return to_dict(addr_range, [ ['start', int], @@ -199,6 +230,7 @@ def damon_ctx_to_dict(ctx): return to_dict(ctx, [ ['ops', ops_to_dict], ['attrs', attrs_to_dict], + ['probes', probes_to_list], ['adaptive_targets', targets_to_list], ['schemes', schemes_to_list], ['pause', bool], From 3126a9d0745cac5b6f4df9760006617f4731245f Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 07:03:11 -0700 Subject: [PATCH 205/539] selftests/damon/sysfs.py: extend commit assertion function for probes Extend DAMON sysfs testing commit assertion helper function to check probes too. Link: https://lore.kernel.org/20260902140313.85983-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Shuah Khan --- tools/testing/selftests/damon/sysfs.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index 88a26422ff44..159cbeac067f 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -178,6 +178,24 @@ def assert_monitoring_attrs_committed(attrs, dump): assert_true(dump['max_nr_regions'] == attrs.max_nr_regions, 'max_nr_regions', dump) +def assert_damon_filters_committed(filters, dump): + assert_true(len(dump) == len(filters.filters), 'probe filters', dump) + for idx, damon_filter in enumerate(filters.filters): + filter_dump = dump[idx] + assert_true(filter_dump['type'] == damon_filter.type_, 'type', + filter_dump) + assert_true(filter_dump['matching'] == damon_filter.matching, + 'matching', filter_dump) + assert_true(filter_dump['allow'] == damon_filter.allow, 'allow', + filter_dump) + +def assert_probes_committed(probes, dump): + assert_true(len(dump) == len(probes.probes), 'probes length', dump) + for idx, probe in enumerate(probes.probes): + probe_dump = dump[idx] + assert_true(probe.weight == probe_dump['weight'], 'weight', probe_dump) + assert_damon_filters_committed(probe.filters, probe_dump['filters']) + def assert_monitoring_target_committed(target, dump): # target.pid is the pid "number", while dump['pid'] is 'struct pid' # pointer, and hence cannot be compared. @@ -196,6 +214,7 @@ def assert_ctx_committed(ctx, dump): } assert_true(dump['ops']['id'] == ops_val[ctx.ops], 'ops_id', dump) assert_monitoring_attrs_committed(ctx.monitoring_attrs, dump['attrs']) + assert_probes_committed(ctx.monitoring_attrs.probes, dump['probes']) assert_monitoring_targets_committed(ctx.targets, dump['adaptive_targets']) assert_schemes_committed(ctx.schemes, dump['schemes']) assert_true(dump['pause'] == ctx.pause, 'pause', dump) From edcc6ad6341bf7d4bb3141f0ef545c34417c4b3c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 07:03:12 -0700 Subject: [PATCH 206/539] selftests/damon/sysfs.py: test damon probes Extend sysfs.py to commit DAMON probes via sysfs, and see if it changed in-kernel DAMON status as expected using drgn. Link: https://lore.kernel.org/20260902140313.85983-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Shuah Khan --- tools/testing/selftests/damon/sysfs.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index 159cbeac067f..66c826189320 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -319,7 +319,21 @@ def main(): intervals_goal=_damon_sysfs.IntervalsGoal( access_bp=400, aggrs=3, min_sample_us=5000, max_sample_us=10000000), - update_us=2000000), + update_us=2000000, + probes=_damon_sysfs.DamonProbes( + probes=[_damon_sysfs.DamonProbe( + weight=42, + filters=_damon_sysfs.DamonFilters( + filters=[ + _damon_sysfs.DamonFilter( + type_='anon', + matching=True, + allow=True, + ), + ]), + ), + ]), + ), schemes=[_damon_sysfs.Damos( action='pageout', access_pattern=_damon_sysfs.DamosAccessPattern( From 2e6a6ca06fec956ac716b124a0fe5651bcaeb255 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 2 Sep 2026 21:16:50 +0800 Subject: [PATCH 207/539] xfs: remove dead kswapd flag inheritance from btree split worker Patch series "mm: replace PF_KCOMPACTD/PF_KSWAPD with kthread_func()". The task_struct->flags field is a limited 32-bit resource. Two flag bits, PF_KCOMPACTD and PF_KSWAPD, are used to identify whether a kernel thread is kcompactd or kswapd. Converting these to use kthread_func() frees up two valuable flag bits for future use. This patch (of 4): Commit 1f6d64829db7 ("xfs: block allocation work needs to be kswapd aware") added PF_MEMALLOC | PF_KSWAPD inheritance to xfs_btree_split_worker() so that block allocation offloaded from kswapd to a workqueue thread could access emergency memory reserves and avoid reclaim throttling. pageout() no longer calls ->writepage() for filesystem folios -- it returns PAGE_ACTIVATE for non-shmem, non-anon pages. kswapd therefore never enters XFS writeback and cannot reach btree split. The only path that offloads btree splits is unwritten extent conversion at IO completion (xfs_end_io -> xfs_iomap_write_unwritten), which runs in a workqueue context where current_is_kswapd() is always false. Let's remove the dead kswapd flag and related codes. Link: https://lore.kernel.org/20260902131653.1338227-1-wangkefeng.wang@huawei.com Link: https://lore.kernel.org/20260902131653.1338227-2-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Signed-off-by: Andrew Morton Reviewed-by: Christoph Hellwig Reviewed-by: Shakeel Butt Cc: Brendan Jackman Cc: Carlos Maiolino Cc: Christian Brauner Cc: "Darrick J. Wong" Cc: David Hildenbrand Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan --- fs/xfs/libxfs/xfs_btree.c | 18 +----------------- fs/xfs/xfs_platform.h | 4 ---- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index 60ef7f08b1d3..6738d9d1511b 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -2994,7 +2994,6 @@ struct xfs_btree_split_args { struct xfs_btree_cur **curp; int *stat; /* success/failure */ int result; - bool kswapd; /* allocation in kswapd context */ struct completion *done; struct work_struct work; }; @@ -3008,33 +3007,18 @@ xfs_btree_split_worker( { struct xfs_btree_split_args *args = container_of(work, struct xfs_btree_split_args, work); - unsigned long pflags; - unsigned long new_pflags = 0; - - /* - * we are in a transaction context here, but may also be doing work - * in kswapd context, and hence we may need to inherit that state - * temporarily to ensure that we don't block waiting for memory reclaim - * in any way. - */ - if (args->kswapd) - new_pflags |= PF_MEMALLOC | PF_KSWAPD; - - current_set_flags_nested(&pflags, new_pflags); xfs_trans_set_context(args->cur->bc_tp); args->result = __xfs_btree_split(args->cur, args->level, args->ptrp, args->key, args->curp, args->stat); xfs_trans_clear_context(args->cur->bc_tp); - current_restore_flags_nested(&pflags, new_pflags); /* * Do not access args after complete() has run here. We don't own args * and the owner may run and free args before we return here. */ complete(args->done); - } /* @@ -3078,7 +3062,7 @@ xfs_btree_split( args.curp = curp; args.stat = stat; args.done = &done; - args.kswapd = current_is_kswapd(); + INIT_WORK_ONSTACK(&args.work, xfs_btree_split_worker); queue_work(xfs_alloc_wq, &args.work); wait_for_completion(&done); diff --git a/fs/xfs/xfs_platform.h b/fs/xfs/xfs_platform.h index 745d715b4c64..e961e36c53fc 100644 --- a/fs/xfs/xfs_platform.h +++ b/fs/xfs/xfs_platform.h @@ -115,10 +115,6 @@ typedef __u32 xfs_nlink_t; #define xfs_blockgc_secs xfs_params.blockgc_timer.val #define current_cpu() (raw_smp_processor_id()) -#define current_set_flags_nested(sp, f) \ - (*(sp) = current->flags, current->flags |= (f)) -#define current_restore_flags_nested(sp, f) \ - (current->flags = ((current->flags & ~(f)) | (*(sp) & (f)))) #define NBBY 8 /* number of bits per byte */ From ff59f71f1c95e24380a13e7b4a59eba612634172 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 2 Sep 2026 21:16:51 +0800 Subject: [PATCH 208/539] iomap: simplify writepages reclaim guard Now that kswapd can no longer reach filesystem writeback (pageout() returns PAGE_ACTIVATE for non-shmem, non-anon folios), any PF_MEMALLOC context in iomap_writepages() indicates a VM regression, simplify the check by removing PF_KSWAPD. Link: https://lore.kernel.org/20260902131653.1338227-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Signed-off-by: Andrew Morton Reviewed-by: Shakeel Butt Cc: Brendan Jackman Cc: Carlos Maiolino Cc: Christian Brauner Cc: Christoph Hellwig Cc: "Darrick J. Wong" Cc: David Hildenbrand Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Vlastimil Babka (SUSE) Cc: Zi Yan --- fs/iomap/buffered-io.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 0a5ebfda90f1..6306ca747f3b 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -2077,8 +2077,7 @@ iomap_writepages(struct iomap_writepage_ctx *wpc) * Writeback from reclaim context should never happen except in the case * of a VM regression so warn about it and refuse to write the data. */ - if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC | PF_KSWAPD)) == - PF_MEMALLOC)) + if (WARN_ON_ONCE((current->flags & PF_MEMALLOC))) return -EIO; while ((folio = writeback_iter(mapping, wpc->wbc, folio, &error))) { From 27517f701ff3b6ab226423cfa67593825d5a60b4 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 2 Sep 2026 21:16:52 +0800 Subject: [PATCH 209/539] mm: replace PF_KSWAPD flag with kthread_func() check The preceding commits removed the last consumer that propagated PF_KSWAPD beyond kswapd itself (XFS btree split worker inheritance). The only remaining setter of PF_KSWAPD is kswapd(), and every current_is_kswapd() caller only needs to check whether the current task *is* the kswapd thread, not whether it inherited the flag. Replace the flag-based test with kthread_func(current) == kswapd, freeing the 0x00020000 PF flag bit. Link: https://lore.kernel.org/20260902131653.1338227-4-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Signed-off-by: Andrew Morton Acked-by: Shakeel Butt Acked-by: Vlastimil Babka (SUSE) Acked-by: Zi Yan Cc: Brendan Jackman Cc: Carlos Maiolino Cc: Christian Brauner Cc: Christoph Hellwig Cc: "Darrick J. Wong" Cc: David Hildenbrand Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan --- include/linux/sched.h | 2 +- include/linux/swap.h | 7 +------ mm/vmscan.c | 10 ++++++++-- tools/sched_ext/include/scx/common.bpf.h | 1 - 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index d35ae49a991f..90393bd53bbc 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1810,7 +1810,7 @@ extern struct pid __rcu *cad_pid; #define PF_USER_WORKER 0x00004000 /* Kernel thread cloned from userspace thread */ #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ #define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ -#define PF_KSWAPD 0x00020000 /* I am kswapd */ +#define PF__HOLE__00020000 0x00020000 #define PF_MEMALLOC_NOFS 0x00040000 /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */ #define PF_MEMALLOC_NOIO 0x00080000 /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */ #define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to, diff --git a/include/linux/swap.h b/include/linux/swap.h index 7a43409879ca..74ce79404247 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -25,12 +25,6 @@ #define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES) - -static inline int current_is_kswapd(void) -{ - return current->flags & PF_KSWAPD; -} - /* * MAX_SWAPFILES defines the maximum number of swaptypes: things which can * be swapped to. The swap type and the offset into that swap type are @@ -339,6 +333,7 @@ void check_move_unevictable_folios(struct folio_batch *fbatch); extern void __meminit kswapd_run(int nid); extern void __meminit kswapd_stop(int nid); +bool current_is_kswapd(void); #ifdef CONFIG_SWAP int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, diff --git a/mm/vmscan.c b/mm/vmscan.c index ba7adf36e69f..245f68c75b28 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7539,7 +7539,7 @@ static int kswapd(void *p) * us from recursively trying to free more memory as we're * trying to free the first piece of memory in the first place). */ - tsk->flags |= PF_MEMALLOC | PF_KSWAPD; + tsk->flags |= PF_MEMALLOC; set_freezable(); WRITE_ONCE(pgdat->kswapd_order, 0); @@ -7589,11 +7589,17 @@ kswapd_try_sleep: goto kswapd_try_sleep; } - tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); + tsk->flags &= ~PF_MEMALLOC; return 0; } +bool current_is_kswapd(void) +{ + return kthread_func(current) == kswapd; +} +EXPORT_SYMBOL_GPL(current_is_kswapd); + /* * A zone is low on free memory or too fragmented for high-order memory. If * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 22f24ebef8a9..5a28a36c5120 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -32,7 +32,6 @@ #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ #define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ -#define PF_KSWAPD 0x00020000 /* I am kswapd */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_EXITING 0x00000004 #define CLOCK_MONOTONIC 1 From 44b74ec3ae5ac2f7773d5656e43aae7b29303b74 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 2 Sep 2026 21:16:53 +0800 Subject: [PATCH 210/539] mm: replace PF_KCOMPACTD flag with kthread_func() check PF_KCOMPACTD was introduced by commit ce6d9c1c2b5c ("NFS: fix nfs_release_folio() to not deadlock via kcompactd writeback") so nfs_release_folio() could detect kcompactd context and skip writeback. The flag is only consumed by current_is_kcompactd(), whose sole caller is nfs_release_folio(). Replace the flag-based check with kthread_func(current) == kcompactd, freeing the 0x00010000 PF flag bit. Link: https://lore.kernel.org/20260902131653.1338227-5-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Signed-off-by: Andrew Morton Acked-by: Shakeel Butt Acked-by: Zi Yan Acked-by: Vlastimil Babka (SUSE) Reviewed-by: David Hildenbrand (Arm) Cc: Brendan Jackman Cc: Carlos Maiolino Cc: Christian Brauner Cc: Christoph Hellwig Cc: "Darrick J. Wong" Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan --- include/linux/compaction.h | 11 ++++++----- include/linux/sched.h | 2 +- mm/compaction.c | 9 ++++++--- tools/sched_ext/include/scx/common.bpf.h | 1 - 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 66a2f70e9e01..691c09f0a697 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -81,10 +81,6 @@ static inline unsigned long compact_gap(unsigned int order) return min(2UL << order, COMPACT_CLUSTER_MAX); } -static inline int current_is_kcompactd(void) -{ - return current->flags & PF_KCOMPACTD; -} #ifdef CONFIG_COMPACTION @@ -103,7 +99,7 @@ extern void compaction_defer_reset(struct zone *zone, int order, bool compaction_zonelist_suitable(struct alloc_context *ac, int order, int alloc_flags, gfp_t gfp_mask); - +bool current_is_kcompactd(void); extern void __meminit kcompactd_run(int nid); extern void __meminit kcompactd_stop(int nid); extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx); @@ -120,6 +116,11 @@ static inline bool compaction_suitable(struct zone *zone, int order, return false; } +static inline bool current_is_kcompactd(void) +{ + return false; +} + static inline void kcompactd_run(int nid) { } diff --git a/include/linux/sched.h b/include/linux/sched.h index 90393bd53bbc..f45b7d43113c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1809,7 +1809,7 @@ extern struct pid __rcu *cad_pid; #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */ #define PF_USER_WORKER 0x00004000 /* Kernel thread cloned from userspace thread */ #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ -#define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ +#define PF__HOLE__00010000 0x00010000 #define PF__HOLE__00020000 0x00020000 #define PF_MEMALLOC_NOFS 0x00040000 /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */ #define PF_MEMALLOC_NOIO 0x00080000 /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */ diff --git a/mm/compaction.c b/mm/compaction.c index a049415512c6..4994e200bbec 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -3197,7 +3197,6 @@ static int kcompactd(void *p) long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC); long timeout = default_timeout; - current->flags |= PF_KCOMPACTD; set_freezable(); pgdat->kcompactd_max_order = 0; @@ -3254,11 +3253,15 @@ static int kcompactd(void *p) pgdat->proactive_compact_trigger = false; } - current->flags &= ~PF_KCOMPACTD; - return 0; } +bool current_is_kcompactd(void) +{ + return kthread_func(current) == kcompactd; +} +EXPORT_SYMBOL_GPL(current_is_kcompactd); + /* * This kcompactd start function will be called by init and node-hot-add. * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added. diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 5a28a36c5120..ab8d676d8cd6 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -31,7 +31,6 @@ #define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ -#define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_EXITING 0x00000004 #define CLOCK_MONOTONIC 1 From 956aeff185e7e16b989202bde0bb29859adba99d Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Wed, 9 Sep 2026 10:11:00 -0700 Subject: [PATCH 211/539] mm: hugetlb: return -ENOSPC on memcg charge failure Patch series "Fix bugs in HugeTLB allocation when mem_cgroup_charge_hugetlb() fails", v2. In hugetlb_alloc_folio(), when mem_cgroup_charge_hugetlb() fails, there are 2 issues: 1. free_huge_folio() expects a non-refcounted folio and will VM_BUG_ON_FOLIO(). 2. -ENOMEM is returned, causing an infinite loop retrying the fault. This patch (of 2): When mem_cgroup_charge_hugetlb() fails with -ENOMEM, alloc_hugetlb_folio() currently propagates this error. This results in the page fault handler returning VM_FAULT_OOM. Because HugeTLB allocations are high-order and use __GFP_RETRY_MAYFAIL, they bypass the OOM killer. Returning VM_FAULT_OOM to the #PF handler without triggering the OOM killer (or having it make progress) leads to an infinite loop of retrying the fault. Avoid this loop by returning -ENOSPC when charging fails, which maps to VM_FAULT_SIGBUS, terminating the process cleanly. Make mem_cgroup_charge_hugetlb() fault handling use a common error handling path, the same handling used for hugetlb_cgroup_uncharge_cgroup{,_rsvd}(), which also don't trigger the OOM killer and hence opt to terminate the process with a SIGBUS. Link: https://lore.kernel.org/20260909-hugetlb-alloc-folio-memcg-charge-error-handling-v2-1-4b4a8a19a7f7@google.com Fixes: 991135774c0e ("memcg/hugetlb: introduce mem_cgroup_charge_hugetlb") Signed-off-by: Ackerley Tng Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Reviewed-by: Joshua Hahn Cc: Alex Shi Cc: David Hildenbrand Cc: David Rientjes Cc: Dongliang Mu Cc: Frank van der Linden Cc: Hongxiang Lou Cc: James Houghton Cc: Johannes Weiner Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Ma Wupeng Cc: Miaohe Lin Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Oscar Salvador Cc: Peter Xu Cc: Roman Gushchin Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vishal Annapurve Cc: Vlastimil Babka Cc: Yanteng Si Cc: --- mm/hugetlb.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 84552c993cbb..a16d04aae619 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2851,7 +2851,6 @@ void wait_for_freed_hugetlb_folios(void) * * Return: A pointer to the allocated folio, or an ERR_PTR on failure. * -ENOSPC if cgroup charging fails or no folio is available. - * -ENOMEM if mem cgroup charging fails. */ struct folio *hugetlb_alloc_folio(struct hstate *h, struct mempolicy_interpreted *mpoli, u8 alloc_flags) @@ -2924,7 +2923,11 @@ struct folio *hugetlb_alloc_folio(struct hstate *h, * were committed to the folio and freeing the folio * would have cleared those up. */ - return ERR_PTR(ret); + /* + * Return -ENOSPC, since retrying the fault is futile: + * the OOM killer is not triggered for HugeTLB. + */ + return ERR_PTR(-ENOSPC); } return folio; From e95d29c5df84f19881cd30b491fc81aa98f44b45 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Wed, 9 Sep 2026 10:11:01 -0700 Subject: [PATCH 212/539] mm: hugetlb: drop refcount before freeing on memcg charge failure When mem_cgroup_charge_hugetlb(folio, gfp) returns -ENOMEM, the folio has its refcount set to 1 via folio_ref_unfreeze(folio, 1). The error path calls free_huge_folio(folio) directly, which expects a refcount of 0. Hence, VM_BUG_ON_FOLIO(folio_ref_count(folio), folio) is triggered. Even with CONFIG_DEBUG_VM disabled, returning a folio with refcount 1 to the freelist can corrupt allocator state later. Use folio_put(folio) instead of free_huge_folio(folio) to properly drop the reference before freeing it. Link: https://lore.kernel.org/20260909-hugetlb-alloc-folio-memcg-charge-error-handling-v2-2-4b4a8a19a7f7@google.com Fixes: 991135774c0e ("memcg/hugetlb: introduce mem_cgroup_charge_hugetlb") Signed-off-by: Ackerley Tng Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Reviewed-by: Joshua Hahn Cc: Alex Shi Cc: David Hildenbrand Cc: David Rientjes Cc: Dongliang Mu Cc: Frank van der Linden Cc: Hongxiang Lou Cc: James Houghton Cc: Johannes Weiner Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Ma Wupeng Cc: Miaohe Lin Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Oscar Salvador Cc: Peter Xu Cc: Roman Gushchin Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vishal Annapurve Cc: Vlastimil Babka Cc: Yanteng Si Cc: --- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a16d04aae619..cdb7d8df666a 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2917,7 +2917,7 @@ struct folio *hugetlb_alloc_folio(struct hstate *h, lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages); if (ret == -ENOMEM) { - free_huge_folio(folio); + folio_put(folio); /* * Skip uncharging hugetlb_cgroup since the charges * were committed to the folio and freeing the folio From 4c770a1ca293b56d77d6375d534f128b7fc9b774 Mon Sep 17 00:00:00 2001 From: Meijing Zhao Date: Wed, 2 Sep 2026 14:45:08 +0800 Subject: [PATCH 213/539] mm: trace: decode arm64 and sparc64 VM_ARCH_1 flags Patch series "mm: trace: decode architecture-specific VMA flags", v2. show_vma_flags(), which is used by VMA tracepoints and %pGv, names generic VM_* bits but does not fully decode architecture-specific flag positions. As a result, trace output and VMA dumps can show a generic "arch_1" name or a raw hexadecimal value instead of the meaning assigned by the target architecture. This series adds symbolic names for the arm64 and sparc64 meanings of VM_ARCH_1, arm64 MTE flags, user shadow stack flags, and protection-key encoding bits under their corresponding configuration guards. This patch (of 3): The VM_ARCH_1 bit has architecture-specific meanings. show_vma_flags(), which is used by VMA tracepoints and %pGv, already reports the powerpc, parisc and no-MMU meanings, but falls back to the generic "arch_1" name on arm64 and sparc64. Report VM_ARM64_BTI as "bti" and VM_SPARC_ADI as "adi" so trace output and %pGv dumps expose the actual architecture-specific state. Link: https://lore.kernel.org/cover.1788330432.git.zhaomeijing@lixiang.com Link: https://lore.kernel.org/bcf0fea58240c4b6daa253739683ed7709e42fbb.1788330432.git.zhaomeijing@lixiang.com Signed-off-by: Meijing Zhao Signed-off-by: Andrew Morton Cc: Lorenzo Stoakes Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Steven Rostedt --- include/trace/events/mmflags.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index 935893e5ea53..02caeaf359ab 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -168,6 +168,10 @@ IF_HAVE_PG_ARCH_3(arch_3) #define __VM_ARCH_SPECIFIC_1 {VM_SAO, "sao" } #elif defined(CONFIG_PARISC) #define __VM_ARCH_SPECIFIC_1 {VM_GROWSUP, "growsup" } +#elif defined(CONFIG_SPARC64) +#define __VM_ARCH_SPECIFIC_1 {VM_SPARC_ADI, "adi" } +#elif defined(CONFIG_ARM64) +#define __VM_ARCH_SPECIFIC_1 {VM_ARM64_BTI, "bti" } #elif !defined(CONFIG_MMU) #define __VM_ARCH_SPECIFIC_1 {VM_MAPPED_COPY,"mappedcopy" } #else From a971ac3a0af7cbc6c4980cdd41d6574bb694ee79 Mon Sep 17 00:00:00 2001 From: Meijing Zhao Date: Wed, 2 Sep 2026 14:45:09 +0800 Subject: [PATCH 214/539] mm: trace: decode MTE and shadow stack VMA flags show_vma_flags(), which is used by VMA tracepoints and %pGv, leaves architecture-specific HIGH_ARCH_* bits unnamed. Arm64 MTE flags and user shadow stack flags are therefore printed as raw hexadecimal values. These bit positions are shared between architectures. For example, bit 37 represents VM_MTE_ALLOWED on arm64 but VM_SHADOW_STACK on x86, so a shared HIGH_ARCH_* bit cannot be given an unconditional name. Add conditionally compiled names for VM_MTE, VM_MTE_ALLOWED and VM_SHADOW_STACK. Keep the configuration guards aligned with the definitions of these aliases so each shared bit position is decoded according to the target architecture. For example, an arm64 VMA containing VM_MTE_ALLOWED is now printed as: ...|account|mte_allowed|... instead of: ...|account|0x2000000000 Link: https://lore.kernel.org/1c7f7003cff8d00209baa8498f1efee113f5b5a4.1788330432.git.zhaomeijing@lixiang.com Signed-off-by: Meijing Zhao Signed-off-by: Andrew Morton Cc: Lorenzo Stoakes Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Steven Rostedt --- include/trace/events/mmflags.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index 02caeaf359ab..2f980d20a9f6 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -202,6 +202,19 @@ IF_HAVE_PG_ARCH_3(arch_3) # define IF_HAVE_VM_DROPPABLE(flag, name) #endif +#ifdef CONFIG_ARM64_MTE +# define IF_HAVE_VM_MTE(flag, name) {flag, name}, +#else +# define IF_HAVE_VM_MTE(flag, name) +#endif + +#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_RISCV_USER_CFI) || \ + defined(CONFIG_ARM64_GCS) +# define IF_HAVE_VM_SHADOW_STACK(flag, name) {flag, name}, +#else +# define IF_HAVE_VM_SHADOW_STACK(flag, name) +#endif + #define __def_vmaflag_names \ {VM_READ, "read" }, \ {VM_WRITE, "write" }, \ @@ -237,6 +250,9 @@ IF_HAVE_VM_SOFTDIRTY(VM_SOFTDIRTY, "softdirty" ) \ {VM_HUGEPAGE, "hugepage" }, \ {VM_NOHUGEPAGE, "nohugepage" }, \ IF_HAVE_VM_DROPPABLE(VM_DROPPABLE, "droppable" ) \ +IF_HAVE_VM_MTE(VM_MTE, "mte") \ +IF_HAVE_VM_MTE(VM_MTE_ALLOWED, "mte_allowed") \ +IF_HAVE_VM_SHADOW_STACK(VM_SHADOW_STACK, "shadow_stack") \ {VM_MERGEABLE, "mergeable" } \ #define show_vma_flags(flags) \ From 18752045436318efa8e782cee5c7824b54ec1289 Mon Sep 17 00:00:00 2001 From: Meijing Zhao Date: Wed, 2 Sep 2026 14:45:10 +0800 Subject: [PATCH 215/539] mm: trace: name protection key encoding bits Protection keys are encoded in architecture-specific HIGH_ARCH_* VMA flag bits. show_vma_flags(), which is used by VMA tracepoints and %pGv, does not name those bits, leaving them as raw hexadecimal values. Name the protection-key encoding bits pkey_bit0 through pkey_bit4 under CONFIG_ARCH_HAS_PKEYS. The names make clear that these are bits of one protection-key value rather than independent protection keys. For example, protection key 3 is represented as: pkey_bit0|pkey_bit1 Honor CONFIG_ARCH_PKEY_BITS when exposing bit 3 and bit 4 so only bits provided by the architecture are included. Link: https://lore.kernel.org/1871da32001243b37147bac5af02d2f2ec64304e.1788330432.git.zhaomeijing@lixiang.com Signed-off-by: Meijing Zhao Signed-off-by: Andrew Morton Cc: Lorenzo Stoakes Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Steven Rostedt --- include/trace/events/mmflags.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index 2f980d20a9f6..ef9aa388b84f 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -202,6 +202,24 @@ IF_HAVE_PG_ARCH_3(arch_3) # define IF_HAVE_VM_DROPPABLE(flag, name) #endif +#ifdef CONFIG_ARCH_HAS_PKEYS +# define IF_HAVE_VM_PKEY(flag, name) {flag, name}, +#if CONFIG_ARCH_PKEY_BITS > 3 +# define IF_HAVE_VM_PKEY3(flag, name) {flag, name}, +#else +# define IF_HAVE_VM_PKEY3(flag, name) +#endif +#if CONFIG_ARCH_PKEY_BITS > 4 +# define IF_HAVE_VM_PKEY4(flag, name) {flag, name}, +#else +# define IF_HAVE_VM_PKEY4(flag, name) +#endif +#else +# define IF_HAVE_VM_PKEY(flag, name) +# define IF_HAVE_VM_PKEY3(flag, name) +# define IF_HAVE_VM_PKEY4(flag, name) +#endif + #ifdef CONFIG_ARM64_MTE # define IF_HAVE_VM_MTE(flag, name) {flag, name}, #else @@ -250,6 +268,11 @@ IF_HAVE_VM_SOFTDIRTY(VM_SOFTDIRTY, "softdirty" ) \ {VM_HUGEPAGE, "hugepage" }, \ {VM_NOHUGEPAGE, "nohugepage" }, \ IF_HAVE_VM_DROPPABLE(VM_DROPPABLE, "droppable" ) \ +IF_HAVE_VM_PKEY(VM_PKEY_BIT0, "pkey_bit0") \ +IF_HAVE_VM_PKEY(VM_PKEY_BIT1, "pkey_bit1") \ +IF_HAVE_VM_PKEY(VM_PKEY_BIT2, "pkey_bit2") \ +IF_HAVE_VM_PKEY3(VM_PKEY_BIT3, "pkey_bit3") \ +IF_HAVE_VM_PKEY4(VM_PKEY_BIT4, "pkey_bit4") \ IF_HAVE_VM_MTE(VM_MTE, "mte") \ IF_HAVE_VM_MTE(VM_MTE_ALLOWED, "mte_allowed") \ IF_HAVE_VM_SHADOW_STACK(VM_SHADOW_STACK, "shadow_stack") \ From b2016f6c0da815d7281eb7d72eddcf630becb9bf Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:34 -0700 Subject: [PATCH 216/539] mm/damon/core: use damon_nr_samples_per_aggr() for max merge threshold Patch series "mm/damon: cleanup code, add test cases, and update guidances in docs". Misc cleanup, improvements and updates of code, test, and documents. Patches 1-5 cleanup DAMON code. Patches 6-10 adds kunit and selftest test cases for recently fixed bugs and a new feature. Patches 11 and 12 update guidelines for AI review and what document to read, on DAMON documents. This patch (of 12): kdamond_merge_regions() open-codes max region merge threshold calculation. What it does is fundamentally the same as damon_nr_samples_per_aggr() but missing a few corner cases. The unhandled corner cases should be rare and make only a negligible level of monitoring results degradation. But having the inconsistency could increase future maintenance burden. Use the dedicated function. Link: https://lore.kernel.org/20260902054747.99370-1-sj@kernel.org Link: https://lore.kernel.org/20260902054747.99370-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index cf4ec122a7f9..1cf4d41f24dc 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3533,8 +3533,7 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, unsigned int max_thres; bool count_age = true; - max_thres = c->attrs.aggr_interval / - (c->attrs.sample_interval ? c->attrs.sample_interval : 1); + max_thres = damon_nr_samples_per_aggr(&c->attrs); while (true) { nr_regions = 0; damon_for_each_target(t, c) { From db3b91276277ff6dfb8d945c17322d44d10007a1 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:35 -0700 Subject: [PATCH 217/539] mm/damon/core: remove debug messages There are a few debug messages in DAMON core. Those have not really been used in a meaningful way for the last few years, though. Remove those. Link: https://lore.kernel.org/20260902054747.99370-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Kunwu Chan --- mm/damon/core.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 1cf4d41f24dc..4d817063a7a2 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3756,10 +3756,6 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme) /* higher than high watermark or lower than low watermark */ if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { - if (scheme->wmarks.activated) - pr_debug("deactivate a scheme (%d) for %s wmark\n", - scheme->action, - str_high_low(metric > scheme->wmarks.high)); scheme->wmarks.activated = false; return scheme->wmarks.interval; } @@ -3769,8 +3765,6 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme) !scheme->wmarks.activated) return scheme->wmarks.interval; - if (!scheme->wmarks.activated) - pr_debug("activate a scheme (%d)\n", scheme->action); scheme->wmarks.activated = true; return 0; } @@ -3913,8 +3907,6 @@ static int kdamond_fn(void *data) struct damon_ctx *ctx = data; unsigned long sz_limit = 0; - pr_debug("kdamond (%d) starts\n", current->pid); - mutex_lock(&ctx->call_controls_lock); ctx->call_controls_obsolete = false; mutex_unlock(&ctx->call_controls_lock); @@ -4068,7 +4060,6 @@ done: mutex_unlock(&ctx->walk_control_lock); damos_walk_cancel(ctx); - pr_debug("kdamond (%d) finishes\n", current->pid); mutex_lock(&ctx->kdamond_lock); ctx->kdamond = NULL; mutex_unlock(&ctx->kdamond_lock); From b0195af30ff01999772207bbd8305d066d408d0f Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 23:08:16 -0700 Subject: [PATCH 218/539] mm/damon/core: remove string_choices.h include It is no more being used. Link: https://lore.kernel.org/20260902061401.104419-1-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Kunwu Chan --- mm/damon/core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 4d817063a7a2..88bd2f505cea 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -15,7 +15,6 @@ #include #include #include -#include /* for damon_get_folio() used by node eligible memory metrics */ #include "ops-common.h" From 477eac743ff867b8e62907e649ecb7a8bb305176 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:36 -0700 Subject: [PATCH 219/539] mm/damon/vaddr: remove a debug message There is a debug message in the DAMON virtual address space operation set. It has not really been used in a meaningful way for the last few years, though. Remove it. Link: https://lore.kernel.org/20260902054747.99370-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/vaddr.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 5b4d16c8db62..91a0d441c1f9 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -189,22 +189,12 @@ static int damon_va_three_regions(struct damon_target *t, * * */ -static void __damon_va_init_regions(struct damon_ctx *ctx, - struct damon_target *t) +static void __damon_va_init_regions(struct damon_target *t) { - struct damon_target *ti; struct damon_addr_range regions[3]; - int tidx = 0; - if (damon_va_three_regions(t, regions)) { - damon_for_each_target(ti, ctx) { - if (ti == t) - break; - tidx++; - } - pr_debug("Failed to get three regions of %dth target\n", tidx); + if (damon_va_three_regions(t, regions)) return; - } damon_set_regions(t, regions, 3, DAMON_MIN_REGION_SZ); } @@ -217,7 +207,7 @@ static void damon_va_init(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { /* the user may set the target regions as they want */ if (!damon_nr_regions(t)) - __damon_va_init_regions(ctx, t); + __damon_va_init_regions(t); } } From c39d36430a29d15c944894b7cd65ff6fe941cbce Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:37 -0700 Subject: [PATCH 220/539] mm/damon/core: validate number of probes in valid_probe_params() Each DAMON context is allowed to have only up to DAMON_MAX_PROBES probes. The central place for validating DAMON probe parameters, damon_valid_probe_params(), is not validating the upper limit, though. Do the validation. Link: https://lore.kernel.org/20260902054747.99370-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 88bd2f505cea..87cc5d756893 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1419,6 +1419,13 @@ static bool damon_valid_probe_params(struct damon_ctx *ctx) unsigned char max_probe_hits; struct damon_probe *probe; unsigned int wsum, wsum_to_add; + int nr_probes; + + nr_probes = 0; + damon_for_each_probe(probe, ctx) + nr_probes++; + if (nr_probes > DAMON_MAX_PROBES) + return false; if (!damon_has_probe_weights(ctx)) return true; From 8879ae27da65634fbbb3888ebced10ffd9297221 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:38 -0700 Subject: [PATCH 221/539] mm/damon/sysfs: remove probes number validation DAMON sysfs interface is disallowing >DAMON_MAX_PROBES nr_probes input, since DAMON_MAX_PROBES is the upper limit of probes per DAMON context. The core layer is validating the upper limit again, though. It is preferred to let DAMON API callers such as sysfs interface to set parameters in flexible ways, and do parameters validation in the core layer. Drop the duplicated validation in the sysfs interface. Link: https://lore.kernel.org/20260902054747.99370-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 7ec14f48d157..b576e97cbfdb 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1479,7 +1479,7 @@ static ssize_t nr_probes_store(struct kobject *kobj, if (err) return err; - if (nr < 0 || nr > DAMON_MAX_PROBES) + if (nr < 0) return -EINVAL; probes = container_of(kobj, struct damon_sysfs_probes, kobj); From 215ca26f9cd0c2410611a6d89ae91e7b23d80634 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:39 -0700 Subject: [PATCH 222/539] mm/damon/tests/core-kunit: extend set_regions() test for error case damon_test_set_regions_for() is designed to test only success-expected damon_set_regions() calls. Extend it to cover error-expected calls, too. Link: https://lore.kernel.org/20260902054747.99370-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Kunwu Chan --- mm/damon/tests/core-kunit.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index f2568fba552e..d20ea12ca616 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -469,11 +469,12 @@ static void damon_test_set_regions_for(struct kunit *test, struct damon_addr_range *old_ranges, int sz_old_ranges, struct damon_addr_range *new_ranges, int sz_new_ranges, unsigned long min_region_sz, - struct damon_addr_range *expect_ranges, int sz_expect_ranges) + struct damon_addr_range *expect_ranges, int sz_expect_ranges, + int expect_err) { struct damon_target *t; struct damon_region *r; - int i; + int i, err; t = damon_new_target(); if (!t) @@ -487,7 +488,8 @@ static void damon_test_set_regions_for(struct kunit *test, damon_add_region(r, t); } - damon_set_regions(t, new_ranges, sz_new_ranges, min_region_sz); + err = damon_set_regions(t, new_ranges, sz_new_ranges, min_region_sz); + KUNIT_EXPECT_EQ(test, err, expect_err); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), sz_expect_ranges); if (damon_nr_regions(t) != sz_expect_ranges) { @@ -516,7 +518,7 @@ static void damon_test_set_regions(struct kunit *test) (struct damon_addr_range[]){ {.start = 5, .end = 15}, {.start = 15, .end = 25}, - }, 2); + }, 2, 0); /* Un-intersecting regions should be removed. */ damon_test_set_regions_for(test, (struct damon_addr_range[]){ @@ -529,7 +531,7 @@ static void damon_test_set_regions(struct kunit *test) 1, (struct damon_addr_range[]){ {.start = 18, .end = 23}, - }, 1); + }, 1, 0); /* * Holes should be filled up with new regions. * @@ -550,7 +552,7 @@ static void damon_test_set_regions(struct kunit *test) {.start = 8, .end = 16}, {.start = 16, .end = 24}, {.start = 24, .end = 28}, - }, 3); + }, 3, 0); /* * New regions should be able to be appended. * @@ -572,7 +574,7 @@ static void damon_test_set_regions(struct kunit *test) {.start = 0, .end = 4}, {.start = 4, .end = 15}, {.start = 25, .end = 40}, - }, 3); + }, 3, 0); /* * New regions should be able to be inserted. * @@ -595,7 +597,7 @@ static void damon_test_set_regions(struct kunit *test) {.start = 0, .end = 15}, {.start = 25, .end = 40}, {.start = 44, .end = 50}, - }, 3); + }, 3, 0); } static void damon_test_update_monitoring_result(struct kunit *test) From 331a40c769070e9c188898b831c9a03f975a56bc Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:40 -0700 Subject: [PATCH 223/539] mm/damon/tests/core-kunit: test <=0 size damon_set_regions() inputs Commit 1292c0ecb1ca ("mm/damon/core: validate ranges in damon_set_regions()") disallowed passing zero or negative size input ranges to damon_set_regions(). Add kunit test cases for those inputs. Link: https://lore.kernel.org/20260902054747.99370-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Kunwu Chan --- mm/damon/tests/core-kunit.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index d20ea12ca616..89f364a8ffeb 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -598,6 +598,20 @@ static void damon_test_set_regions(struct kunit *test) {.start = 25, .end = 40}, {.start = 44, .end = 50}, }, 3, 0); + /* Zero size regions should return -EINVAL. */ + damon_test_set_regions_for(test, + (struct damon_addr_range[]){}, 0, + (struct damon_addr_range[]){ + {.start = 42, .end = 42}, + }, 1, 1, + (struct damon_addr_range[]){}, 0, -EINVAL); + /* Negative size regions should return -EINVAL. */ + damon_test_set_regions_for(test, + (struct damon_addr_range[]){}, 0, + (struct damon_addr_range[]){ + {.start = 42, .end = 21}, + }, 1, 1, + (struct damon_addr_range[]){}, 0, -EINVAL); } static void damon_test_update_monitoring_result(struct kunit *test) From cd397f5c145c796ea59a0d423936dc2cd9e28004 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:41 -0700 Subject: [PATCH 224/539] mm/damon/tests/core-kunit: test overlapping ranges for set_regions() Commit 954157679ec3 ("mm/damon/core: disallow overlapping input ranges for damon_set_regions()") disallowed passing overlapping input ranges to damon_set_regions(). Add a kunit test case for the overlapping input. Link: https://lore.kernel.org/20260902054747.99370-9-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/tests/core-kunit.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 89f364a8ffeb..ebb695090bb4 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -612,6 +612,17 @@ static void damon_test_set_regions(struct kunit *test) {.start = 42, .end = 21}, }, 1, 1, (struct damon_addr_range[]){}, 0, -EINVAL); + /* + * Regions resulting in same region after alignment should return + * -EINVAL. + */ + damon_test_set_regions_for(test, + (struct damon_addr_range[]){}, 0, + (struct damon_addr_range[]){ + {.start = 10, .end = 20}, + {.start = 20, .end = 30}, + }, 2, 4096, + (struct damon_addr_range[]){}, 0, -EINVAL); } static void damon_test_update_monitoring_result(struct kunit *test) From 0e51d600b7ec23868a65ae409bca7ac983a81dde Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:42 -0700 Subject: [PATCH 225/539] mm/damon/tests/core-kunit: test damon_nr_samples_per_aggr() damon_max_nr_accesses(), which is a previous version of damon_nr_samples_per_aggr() before the renaming, was wrongly returning zero or random overflowed values for extreme intervals setup. Commit 35d4a3cf70a8 ("mm/damon/ops-common: handle extreme intervals in damon_hot_score()") updated the function to return correct or more valid values. Add a kunit test to ensure it is working as expected. Link: https://lore.kernel.org/20260902054747.99370-10-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/tests/core-kunit.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index ebb695090bb4..c01e6a75cadc 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -625,6 +625,27 @@ static void damon_test_set_regions(struct kunit *test) (struct damon_addr_range[]){}, 0, -EINVAL); } +static void damon_test_nr_samples_per_aggr(struct kunit *test) +{ + struct damon_attrs attrs = { + .sample_interval = 0, + .aggr_interval = 0, + }; + + /* Zero aggregation interval doesn't cause division by zero */ + KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), 1); + + /* + * Too large aggregation interval on 64 bit system doesn't cause + * overflow + */ + if (ULONG_MAX > UINT_MAX) { + attrs.aggr_interval = (unsigned long)UINT_MAX + 1; + KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), + UINT_MAX); + } +} + static void damon_test_update_monitoring_result(struct kunit *test) { struct damon_attrs old_attrs = { @@ -1858,6 +1879,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_split_above_half_progresses), KUNIT_CASE(damon_test_ops_registration), KUNIT_CASE(damon_test_set_regions), + KUNIT_CASE(damon_test_nr_samples_per_aggr), KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), KUNIT_CASE(damon_test_mvsum), From df31e773941d0f425189194f21c61d1c34aca51d Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:43 -0700 Subject: [PATCH 226/539] selftests/damon/sysfs.sh: test hugepage_mem_bp quota goal DAMON sysfs quota goal target_metric file now accepts 'hugepage_mem_bp' input. Test it is accepted in fundamental DAMON sysfs file operation selftest. Link: https://lore.kernel.org/20260902054747.99370-11-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/damon/sysfs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index ddebde6edabe..b66593c9ac47 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -210,6 +210,7 @@ test_goal() ensure_write_succ "$fpath" "active_mem_bp" "valid input" ensure_write_succ "$fpath" "inactive_mem_bp" "valid input" ensure_write_succ "$fpath" "node_eligible_mem_bp" "valid input" + ensure_write_succ "$fpath" "hugepage_mem_bp" "valid input" ensure_write_fail "$fpath" "foo" "invalid input" ensure_file "$goal_dir/nid" "exist" "600" ensure_file "$goal_dir/path" "exist" "600" From 179e31269a151c149bbf444ac7da06bf1dbde1d8 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:44 -0700 Subject: [PATCH 227/539] Docs/mm/damon/maintainer-profile: update AI review for Sashiko replies In the past, sharing Sashiko review results was tedious. Also it was suggested to minimize the recipients on the sharing mails. Hence the AI review section of DAMON maintainer profile document was updated to give guidance about available tools for making the sharing easier, and how the recipients list should be managed. Now DAMON is onboarded [1] to Sashiko's automatic review replies feature. Sashiko directly sends its reviews as replies to the patch mail thread. It also reduces the recipients list to deliver the reporting to only the author and the mailing list. Remove the old guidance that is no more necessary but only confusing. Link: https://lore.kernel.org/20260902054747.99370-12-sj@kernel.org Link: https://github.com/sashiko-dev/sashiko/commit/b554c7b6e733 [1] Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Kunwu Chan --- Documentation/mm/damon/maintainer-profile.rst | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/Documentation/mm/damon/maintainer-profile.rst b/Documentation/mm/damon/maintainer-profile.rst index fb2fa00cc9aa..a7c135233937 100644 --- a/Documentation/mm/damon/maintainer-profile.rst +++ b/Documentation/mm/damon/maintainer-profile.rst @@ -106,18 +106,9 @@ AI Review For patches that are publicly posted to DAMON mailing list (damon@lists.linux.dev), AI reviews of the patches will be available at -sashiko.dev. The reviews could also be sent as mails to the author of the -patch. +sashiko.dev. The reviews will also be sent as replies to the author of the +patch and the mailing list. -Patch authors are encouraged to check the AI reviews and share their opinions. -The sharing could be done as a reply to the mail thread. Consider reducing the -recipients list for such sharing, since some people are not really interested -in AI reviews. As a rule of thumb, drop stable@vger.kernel.org and individuals -except DAMON maintainer. - -`hkml` also provides a `feature -`_ -for such sharing. Please feel free to use the feature. - -It is only an optional recommendation. DAMON maintainer could also ask any -question about the AI reviews, though. +Patch authors are encouraged to check the AI reviews and share their opinions +by replying on the mail thread. It is only an optional recommendation. DAMON +maintainer could also ask any question about the AI reviews, though. From 6f641002218c83ad7ac7bf0a4c914fe6dfedf0fe Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 1 Sep 2026 22:47:45 -0700 Subject: [PATCH 228/539] Docs/ABI/damon: recommend subsystem doc instead of admin-guide DAMON ABI doc is recommending DAMON admin-guide for people who are willing to know further about DAMON. Nowadays the subsystem doc (Documentation/mm/damon) is a more recommended place for even beginners. Recommend the subsystem doc over admin-guide. Link: https://lore.kernel.org/20260902054747.99370-13-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index f8d2601e8290..ad21f58f3c91 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -3,7 +3,7 @@ Date: Mar 2022 Contact: SJ Park Description: Interface for Data Access MONitoring (DAMON). Contains files for controlling DAMON. For more details on DAMON itself, - please refer to Documentation/admin-guide/mm/damon/index.rst. + please refer to Documentation/mm/damon/index.rst. What: /sys/kernel/mm/damon/admin/ Date: Mar 2022 From ae62466cdaf51ca99499a1244ffdfe8e4f44aaec Mon Sep 17 00:00:00 2001 From: zhaozhengzhuo Date: Wed, 2 Sep 2026 11:12:29 +0800 Subject: [PATCH 229/539] mm/migrate_device: fix function name in kernel-doc The kernel-doc for migrate_device_range() says that migrate_vma_setup() is similar to itself. Refer to migrate_device_range() as the subject of the comparison, making the distinction between virtual-address-based and device-PFN-based migration clear. Link: https://lore.kernel.org/13768B0F4A5FC1F5+20260902031229.1821112-1-zhaozhengzhuo@uniontech.com Fixes: e778406b40db ("mm/migrate_device.c: add migrate_device_range()") Signed-off-by: zhaozhengzhuo Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Cc: Alistair Popple Cc: David Hildenbrand Cc: Byungchul Park Cc: Gregory Price Cc: "Huang, Ying" Cc: Joshua Hahn Cc: Matthew Brost Cc: Rakie Kim --- mm/migrate_device.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 009bfa8b212d..0c437004329d 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -1398,9 +1398,9 @@ static unsigned long migrate_device_pfn_lock(unsigned long pfn) * @start: starting pfn in the range to migrate. * @npages: number of pages to migrate. * - * migrate_vma_setup() is similar in concept to migrate_vma_setup() except that - * instead of looking up pages based on virtual address mappings a range of - * device pfns that should be migrated to system memory is used instead. + * migrate_device_range() is similar in concept to migrate_vma_setup(), except + * that instead of looking up pages based on virtual address mappings, a range + * of device pfns that should be migrated to system memory is used. * * This is useful when a driver needs to free device memory but doesn't know the * virtual mappings of every page that may be in device memory. For example this From 908dd0101e462e317b980bd8c139f47461499540 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Wed, 24 Jun 2026 08:23:59 +0000 Subject: [PATCH 230/539] mm/page_vma_mapped: guard check_pmd() with CONFIG_TRANSPARENT_HUGEPAGE The kernel test robot reported a build failure on the parisc architecture when expanding HPAGE_PMD_NR in check_pmd(). mm/page_vma_mapped.c:142:13: note: in expansion of macro 'HPAGE_PMD_NR' if ((pfn + HPAGE_PMD_NR - 1) < pvmw->pfn) ^~~~~~~~~~~~ The config [1] in report link shows neither TRANSPARENT_HUGEPAGE nor HUGETLB_PAGE is defined. Then trigger the BUILD_BUG. Fix it by define check_pmd() under CONFIG_TRANSPARENT_HUGEPAGE. Link: https://lore.kernel.org/20260624082359.2869-1-richard.weiyang@gmail.com Link: https://download.01.org/0day-ci/archive/20260624/202606240042.ffPsEXVc-lkp@intel.com/config [1] Fixes: 2aff7a4755be ("mm: Convert page_vma_mapped_walk to work on PFNs") Signed-off-by: Wei Yang Signed-off-by: Andrew Morton Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202606240042.ffPsEXVc-lkp@intel.com/ Cc: David Hildenbrand Cc: Harry Yoo Cc: Jann Horn Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Rik van Riel Cc: Vlastimil Babka --- mm/page_vma_mapped.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 4e964545e5e8..28e306fdb3a5 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -142,6 +142,7 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) return true; } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE /* Returns true if the two ranges overlap. Careful to not overflow. */ static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw) { @@ -151,6 +152,12 @@ static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw) return false; return true; } +#else +static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw) +{ + return false; +} +#endif static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size) { From aa4998d62b84e4d68f93ab2aac8f9969964ec241 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Thu, 3 Sep 2026 21:52:51 +0800 Subject: [PATCH 231/539] selftests/mm: remove unreachable returns after ksft exit helpers The ksft_exit*() helpers such as ksft_exit_fail_msg() are declared __noreturn, and the ksft_exit() and ksft_finished() macros expand to calls of them, always terminating the process via exit(). Any return statements following such calls are unreachable, both at the end of main() and on error paths of helper functions. Remove all of them. No functional change. Link: https://lore.kernel.org/20260903135251.39593-1-zenghui.yu@linux.dev Signed-off-by: Zenghui Yu (Huawei) Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: SJ Park Reviewed-by: Zi Yan Assisted-by: GLM-5.3 OpenCode Cc: Kiryl Shutsemau Cc: Baolin Wang Cc: Barry Song Cc: David Hildenbrand Cc: Dev Jain Cc: Lance Yang Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Ryan Roberts Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/folio_split_race_test.c | 2 -- tools/testing/selftests/mm/mlock-random-test.c | 1 - tools/testing/selftests/mm/pkey_sighandler_tests.c | 1 - tools/testing/selftests/mm/split_huge_page_test.c | 4 ---- 4 files changed, 8 deletions(-) diff --git a/tools/testing/selftests/mm/folio_split_race_test.c b/tools/testing/selftests/mm/folio_split_race_test.c index 45b84f7b364e..1960635a953e 100644 --- a/tools/testing/selftests/mm/folio_split_race_test.c +++ b/tools/testing/selftests/mm/folio_split_race_test.c @@ -269,6 +269,4 @@ int main(void) NUM_ITERATIONS); ksft_exit(iter == NUM_ITERATIONS); - - return 0; } diff --git a/tools/testing/selftests/mm/mlock-random-test.c b/tools/testing/selftests/mm/mlock-random-test.c index 16294bc7dae6..58772914fd79 100644 --- a/tools/testing/selftests/mm/mlock-random-test.c +++ b/tools/testing/selftests/mm/mlock-random-test.c @@ -71,7 +71,6 @@ int get_proc_locked_vm_size(void) fclose(f); ksft_exit_fail_msg("cannot parse VmLck in /proc/self/status: %s\n", strerror(errno)); - return -1; } /* diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 74bf79a5399d..f9c728ba96a5 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -556,5 +556,4 @@ int main(int argc, char *argv[]) } ksft_finished(); - return 0; } diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 86a603692826..c01d227d7fd6 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -101,7 +101,6 @@ static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, return (pfn_flags & folio_tail_flags) != folio_tail_flags; fail: ksft_exit_fail_msg("Failed to get folio info\n"); - return false; } static int check_after_split_folio_orders(char *vaddr_start, size_t len, @@ -548,7 +547,6 @@ err_out_close: err_out_unlink: unlink(testfile); ksft_exit_fail_msg("Failed to create large pagecache folios\n"); - return -1; } static void split_thp_in_pagecache_to_order_at(size_t fd_size, @@ -711,6 +709,4 @@ int main(int argc, char **argv) free(expected_orders); ksft_finished(); - - return 0; } From 600c820b8ee5016b8d2985c6181d002125062dc1 Mon Sep 17 00:00:00 2001 From: Christos Skarlos Date: Thu, 3 Sep 2026 12:22:00 +0300 Subject: [PATCH 232/539] mm/huge_memory: fix various coding style warnings Resolve coding style issues flagged by checkpatch.pl Specifically: - Add missing blank lines after variable declarations. - Remove unnecessary braces {} for a single statement block. No functional changes are introduced. Link: https://lore.kernel.org/20260903092200.88910-1-christosskarlos.kernel@gmail.com Signed-off-by: Christos Skarlos Signed-off-by: Andrew Morton Reviewed-by: Barry Song Reviewed-by: Zi Yan Reviewed-by: Lorenzo Stoakes (ARM) Cc: Baolin Wang Cc: David Hildenbrand Cc: Dev Jain Cc: Lance Yang Cc: Liam R. Howlett Cc: Ryan Roberts --- mm/huge_memory.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c5d11147b69a..dd66c6ad5af1 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1146,6 +1146,7 @@ subsys_initcall(hugepage_init); static int __init setup_transparent_hugepage(char *str) { int ret = 0; + if (!str) goto out; if (!strcmp(str, "always")) { @@ -1552,6 +1553,7 @@ static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, struct folio *zero_folio) { pmd_t entry; + entry = folio_mk_pmd(zero_folio, vma->vm_page_prot); entry = pmd_mkspecial(entry); pgtable_trans_huge_deposit(mm, pmd, pgtable); @@ -2661,6 +2663,7 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { pgtable_t pgtable; + pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); pgtable_trans_huge_deposit(mm, new_pmd, pgtable); } @@ -3949,9 +3952,8 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, * swapcache folio split. Only uniform split to order-0 can be used * here. */ - if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && folio_test_swapcache(folio)) { + if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && folio_test_swapcache(folio)) return -EINVAL; - } if (is_huge_zero_folio(folio)) return -EINVAL; From e1e86503027a00a843a77a0477899e5423c43a59 Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Thu, 3 Sep 2026 17:21:26 +0800 Subject: [PATCH 233/539] mm/page_owner: preserve original free_pid/free_tgid during folio migration __update_page_owner_free_handle() accepts pid and tgid, but ignores them, always storing current->pid and current->tgid. __folio_copy_owner() forwards the original free pid/tgid during folio migration, yet these values were overwritten by the migrating task. Store the passed-in pid/tgid so the migrated folio keeps the original free attribution. Link: https://lore.kernel.org/20260903092126.24685-1-hongfu.li@linux.dev Signed-off-by: Hongfu Li Signed-off-by: Andrew Morton Acked-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Zi Yan --- mm/page_owner.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 3fc37d9b908e..cfc31c92d765 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -307,8 +307,8 @@ static inline void __update_page_owner_free_handle(struct page *page, page_owner->free_handle = handle; } page_owner->free_ts_nsec = free_ts_nsec; - page_owner->free_pid = current->pid; - page_owner->free_tgid = current->tgid; + page_owner->free_pid = pid; + page_owner->free_tgid = tgid; } rcu_read_unlock(); } From c2deb1356d82c9d54bd2fcf1241cc97eb66fd81f Mon Sep 17 00:00:00 2001 From: Jinmeng Zhou Date: Thu, 3 Sep 2026 15:50:48 +0800 Subject: [PATCH 234/539] mm/hugetlb: charge folios to the target mm's memcg HugeTLB folios are currently charged to the memcg of the allocating task. This gives the wrong result when a userfaultfd handler populates a HugeTLB VMA that belongs to another process. The UFFDIO_COPY ioctl operates on the userfaultfd context's mm, but get_mem_cgroup_from_current() charges the folio to the handler's memcg instead. This can be reproduced by placing the faulting process and its userfaultfd handler in different memory cgroups. Have the target process register a HugeTLB mapping with userfaultfd, trigger a missing fault, and let the handler resolve it with UFFDIO_COPY. The hugepage usage is then reported in the handler's memory.current instead of the target's. The generic userfaultfd population path avoids this problem by charging folios to dst_vma->vm_mm. Pass the target mm through hugetlb_alloc_folio() and charge the folio by using get_mem_cgroup_from_mm(). This preserves the existing charge timing and error handling while making HugeTLB userfaultfd population consistent with the generic path. Link: https://lore.kernel.org/20260903075048.3316-1-zhoujinmeng@bytedance.com Fixes: 8cba9576df60 ("hugetlb: memcg: account hugetlb-backed memory in memory controller") Signed-off-by: Jinmeng Zhou Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Reviewed-by: Hongfu Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Michal Hocko Cc: Nhat Pham Cc: Oscar Salvador Cc: Roman Gushchin Cc: Shakeel Butt Cc: --- include/linux/hugetlb.h | 3 ++- include/linux/memcontrol.h | 8 +++++--- mm/hugetlb.c | 9 ++++++--- mm/memcontrol.c | 6 ++++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 900c95e346b2..80a5a03e9cee 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -694,7 +694,8 @@ enum hugetlb_alloc_flag { #define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT) struct folio *hugetlb_alloc_folio(struct hstate *h, - struct mempolicy_interpreted *mpoli, u8 alloc_flags); + struct mempolicy_interpreted *mpoli, struct mm_struct *mm, + u8 alloc_flags); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, bool cow_from_owner); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index c79992643556..058ebd73ff16 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -647,7 +647,8 @@ static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, return __mem_cgroup_charge(folio, mm, gfp); } -int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp); +int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, + gfp_t gfp); int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp); @@ -1146,9 +1147,10 @@ static inline int mem_cgroup_charge(struct folio *folio, return 0; } -static inline int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp) +static inline int mem_cgroup_charge_hugetlb(struct folio *folio, + struct mm_struct *mm, gfp_t gfp) { - return 0; + return 0; } static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index cdb7d8df666a..c59ebbcf6684 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2844,6 +2844,7 @@ void wait_for_freed_hugetlb_folios(void) * hugetlb_alloc_folio - Allocate a hugetlb folio. * @h: Hugetlb state control block. * @mpoli: Interpreted memory policy to use for allocation. + * @mm: Memory descriptor of the allocation target. * @alloc_flags: Flags controlling the allocation behavior. * * Allocates a hugetlb folio and handles cgroup charging and global hstate @@ -2853,7 +2854,8 @@ void wait_for_freed_hugetlb_folios(void) * -ENOSPC if cgroup charging fails or no folio is available. */ struct folio *hugetlb_alloc_folio(struct hstate *h, - struct mempolicy_interpreted *mpoli, u8 alloc_flags) + struct mempolicy_interpreted *mpoli, struct mm_struct *mm, + u8 alloc_flags) { bool charge_hugetlb_cgroup_rsvd = alloc_flags & HUGETLB_ALLOC_CHARG_CGROUP_RSVD; @@ -2908,7 +2910,8 @@ struct folio *hugetlb_alloc_folio(struct hstate *h, spin_unlock_irq(&hugetlb_lock); - ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); + ret = mem_cgroup_charge_hugetlb(folio, mm, + gfp | __GFP_RETRY_MAYFAIL); /* * Unconditionally increment NR_HUGETLB here because if * mem_cgroup_charge_hugetlb failed, freeing the page will @@ -3051,7 +3054,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, .nodemask = nodemask, }; - folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags); + folio = hugetlb_alloc_folio(h, &mpoli, vma->vm_mm, alloc_flags); mpol_cond_put(mpol); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index bd1e7e154426..86ff580c7018 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5265,6 +5265,7 @@ int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp) /** * mem_cgroup_charge_hugetlb - charge the memcg for a hugetlb folio * @folio: folio being charged + * @mm: mm context of the allocation target * @gfp: reclaim mode * * This function is called when allocating a huge page folio, after the page has @@ -5274,9 +5275,10 @@ int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp) * Returns ENOMEM if the memcg is already full. * Returns 0 if either the charge was successful, or if we skip the charging. */ -int mem_cgroup_charge_hugetlb(struct folio *folio, gfp_t gfp) +int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, + gfp_t gfp) { - struct mem_cgroup *memcg = get_mem_cgroup_from_current(); + struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); int ret = 0; /* From 2492d9c0afc02cebe8318155829e7d03bea617b8 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 3 Sep 2026 13:35:34 +0800 Subject: [PATCH 235/539] mm/page-flags: define HWPoison test-and-change helpers unconditionally Page flag helpers for configuration-dependent flags provide false or no-op variants so that their users can be independent of the configuration. The HWPoison helpers do not fully follow this pattern. When CONFIG_MEMORY_FAILURE is enabled, PAGEFLAG() and TESTSCFLAG() provide the regular, test-and-set, and test-and-clear operations. When it is disabled, only PAGEFLAG_FALSE() is instantiated, leaving TestSetPageHWPoison() and TestClearPageHWPoison() undefined. Use TESTSCFLAG_FALSE() to provide the missing accessors when memory failure handling is disabled. Both accessors return false, which is consistent with HWPoison state being unavailable, and makes the accessor interface consistent across configurations. Also remove now-unneeded test_and_clear_pmem_poison() from nvdimm/pmem. Link: https://lore.kernel.org/20260903053535.17611-1-kaitao.cheng@linux.dev Link: https://lore.kernel.org/20260903053535.17611-2-kaitao.cheng@linux.dev Link: https://lore.kernel.org/20260903053535.17611-3-kaitao.cheng@linux.dev Signed-off-by: Kaitao Cheng Signed-off-by: Andrew Morton Acked-by: Muchun Song Acked-by: David Hildenbrand (Arm) Reviewed-by: Oscar Salvador Cc: Alison Schofield Cc: Dave Jiang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Cc: Ira Weiny --- drivers/nvdimm/pmem.c | 2 +- drivers/nvdimm/pmem.h | 12 ------------ include/linux/page-flags.h | 1 + 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 30a51c365ce8..5fb86595e8bd 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -80,7 +80,7 @@ static void pmem_mkpage_present(struct pmem_device *pmem, phys_addr_t offset, * here since we're in the driver I/O path and * outstanding I/O requests pin the dev_pagemap. */ - if (test_and_clear_pmem_poison(page)) + if (TestClearPageHWPoison(page)) clear_mce_nospec(pfn); } } diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h index a48509f90196..76870505dd79 100644 --- a/drivers/nvdimm/pmem.h +++ b/drivers/nvdimm/pmem.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NVDIMM_PMEM_H__ #define __NVDIMM_PMEM_H__ -#include #include #include #include @@ -31,15 +30,4 @@ long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, void **kaddr, unsigned long *pfn); -#ifdef CONFIG_MEMORY_FAILURE -static inline bool test_and_clear_pmem_poison(struct page *page) -{ - return TestClearPageHWPoison(page); -} -#else -static inline bool test_and_clear_pmem_poison(struct page *page) -{ - return false; -} -#endif #endif /* __NVDIMM_PMEM_H__ */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index ae2ebaed6d4d..3dc79c0c5adf 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -655,6 +655,7 @@ TESTSCFLAG(HWPoison, hwpoison, PF_ANY) #define __PG_HWPOISON (1UL << PG_hwpoison) #else PAGEFLAG_FALSE(HWPoison, hwpoison) +TESTSCFLAG_FALSE(HWPoison, hwpoison) #define __PG_HWPOISON 0 #endif From 610c4438fb66e06d9e693c0ad3b2c1be54c0d61f Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 18:07:19 -0700 Subject: [PATCH 236/539] mm/damon/core: error damos_commit_quota_goal() for zero target_value Patch series "mm/damon: move zero damos quota target_value handling to the core layer". Having zero DAMOS quota target value can cause division by zero. DAMON API callers are checking the target value parameters to avoid that. It is easy to make mistakes in some of the multiple API callers. Move that to the core layer. Patch 1 adds the corner case handling into the core layer DAMON parameters validation logic. Patches 2 and 3 remove no more needed DAMON API callers side handling of the corner case in DAMON_LRU_SORT and DMON_SAMPLIE_MTIER, respectively. This patch (of 3): If a DAMOS scheme has a damos_quota_goal of zero target_value, damos_quota_goal() could trigger division-by-zero error. Hence each DAMON API callers should do the zero target_value validation. It is easy to make mistakes. Actually such bugs in DAMON_LRU_SORT and DAMON_SAMPLE_MTIER were found and fixed [1]. It is better to handle the corner case only once in the core layer, instead of multiple places in all DAMON API callers. One straightforward option is using an alternative denominator for the corner case in the damos_quota_goal(). However, the zero target_value is meaningless. In this case, the quota goal is always evaluated as achieved or over-achieved. The quota will only keep being reduced. Simply avoid using zero target_value by adding a check in the core layer DAMOS quota goal parameters validation/commit path, damos_commit_quota_goal(). Update it to return an error in the case. Also update its caller to propagate the error. Link: https://lore.kernel.org/20260903010722.94244-1-sj@kernel.org Link: https://lore.kernel.org/20260903010722.94244-2-sj@kernel.org Link: https://lore.kernel.org/20260803134034.15217-1-sj@kernel.org [1] Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 87cc5d756893..15a773614935 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1211,14 +1211,17 @@ static void damos_commit_quota_goal_union( } } -static void damos_commit_quota_goal( +static int damos_commit_quota_goal( struct damos_quota_goal *dst, struct damos_quota_goal *src) { + if (!src->target_value) + return -EINVAL; dst->metric = src->metric; dst->target_value = src->target_value; if (dst->metric == DAMOS_QUOTA_USER_INPUT) dst->current_value = src->current_value; damos_commit_quota_goal_union(dst, src); + return 0; } /** @@ -1236,14 +1239,17 @@ static void damos_commit_quota_goal( int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src) { struct damos_quota_goal *dst_goal, *next, *src_goal, *new_goal; - int i = 0, j = 0; + int i = 0, j = 0, err; damos_for_each_quota_goal_safe(dst_goal, next, dst) { src_goal = damos_nth_quota_goal(i++, src); - if (src_goal) - damos_commit_quota_goal(dst_goal, src_goal); - else + if (src_goal) { + err = damos_commit_quota_goal(dst_goal, src_goal); + if (err) + return err; + } else { damos_destroy_quota_goal(dst_goal); + } } damos_for_each_quota_goal_safe(src_goal, next, src) { if (j++ < i) @@ -1252,7 +1258,11 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src) src_goal->metric, src_goal->target_value); if (!new_goal) return -ENOMEM; - damos_commit_quota_goal(new_goal, src_goal); + err = damos_commit_quota_goal(new_goal, src_goal); + if (err) { + damos_free_quota_goal(new_goal); + return err; + } damos_add_quota_goal(dst, new_goal); } return 0; From dd07497616f2bbddd251828f2f36304500b6eb00 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 18:07:20 -0700 Subject: [PATCH 237/539] Revert "mm/damon/lru_sort: error out for >10000 active_mem_bp" This reverts commit 06befa61c427e74319781e6f35a364cfc32dbae8. The commit was made to avoid zero damos quota goal target value, because it can trigger division-by-zero. Now the core layer handles the corner case. It returns an error for any attempt setting the aero target_value. The corner case handling in DAMON_LRU_SORT is hence no more needed. Remove it. Note that this slightly changes the user behavior. It still disallows active_mem_bp of 10,002. But now it allows other >10,000 active_mem_bp values. Setting >10,000 active_mem_bp makes not much sense. But it doesn't cause critical problems such as memory leak or crash, either. Arguably that doesn't deserve additional code complexity. Just allow it. Link: https://lore.kernel.org/20260903010722.94244-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/lru_sort.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index bd847829a990..7df45f9a0b3a 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -233,8 +233,6 @@ static int damon_lru_sort_add_quota_goals(struct damos *hot_scheme, if (!active_mem_bp) return 0; - if (10000 < active_mem_bp) - return -EINVAL; goal = damos_new_quota_goal(DAMOS_QUOTA_ACTIVE_MEM_BP, active_mem_bp); if (!goal) return -ENOMEM; From f305464862ec47d4e1fbc526354f61fc1e4d9e55 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 18:07:21 -0700 Subject: [PATCH 238/539] Revert "samples/damon/mtier: error out for zero quota goal target values" This reverts commit a16fd3ad9d89b05475864da97327870464611736. The commit was made to avoid zero damos quota goal target value, because it can trigger division-by-zero. Now the core layer handles the corner case. It returns an error for any attempt setting the aero target_value. The corner case handling in DAMON_SAMPLE_MTIER is hence no more needed. Remove it. Link: https://lore.kernel.org/20260903010722.94244-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- samples/damon/mtier.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index bea45c87cc9b..27dc88bdf7a0 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -161,9 +161,6 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) if (!scheme) goto free_out; damon_set_schemes(ctx, &scheme, 1); - /* zero target value causes division by zero in damos_quota_store() */ - if (!node0_mem_used_bp || !node0_mem_free_bp) - goto free_out; quota_goal = damos_new_quota_goal( promote ? DAMOS_QUOTA_NODE_MEM_USED_BP : DAMOS_QUOTA_NODE_MEM_FREE_BP, From 616e096d43822cc05e926b1595e293a9c49ac2ed Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 18:03:30 -0700 Subject: [PATCH 239/539] mm/damon/core: handle NULL ctx parameter in damon_call() Patch series "mm/damon: allow NULL or unstarted damon_ctx parameter for damon_call()". Callers of damon_call() should validate the damon_ctx object parameter. If it is NULL or never damon_start()-ed object, damon_call() could dereference the NULL pointer or indefinitely hang. Ensuring all callers doing the validation correctly has turned out to be difficult. Handle the corner cases inside the core layer and remove callers' validations. Patches 1 and 2 respectively allow passing NULL and not yet damon_start()-ed ctx parameter to damon_call(). Patches 3 and 4 remove the callers side validations in DMON_RECLAIM and DASMON_LRU_SORT, respectively. This patch (of 4): When NULL damon_ctx pointer parameter is passed, damon_call() could do NULL dereference. The caller is responsible to avoid that. It is easy to forget, and there are many damon_call() callers. Meanwhile, damon_call() is never meant to be performance critical. It uses mutex and completion. Add the NULL pointer check inside damon_call() so that callers can pass the parameter without NULL checks. Link: https://lore.kernel.org/20260903010334.93622-1-sj@kernel.org Link: https://lore.kernel.org/20260903010334.93622-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 15a773614935..a4f78d10ef4e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2216,6 +2216,8 @@ int damon_kdamond_pid(struct damon_ctx *ctx) */ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) { + if (!ctx) + return -EINVAL; if (!control->repeat) init_completion(&control->completion); control->canceled = false; From 1fecdf1d5637a10a33ccf669e8339db8172e6279 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 18:03:31 -0700 Subject: [PATCH 240/539] mm/damon/core: set ctx->call_controls_obsolete in damon_new_ctx() damon_ctx->call_controls_obsolete is used to disallow damon_call() requests when the request cannot be served. The field is unset and set when the context execution is started and terminated, respectively. The intention is to allow damon_call() requests only while the context is actively being executed. damon_ctx constructor, damon_new_ctx() unsets the field, though. As a result, passing the damon_ctx parameter that never successfully damon_start()-ed to damon_call() can indefinitely hang. The callers should ensure to avoid the case. Such parameter validation is not always simple. Actually such bugs in DAMON_RECLAIM and DAMON_LRU_SORT have been found and fixed [1]. Set the field in damon_new_ctx(), so that DAMON API callers can pass the context parameter to damon_call() without the additional check. Link: https://lore.kernel.org/20260903010334.93622-3-sj@kernel.org Link: https://lore.kernel.org/20260803134646.16640-1-sj@kernel.org [1] Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index a4f78d10ef4e..ea6df4311ceb 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -933,6 +933,7 @@ struct damon_ctx *damon_new_ctx(void) INIT_LIST_HEAD(&ctx->adaptive_targets); INIT_LIST_HEAD(&ctx->schemes); + ctx->call_controls_obsolete = true; prandom_seed_state(&ctx->rnd_state, get_random_u64()); return ctx; @@ -2206,10 +2207,6 @@ int damon_kdamond_pid(struct damon_ctx *ctx) * synchronization. The return value of the function will be saved in * &damon_call_control->return_code. * - * Note that this function should be called only after damon_start() with the - * @ctx has succeeded. Otherwise, this function could fall into an indefinite - * wait. - * * When this function is failed, the @ctx is guaranteed to be stopped. * * Return: 0 on success, negative error code otherwise. From f306b0b41db8c53cf081f9022a03fffdc9386d28 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 18:03:32 -0700 Subject: [PATCH 241/539] mm/damon/reclaim: remove unnecessary damon_call() param validation DAMON_RECLAIM avoids passing NULL or unstarted damon_ctx to damon_call() with its own validation. The validation is no longer needed, because the DAMON core layer now handles the corner cases itself. Remove the unnecessary check. Link: https://lore.kernel.org/20260903010334.93622-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/reclaim.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 45d5557cc575..42a2c9cb1343 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -271,8 +271,6 @@ static int damon_reclaim_commit_inputs_fn(void *arg) return damon_reclaim_apply_parameters(); } -static bool damon_reclaim_damon_has_started; - static int damon_reclaim_commit_inputs_store(const char *val, const struct kernel_param *kp) { @@ -293,10 +291,6 @@ static int damon_reclaim_commit_inputs_store(const char *val, if (!commit_inputs_request) return 0; - /* Skip damon_call() if ctx has not successfully started. */ - if (!damon_reclaim_damon_has_started) - return -EINVAL; - err = damon_call(ctx, &control); return err ? err : control.return_code; @@ -343,8 +337,6 @@ static int damon_reclaim_turn(bool on) err = damon_start(&ctx, 1, true); if (err) return err; - if (!damon_reclaim_damon_has_started) - damon_reclaim_damon_has_started = true; return damon_call(ctx, &call_control); } From 0e3846146f069b4afe5fa0f8adc6b70cfc9ff7e6 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 2 Sep 2026 18:03:33 -0700 Subject: [PATCH 242/539] mm/damon/lru_sort: remove unnecessary damon_call() param validation DAMON_LRU_SORT avoids passing NULL or unstarted damon_ctx to damon_call() with its own validation. The validation is no longer needed, because the DAMON core layer now handles the corner cases itself. Remove the unnecessary check. Link: https://lore.kernel.org/20260903010334.93622-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/lru_sort.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 7df45f9a0b3a..ad8e86dd3a93 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -344,8 +344,6 @@ static int damon_lru_sort_commit_inputs_fn(void *arg) return damon_lru_sort_apply_parameters(); } -static bool damon_lru_sort_damon_has_started; - static int damon_lru_sort_commit_inputs_store(const char *val, const struct kernel_param *kp) { @@ -366,10 +364,6 @@ static int damon_lru_sort_commit_inputs_store(const char *val, if (!commit_inputs_request) return 0; - /* Skip damon_call() if ctx has not successfully started. */ - if (!damon_lru_sort_damon_has_started) - return -EINVAL; - err = damon_call(ctx, &control); return err ? err : control.return_code; @@ -420,8 +414,6 @@ static int damon_lru_sort_turn(bool on) err = damon_start(&ctx, 1, true); if (err) return err; - if (!damon_lru_sort_damon_has_started) - damon_lru_sort_damon_has_started = true; return damon_call(ctx, &call_control); } From a85a8d03278fd294e819e2d719805f54225b76ee Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Wed, 2 Sep 2026 15:55:07 -0400 Subject: [PATCH 243/539] mm/memory_hotplug: factor out node_is_memoryless() A memoryless node neither spans present pages (populated or ZONE_DEVICE) nor has an offline-but-added memory block still linked to it in sysfs. try_offline_node() presently open-codes this memoryless check. Pull that into a node_is_memoryless() helper and pull the existing check_no_memblock_for_node_cb() helper ahead of the add/online path so it's clearer what is happening here. No functional change. [akpm@linux-foundation.org: move node_is_memoryless() inside CONFIG_MEMORY_HOTREMOVE] Link: https://lore.kernel.org/20260902195507.88655-1-gourry@gourry.net Signed-off-by: Gregory Price Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador --- mm/memory_hotplug.c | 61 ++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 226ab9cb078a..b428da66d279 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1785,6 +1785,37 @@ bool mhp_range_allowed(u64 start, u64 size, bool need_mapping) } #ifdef CONFIG_MEMORY_HOTREMOVE + +static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) +{ + int nid = *(int *)arg; + + /* + * If a memory block belongs to multiple nodes, the stored nid is not + * reliable. However, such blocks are always online (e.g., cannot get + * offlined) and, therefore, are still spanned by the node. + */ + return mem->nid == nid ? -EEXIST : 0; +} + +/* Caller must hold the memory hotplug lock for this check. */ +static bool node_is_memoryless(int nid) +{ + /* + * A node still spanning pages (especially ZONE_DEVICE) is not + * memoryless. A node spans memory after move_pfn_range_to_zone(), + * e.g. once a memory block has been onlined. + */ + if (node_spanned_pages(nid)) + return false; + /* + * Offline memory blocks may not be spanned by the node yet, but they + * link to it in sysfs and can be onlined later, so the node is not + * memoryless while any remain. + */ + return !for_each_memory_block(&nid, check_no_memblock_for_node_cb); +} + /* * Scan pfn range [start,end) to find movable/migratable pages (LRU and * hugetlb folio, movable_ops pages). Will skip over most unmovable @@ -2214,18 +2245,6 @@ static int check_cpu_on_node(int nid) return 0; } -static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) -{ - int nid = *(int *)arg; - - /* - * If a memory block belongs to multiple nodes, the stored nid is not - * reliable. However, such blocks are always online (e.g., cannot get - * offlined) and, therefore, are still spanned by the node. - */ - return mem->nid == nid ? -EEXIST : 0; -} - /** * try_offline_node * @nid: the node ID @@ -2237,23 +2256,7 @@ static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) */ void try_offline_node(int nid) { - int rc; - - /* - * If the node still spans pages (especially ZONE_DEVICE), don't - * offline it. A node spans memory after move_pfn_range_to_zone(), - * e.g., after the memory block was onlined. - */ - if (node_spanned_pages(nid)) - return; - - /* - * Especially offline memory blocks might not be spanned by the - * node. They will get spanned by the node once they get onlined. - * However, they link to the node in sysfs and can get onlined later. - */ - rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb); - if (rc) + if (!node_is_memoryless(nid)) return; if (check_cpu_on_node(nid)) From 9ea9dc1606be5bc73f636f43c4547953f4925416 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Sat, 5 Sep 2026 15:08:30 -0400 Subject: [PATCH 244/539] mm: remove PageWriteback The last caller of PageWriteback() was removed in commit f75987e543c2 ("libceph: remove pinning assertion in ceph_msg_data_iter_next()"). This flag is now only used on folios, so we can remove all the page accessors. folio_test_clear_writeback() is not used, so don't add FOLIO_TEST_CLEAR_FLAG() for it. Link: https://lore.kernel.org/20260905-remove-pagewriteback-v1-1-06f41c00db03@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: SJ Park Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/page-flags.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 3dc79c0c5adf..86dd0470da11 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -588,8 +588,8 @@ FOLIO_FLAG(owner_2, FOLIO_HEAD_PAGE) * Only test-and-set exist for PG_writeback. The unconditional operators are * risky: they bypass page accounting. */ -TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL) - TESTSCFLAG(Writeback, writeback, PF_NO_TAIL) +FOLIO_TEST_FLAG(writeback, FOLIO_HEAD_PAGE) + FOLIO_TEST_SET_FLAG(writeback, FOLIO_HEAD_PAGE) FOLIO_FLAG(mappedtodisk, FOLIO_HEAD_PAGE) /* PG_readahead is only used for reads; PG_reclaim is only for writes */ From 20bf009db40d8bec7896eca6854f5ac10c34e210 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Sun, 6 Sep 2026 00:51:06 +0800 Subject: [PATCH 245/539] mm/memcontrol: move the lru_zone_size sanity check to the reader side Patch series "mm/mglru: clean up folio counters and flag usage", v6. This is a cleanup series separated out from the MGLRU-FG series [1]. As that series is getting too long in following updates, separate out the clean up part for easier review and merge. No feature change is intended, except one bugfix. It mostly replaces the open-coded bit operations scattered throughout the MGLRU code with new helpers, with proper kdocs, sanity debug checks, and hardens a few MGLRU functions. A subtle generation counter leak is also found during the refactoring and the fix is included. Also collected review feedbacks on the cleanup part from the posted series. This patch (of 6): Instead of using an unsigned long and checking the counter value at the updater side, turn the counter into a signed long and check at the reader side. This reduces overhead and simplifies the code. commit ca707239e8a7 ("mm: update_lru_size warn and reset bad lru_size") added a sanity check for memcg counter underflow: lru_zone_size is unsigned, so an underflow wraps it around and returns an enormously large number, then the memcg shrinker loops almost forever as the calculated number of folios to shrink is huge. It also checked if a zero value matches the empty LRU list, so the positive and negative deltas had to be handled separately. However that emptiness check was already removed by commit b4536f0c829c ("mm, memcg: fix the active list aging for lowmem requests when memcg is enabled"), so handling the deltas separately is no longer needed. The remaining update-side check is costly and cannot really catch the leak it is after anyway. It runs on every LRU folio, and if a folio was removed without updating the counter while other folios remain on the LRU, the WARN only triggers much later, from a likely innocent callsite. While readers are much rarer than writers, only the reclaim and reparenting paths read it, once per batch. Checking at the reader side instead leaves the update path a plain addition, and puts the warning where the value is actually consumed. Note this changes the behavior on underflow: the correction is removed and a negative value is kept. A massive leak of the LRU size counter would indicate that something else has gone very wrong, and one should fix that leaking site instead. Besides, the original behavior might cause false positives, or make things worse if the accounting happens after the actual insertion: the value is not leaked, just delayed, so force-fixing it would cause a bigger problem. The warning now only kicks in when a consumer actually uses it, in which case the reader gets zero. Link: https://lore.kernel.org/20260906-mglru-flags-cleanup-v6-0-9aacbd77d4ca@tencent.com Link: https://lore.kernel.org/20260906-mglru-flags-cleanup-v6-1-9aacbd77d4ca@tencent.com Link: https://lore.kernel.org/linux-mm/20260804-mglru-fg-v1-0-4d8dad39dad6@tencent.com/ [1] Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Ridong Chen Reviewed-by: Barry Song Acked-by: Shakeel Butt Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao Cc: Zi Yan Cc: Lian Wang Cc: Qi Zheng --- include/linux/memcontrol.h | 9 +++++++-- mm/memcontrol.c | 18 +----------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 058ebd73ff16..a03b6e3e5470 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -100,7 +100,7 @@ struct mem_cgroup_per_node { /* Fields which get updated often at the end. */ struct lruvec lruvec; CACHELINE_PADDING(_pad2_); - unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; + long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter; /* @@ -888,10 +888,15 @@ static inline unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx) { + long val; struct mem_cgroup_per_node *mz; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - return READ_ONCE(mz->lru_zone_size[zone_idx][lru]); + val = READ_ONCE(mz->lru_zone_size[zone_idx][lru]); + if (WARN_ON_ONCE(val < 0)) + return 0; + + return val; } void __mem_cgroup_handle_over_high(gfp_t gfp_mask); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 86ff580c7018..4cb2db8c0923 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1529,28 +1529,12 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages) { struct mem_cgroup_per_node *mz; - unsigned long *lru_size; - long size; if (mem_cgroup_disabled()) return; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - lru_size = &mz->lru_zone_size[zid][lru]; - - if (nr_pages < 0) - *lru_size += nr_pages; - - size = *lru_size; - if (WARN_ONCE(size < 0, - "%s(%p, %d, %ld): lru_size %ld\n", - __func__, lruvec, lru, nr_pages, size)) { - VM_BUG_ON(1); - *lru_size = 0; - } - - if (nr_pages > 0) - *lru_size += nr_pages; + mz->lru_zone_size[zid][lru] += nr_pages; } /** From 64169ceea5b1e36ae3478598d4462bf7e23876d0 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Sun, 6 Sep 2026 00:51:07 +0800 Subject: [PATCH 246/539] mm/mglru: introduce helpers for manipulating gen and refs flags Instead of doing bit ops on folio->flags.f, introduce helpers for adjusting a folio's refs and generation info, making the code easier to debug and understand. No functional change is intended: some combined atomic operations are split into two, which only creates harmless transient states. There is no measurable performance impact, and some paths even look slightly better in the generated assembly. Link: https://lore.kernel.org/20260906-mglru-flags-cleanup-v6-2-9aacbd77d4ca@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Reviewed-by: Baolin Wang Reviewed-by: Barry Song Cc: Axel Rasmussen Cc: Baoquan He Cc: Chris Li Cc: David Hildenbrand (Arm) Cc: Johannes Weiner Cc: Kairui Song Cc: Liam R. Howlett Cc: Lian Wang Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Ridong Chen Cc: Roman Gushchin Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao Cc: Zi Yan --- include/linux/mm_inline.h | 84 +++++++++++++++++++++++++++++++++++---- include/linux/mmzone.h | 1 + mm/folio.c | 19 +++++---- mm/vmscan.c | 60 ++++++++++++++++------------ 4 files changed, 122 insertions(+), 42 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 621c8653d8f7..3f4bd5b02b54 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -142,10 +142,66 @@ static inline int lru_tier_from_refs(int refs, bool workingset) return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs); } -static inline int folio_lru_refs(const struct folio *folio) +/** + * lru_set_gen_flags - Set the LRU generation number to specified folio flags. + * @flags: pointer to the folio flags + * @gen: generation number, between 0 and (MAX_NR_GENS - 1), inclusive. + */ +static inline void lru_set_gen_flags(unsigned long *flags, int gen) { - unsigned long flags = READ_ONCE(folio->flags.f); + BUILD_BUG_ON(LRU_GEN_MASK & LRU_REFS_MASK); + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS || gen < 0); + /* Store gen offset by 1, zero means the folio is off-list. */ + *flags &= ~LRU_GEN_MASK; + *flags |= (gen + 1UL) << LRU_GEN_PGOFF; +} +/** + * lru_get_gen_flags - Return the LRU generation number from folio flags. + * @flags: folio flags + * + * Returns: A number between 0 and (MAX_NR_GENS - 1), inclusive. Returns + * -1 if the flags indicate the folio is off the list (e.g., isolated). + */ +static inline int lru_get_gen_flags(unsigned long flags) +{ + int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + + /* Exclude the legal -1 from the unsigned MAX_NR_GENS comparison */ + VM_WARN_ON_ONCE(gen != -1 && gen >= MAX_NR_GENS); + return gen; +} + +/** + * lru_set_refs_flags - Set the LRU referenced count to folio flags. + * @flags: pointer to the folio flags + * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive. + * + * For MGLRU, PG_referenced holds the first ref, and the extra bits hold the + * remaining refs. For classical LRU the extra bits are not used, so it can + * also be seen as the refs count never exceeds 1. In both cases, refs == 1 + * means PG_referenced is set and the extra bits are zero, and refs == 0 means + * PG_referenced and the extra bits are all unset. + */ +static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs) +{ + VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); + BUILD_BUG_ON(LRU_REFS_MAX != (LRU_REFS_MASK >> LRU_REFS_PGOFF) + 1); + + *flags &= ~LRU_REFS_FLAGS; + if (!refs) + return; + *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF)); +} + +/** + * lru_get_refs_flags - Return LRU referenced / access count from folio flags. + * @flags: folio flags + * + * Reads the LRU referenced count set by lru_set_refs_flags(). + */ +static inline int lru_get_refs_flags(unsigned long flags) +{ if (!(flags & BIT(PG_referenced))) return 0; /* @@ -155,11 +211,24 @@ static inline int folio_lru_refs(const struct folio *folio) return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1; } +static inline int folio_lru_refs(const struct folio *folio) +{ + return lru_get_refs_flags(READ_ONCE(*const_folio_flags(folio, 0))); +} + +static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs) +{ + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + + do { + new_flags = old_flags; + lru_set_refs_flags(&new_flags, refs); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); +} + static inline int folio_lru_gen(const struct folio *folio) { - unsigned long flags = READ_ONCE(folio->flags.f); - - return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0))); } static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen) @@ -270,7 +339,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, gen = lru_gen_from_seq(seq); flags = (gen + 1UL) << LRU_GEN_PGOFF; /* see the comment on MIN_NR_GENS about PG_active */ - set_mask_bits(&folio->flags.f, LRU_GEN_MASK | BIT(PG_active), flags); + set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags); lru_gen_update_size(lruvec, folio, -1, gen); /* for folio_rotate_reclaimable() */ @@ -295,7 +364,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, /* for folio_migrate_flags() */ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; - flags = set_mask_bits(&folio->flags.f, LRU_GEN_MASK, flags); + flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, flags); gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; lru_gen_update_size(lruvec, folio, gen, -1); @@ -339,7 +408,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, static inline void folio_migrate_refs(struct folio *new, const struct folio *old) { - } #endif /* CONFIG_LRU_GEN */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index c070b867e2f3..593cc92fe471 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -500,6 +500,7 @@ enum lruvec_flags { #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) +#define LRU_REFS_MAX BIT(LRU_REFS_WIDTH) /* * For folios accessed multiple times through file descriptors, diff --git a/mm/folio.c b/mm/folio.c index 50a6dbe55998..47a437e0f7fd 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -354,26 +354,28 @@ static void __lru_cache_activate_folio(struct folio *folio) static void lru_gen_inc_refs(struct folio *folio) { - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + int refs; if (folio_test_unevictable(folio)) return; /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio)) { - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); + if (!folio_lru_refs(folio)) { + folio_set_lru_refs(folio, 1); return; } do { - if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) { + new_flags = old_flags; + refs = lru_get_refs_flags(old_flags); + if (refs == LRU_REFS_MAX) { if (!folio_test_workingset(folio)) folio_set_workingset(folio); return; } - - new_flags = old_flags + BIT(LRU_REFS_PGOFF); - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); + lru_set_refs_flags(&new_flags, refs + 1); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); } static bool lru_gen_clear_refs(struct folio *folio) @@ -385,7 +387,8 @@ static bool lru_gen_clear_refs(struct folio *folio) if (gen < 0) return true; - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS | BIT(PG_workingset), 0); + folio_set_lru_refs(folio, 0); + folio_clear_workingset(folio); rcu_read_lock(); seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]); diff --git a/mm/vmscan.c b/mm/vmscan.c index 245f68c75b28..c6e03a5047ae 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -863,19 +863,22 @@ static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { /* Activate file-backed executable folios after first usage. */ if (is_exec_file_folio(folio, vma_flags)) { - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); + folio_set_workingset(folio); + folio_set_lru_refs(folio, 0); return true; } - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); + folio_set_lru_refs(folio, 1); return false; } /* Promote on second access */ - if (folio_lru_refs(folio) > 1) - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); - else + if (folio_lru_refs(folio) > 1) { + folio_set_workingset(folio); + folio_set_lru_refs(folio, 0); + } else { folio_mark_accessed(folio); + } return true; } #else @@ -3291,11 +3294,10 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags) +static int folio_update_gen(struct folio *folio, int new_gen, const vma_flags_t *vma_flags) { - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); - - VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + int old_gen; /* * See the comment on LRU_REFS_FLAGS, and activate file-backed @@ -3304,31 +3306,34 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma */ if (!folio_test_referenced(folio) && !folio_test_workingset(folio) && !is_exec_file_folio(folio, vma_flags)) { - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); + folio_set_lru_refs(folio, 1); return -1; } do { + old_gen = lru_get_gen_flags(old_flags); + new_flags = old_flags; + /* lru_gen_del_folio() has isolated this page? */ - if (!(old_flags & LRU_GEN_MASK)) - return -1; + if (old_gen < 0) + break; - new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); - new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset); - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); + lru_set_gen_flags(&new_flags, new_gen); + lru_set_refs_flags(&new_flags, 0); + new_flags |= BIT(PG_workingset); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); - return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + return old_gen; } static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) { - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); int new_gen; - VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); - do { - new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + new_gen = lru_get_gen_flags(old_flags); + /* folio_update_gen() has promoted this page? */ if (new_gen >= 0 && new_gen != old_gen) { if (increased) @@ -3336,11 +3341,12 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) return new_gen; } + new_flags = old_flags; new_gen = (old_gen + 1) % MAX_NR_GENS; - new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); - new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); + lru_set_gen_flags(&new_flags, new_gen); + lru_set_refs_flags(&new_flags, 0); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); if (increased) *increased = true; @@ -4785,7 +4791,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca /* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced(folio)) - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); + folio_set_lru_refs(folio, 0); success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); @@ -5017,8 +5023,10 @@ retry: } /* don't add rejected folios to the oldest generation */ - if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active)); + if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) { + folio_set_lru_refs(folio, 0); + folio_set_active(folio); + } } move_folios_to_lru(&list); From d982ba0b970e588695461d969cadfcd2d28bea5e Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Sun, 6 Sep 2026 00:51:08 +0800 Subject: [PATCH 247/539] mm/migrate: copy all referenced state via folio_migrate_lru_refs folio_migrate_flags() copies PG_referenced separately from the MGLRU refs counter, which folio_migrate_refs() transfers. Yet under MGLRU, PG_referenced and the refs counter bits together describe the referenced status of a folio. Consolidate the two: rename folio_migrate_refs() to folio_migrate_lru_refs() and let it copy the complete referenced status, i.e., the MGLRU refs count including PG_referenced, or just PG_referenced for the active/inactive LRU. Drop the open-coded PG_referenced copy so the referenced status is transferred in one place. No behavior change is intended: under the active/inactive LRU the extra bits are unused, so operating on them is a noop. Transfer the reference state first, before the destination folio is marked up to date, as a best effort to retain referenced status. Link: https://lore.kernel.org/20260906-mglru-flags-cleanup-v6-3-9aacbd77d4ca@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Reviewed-by: Baolin Wang Acked-by: David Hildenbrand (Arm) Reviewed-by: Lian Wang Reviewed-by: Barry Song Reviewed-by: Ridong Chen Cc: Axel Rasmussen Cc: Chris Li Cc: Johannes Weiner Cc: Kairui Song Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Qi Zheng Cc: Roman Gushchin Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao Cc: Zi Yan --- include/linux/mm_inline.h | 20 +++++++++++++++----- mm/migrate.c | 6 +++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 3f4bd5b02b54..047295ae6e8a 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -373,11 +373,19 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, return true; } -static inline void folio_migrate_refs(struct folio *new, const struct folio *old) +/** + * folio_migrate_lru_refs - copy the reference state to a new folio + * @new: the destination folio + * @old: the source folio + * + * Transfer the reference state to @new during migration: the MGLRU + * refs count, including PG_referenced, or just PG_referenced for the + * active/inactive LRU. + */ +static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { - unsigned long refs = READ_ONCE(old->flags.f) & LRU_REFS_MASK; - - set_mask_bits(&new->flags.f, LRU_REFS_MASK, refs); + BUILD_BUG_ON(LRU_REFS_MASK & BIT(PG_referenced)); + folio_set_lru_refs(new, folio_lru_refs(old)); } #else /* !CONFIG_LRU_GEN */ @@ -406,8 +414,10 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, return false; } -static inline void folio_migrate_refs(struct folio *new, const struct folio *old) +static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { + if (folio_test_referenced(old)) + folio_set_referenced(new); } #endif /* CONFIG_LRU_GEN */ diff --git a/mm/migrate.c b/mm/migrate.c index 15b45832bcfa..a369d0c95c38 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -776,8 +776,9 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) { int cpupid; - if (folio_test_referenced(folio)) - folio_set_referenced(newfolio); + /* Copy the reference state, including PG_referenced */ + folio_migrate_lru_refs(newfolio, folio); + if (folio_test_uptodate(folio)) folio_mark_uptodate(newfolio); if (folio_test_clear_active(folio)) { @@ -807,7 +808,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) if (folio_test_idle(folio)) folio_set_idle(newfolio); - folio_migrate_refs(newfolio, folio); /* * Copy NUMA information to the new page, to prevent over-eager * future migrations of this same page. From d896c1a8379bbd8c957649466a6368c73b5514db Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Sun, 6 Sep 2026 00:51:09 +0800 Subject: [PATCH 248/539] mm/mglru: move max_seq read into walk_update_folio walk_pte_range(), walk_pmd_range_locked(), and lru_gen_look_around() each read lrugen->max_seq to compute the target generation used by walk_update_folio(), then pass it as a parameter. Move the read into walk_update_folio() itself so the callers no longer need to compute or pass the value. The max_seq read now happens once per folio update rather than once per walk range, so folios always get promoted to the current youngest generation. Link: https://lore.kernel.org/20260906-mglru-flags-cleanup-v6-4-9aacbd77d4ca@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Reviewed-by: Baolin Wang Reviewed-by: Ridong Chen Reviewed-by: Lian Wang Reviewed-by: Barry Song Cc: Axel Rasmussen Cc: Chris Li Cc: David Hildenbrand (Arm) Cc: Johannes Weiner Cc: Kairui Song Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Qi Zheng Cc: Roman Gushchin Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao Cc: Zi Yan --- mm/vmscan.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index c6e03a5047ae..4732b987aa67 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3557,13 +3557,15 @@ static bool suitable_to_scan(int total, int young) } static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, - struct folio *folio, int new_gen, bool dirty) + struct lruvec *lruvec, struct folio *folio, bool dirty) { - int old_gen; + int new_gen, old_gen; if (!folio) return; + new_gen = lru_gen_from_seq(READ_ONCE(lruvec->lrugen.max_seq)); + if (dirty && !folio_test_dirty(folio) && !(folio_test_anon(folio) && folio_test_swapbacked(folio) && !folio_test_swapcache(folio))) @@ -3594,8 +3596,6 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, struct lru_gen_mm_walk *walk = args->private; struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); - DEFINE_MAX_SEQ(walk->lruvec); - int gen = lru_gen_from_seq(max_seq); unsigned int nr; pmd_t pmdval; @@ -3646,7 +3646,7 @@ restart: continue; if (last != folio) { - walk_update_folio(walk, args->vma, last, gen, dirty); + walk_update_folio(walk, args->vma, walk->lruvec, last, dirty); last = folio; dirty = false; @@ -3659,7 +3659,7 @@ restart: walk->mm_stats[MM_LEAF_YOUNG] += nr; } - walk_update_folio(walk, args->vma, last, gen, dirty); + walk_update_folio(walk, args->vma, walk->lruvec, last, dirty); last = NULL; if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) @@ -3682,8 +3682,6 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area struct lru_gen_mm_walk *walk = args->private; struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); - DEFINE_MAX_SEQ(walk->lruvec); - int gen = lru_gen_from_seq(max_seq); VM_WARN_ON_ONCE(pud_leaf(*pud)); @@ -3737,7 +3735,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area goto next; if (last != folio) { - walk_update_folio(walk, vma, last, gen, dirty); + walk_update_folio(walk, vma, walk->lruvec, last, dirty); last = folio; dirty = false; @@ -3751,7 +3749,7 @@ next: i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; } while (i <= MIN_LRU_BATCH); - walk_update_folio(walk, vma, last, gen, dirty); + walk_update_folio(walk, vma, walk->lruvec, last, dirty); lazy_mmu_mode_disable(); spin_unlock(ptl); @@ -4357,8 +4355,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) struct pglist_data *pgdat = folio_pgdat(folio); struct lruvec *lruvec; struct lru_gen_mm_state *mm_state; - unsigned long max_seq; - int gen; lockdep_assert_held(pvmw->ptl); VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); @@ -4395,8 +4391,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) memcg = get_mem_cgroup_from_folio(folio); lruvec = mem_cgroup_lruvec(memcg, pgdat); - max_seq = READ_ONCE((lruvec)->lrugen.max_seq); - gen = lru_gen_from_seq(max_seq); mm_state = get_mm_state(lruvec); lazy_mmu_mode_enable(); @@ -4428,7 +4422,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (last != folio) { - walk_update_folio(walk, vma, last, gen, dirty); + walk_update_folio(walk, vma, lruvec, last, dirty); last = folio; dirty = false; @@ -4440,13 +4434,14 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) young += nr; } - walk_update_folio(walk, vma, last, gen, dirty); + walk_update_folio(walk, vma, lruvec, last, dirty); lazy_mmu_mode_disable(); /* feedback from rmap walkers to page table walkers */ if (mm_state && suitable_to_scan(i, young)) - update_bloom_filter(mm_state, max_seq, pvmw->pmd); + update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq), + pvmw->pmd); mem_cgroup_put(memcg); From c1d0e799321c817a22eab2abf2485250cffad7b4 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Sun, 6 Sep 2026 00:51:10 +0800 Subject: [PATCH 249/539] mm/mglru: use explicit tier range in read_ctrl_pos() read_ctrl_pos() encodes the tier range in a single "tier" parameter via "tier % MAX_NR_TIERS" as the start and "min(tier, MAX_NR_TIERS-1)" as the end. This is hard to follow, maintain, or extend. Tier values 0..3 select a single tier, while tier == MAX_NR_TIERS selects the full range. Replace it with explicit (tier_min, tier_max) parameters using a closed [tier_min, tier_max] interval, and add LRU_TIER_MIN and LRU_TIER_MAX for the tier bounds. The call sites now become self-documenting: - get_tier_idx: (LRU_TIER_MIN, LRU_TIER_MIN) for the first tier, (tier, tier) for each subsequent tier - get_type_to_scan: (LRU_TIER_MIN, LRU_TIER_MAX) for the full range No functional change. Link: https://lore.kernel.org/20260906-mglru-flags-cleanup-v6-5-9aacbd77d4ca@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Reviewed-by: Baoquan He Reviewed-by: Barry Song Reviewed-by: Ridong Chen Cc: Axel Rasmussen Cc: Chris Li Cc: David Hildenbrand (Arm) Cc: Johannes Weiner Cc: Kairui Song Cc: Liam R. Howlett Cc: Lian Wang Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Qi Zheng Cc: Roman Gushchin Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao Cc: Zi Yan --- include/linux/mmzone.h | 2 ++ mm/vmscan.c | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 593cc92fe471..acd94cecc0d3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -495,6 +495,8 @@ enum lruvec_flags { * folio->flags, masked by LRU_REFS_MASK. */ #define MAX_NR_TIERS 4U +#define LRU_TIER_MIN 0U +#define LRU_TIER_MAX (MAX_NR_TIERS - 1) #ifndef __GENERATING_BOUNDS_H diff --git a/mm/vmscan.c b/mm/vmscan.c index 4732b987aa67..ec5832b0d867 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3223,8 +3223,8 @@ struct ctrl_pos { int gain; }; -static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, - struct ctrl_pos *pos) +static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier_min, + int tier_max, int gain, struct ctrl_pos *pos) { int i; struct lru_gen_folio *lrugen = &lruvec->lrugen; @@ -3233,7 +3233,7 @@ static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, pos->gain = gain; pos->refaulted = pos->total = 0; - for (i = tier % MAX_NR_TIERS; i <= min(tier, MAX_NR_TIERS - 1); i++) { + for (i = tier_min; i <= tier_max; i++) { pos->refaulted += lrugen->avg_refaulted[type][i] + atomic_long_read(&lrugen->refaulted[hist][type][i]); pos->total += lrugen->avg_total[type][i] + @@ -4879,9 +4879,9 @@ static int get_tier_idx(struct lruvec *lruvec, int type) * This value is chosen because any other tier would have at least twice * as many refaults as the first tier. */ - read_ctrl_pos(lruvec, type, 0, 2, &sp); - for (tier = 1; tier < MAX_NR_TIERS; tier++) { - read_ctrl_pos(lruvec, type, tier, 3, &pv); + read_ctrl_pos(lruvec, type, LRU_TIER_MIN, LRU_TIER_MIN, 2, &sp); + for (tier = LRU_TIER_MIN + 1; tier <= LRU_TIER_MAX; tier++) { + read_ctrl_pos(lruvec, type, tier, tier, 3, &pv); if (!positive_ctrl_err(&sp, &pv)) break; } @@ -4902,8 +4902,10 @@ static int get_type_to_scan(struct lruvec *lruvec, int swappiness) * Compare the sum of all tiers of anon with that of file to determine * which type to scan. */ - read_ctrl_pos(lruvec, LRU_GEN_ANON, MAX_NR_TIERS, swappiness, &sp); - read_ctrl_pos(lruvec, LRU_GEN_FILE, MAX_NR_TIERS, MAX_SWAPPINESS - swappiness, &pv); + read_ctrl_pos(lruvec, LRU_GEN_ANON, LRU_TIER_MIN, LRU_TIER_MAX, + swappiness, &sp); + read_ctrl_pos(lruvec, LRU_GEN_FILE, LRU_TIER_MIN, LRU_TIER_MAX, + MAX_SWAPPINESS - swappiness, &pv); return positive_ctrl_err(&sp, &pv); } From 766f584cb6270dadce62c40c0cd4327aff7d531a Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Sun, 6 Sep 2026 00:51:11 +0800 Subject: [PATCH 250/539] mm/mglru: fix potential generation folio number leak Each generation of MGLRU accounts anon and file folio numbers separately, and the page table walker updates each generation's counters in batch once the walk is done. The walker promotes a folio's generation with a cmpxchg on folio->flags, and update_batch_size() then reads the live flags again to pick the anon/file column to charge. The walk holds neither the lruvec lock nor the folio lock, so the type can flip between the cmpxchg and that read: the lazyfree path clears PG_swapbacked, and reclaim sets it back on a dirty lazyfree folio. The batched delta pair is then recorded in the wrong type column. Nothing reconciles it afterwards, permanently skewing lrugen->nr_pages and the reclaim budgets derived from it. Fix it by capturing the type from the flags snapshot the cmpxchg linearized against: folio_update_gen() returns the type of the state it transitioned from, and update_batch_size() accounts with that. A folio's type only changes while it is off the LRU list, inside a del/add pair under the lruvec lock, with the gen bits cleared in between. The generation and PG_swapbacked sit in the same folio->flags word, so the cmpxchg snapshot captures them together. Let G be the generation that snapshot captured (old_gen) and G' the one it wrote (new_gen); the CAS can land in only three places: - before the del: the folio is anon at G; the batch records anon G -> G', and the del later removes the folio from the anon counters; - between del and add: gen == -1, so folio_update_gen() returns -1 without touching the flags and no batch is recorded; the del/add pair accounts for the move alone; - after the add: the folio is file at the fresh generation the add charged; the batch records file, that gen -> G', matching that charge. Unlike the drift of lazy promotions, which sort_folio() repairs under the lruvec lock, the phantom deltas from before this fix land in a column the folio never occupies again, so nothing ever repairs them. Link: https://lore.kernel.org/20260906-mglru-flags-cleanup-v6-6-9aacbd77d4ca@tencent.com Fixes: bd74fdaea146 ("mm: multi-gen LRU: support page table walks") Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Barry Song Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Chris Li Cc: David Hildenbrand (Arm) Cc: Johannes Weiner Cc: Kairui Song Cc: Liam R. Howlett Cc: Lian Wang Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Qi Zheng Cc: Ridong Chen Cc: Roman Gushchin Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao Cc: Zi Yan --- include/linux/mm_inline.h | 11 +++++++++++ mm/vmscan.c | 13 +++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 047295ae6e8a..ab69b9930893 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -30,6 +30,17 @@ static inline int folio_is_file_lru(const struct folio *folio) return !folio_test_swapbacked(folio); } +/** + * folio_flags_is_file_lru - Should the folio be on a file LRU or anon LRU? + * @flags: The folio's flags. + * + * Just like folio_is_file_lru but take the folio flags directly instead. + */ +static inline int folio_flags_is_file_lru(const unsigned long *flags) +{ + return !test_bit(PG_swapbacked, flags); +} + static __always_inline void __update_lru_size(struct lruvec *lruvec, enum lru_list lru, enum zone_type zid, long nr_pages) diff --git a/mm/vmscan.c b/mm/vmscan.c index ec5832b0d867..40d3f1b48a74 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3294,7 +3294,8 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int new_gen, const vma_flags_t *vma_flags) +static int folio_update_gen(struct folio *folio, int new_gen, int *type, + const vma_flags_t *vma_flags) { unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); int old_gen; @@ -3323,6 +3324,7 @@ static int folio_update_gen(struct folio *folio, int new_gen, const vma_flags_t new_flags |= BIT(PG_workingset); } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); + *type = folio_flags_is_file_lru(&old_flags); return old_gen; } @@ -3368,9 +3370,8 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) } static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, - int old_gen, int new_gen) + int old_gen, int new_gen, int type) { - int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); @@ -3559,7 +3560,7 @@ static bool suitable_to_scan(int total, int young) static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, struct lruvec *lruvec, struct folio *folio, bool dirty) { - int new_gen, old_gen; + int new_gen, old_gen, type; if (!folio) return; @@ -3572,9 +3573,9 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struc folio_mark_dirty(folio); if (walk) { - old_gen = folio_update_gen(folio, new_gen, &vma->flags); + old_gen = folio_update_gen(folio, new_gen, &type, &vma->flags); if (old_gen >= 0 && old_gen != new_gen) - update_batch_size(walk, folio, old_gen, new_gen); + update_batch_size(walk, folio, old_gen, new_gen, type); } else if (lru_gen_set_refs(folio, &vma->flags)) { old_gen = folio_lru_gen(folio); if (old_gen >= 0 && old_gen != new_gen) From ca6e32f3f4240d0335775a11afc21e0451dd8d8f Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 16 Sep 2026 10:34:46 +0200 Subject: [PATCH 251/539] mm/vmscan: avoid false-positive -Wuninitialized warning, again I previously worked around a false-postive gcc-16 warning in the get_tier_idx() function, by adding a fake initializer. This happens with the -fsanitize=bounds sanitizer when the compiler creates a specialized variant of isolate_folios(): In function 'get_tier_idx', inlined from 'isolate_folios.constprop' at mm/vmscan.c:4982:9: mm/vmscan.c:4934:9: error: 'sp.refaulted' is used uninitialized [-Werror=uninitialized] 4934 | read_ctrl_pos(lruvec, type, LRU_TIER_MIN, LRU_TIER_MIN, 2, &sp); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ mm/vmscan.c: In function 'isolate_folios.constprop': mm/vmscan.c:4946:25: note: 'sp.refaulted' was declared here 4946 | struct ctrl_pos sp, pv = {}; | ^~ Adding another "= {}" would solve the problem as well, but to prevent this from happening again after the next code refactoring, try instead to prevent this by forbidding interprocedural optimizations on this function. Link: https://lore.kernel.org/all/20260213123902.3466040-1-arnd@kernel.org/ Link: https://lore.kernel.org/20260916083456.4136132-1-arnd@kernel.org Fixes: 3de705a43a46 ("mm/vmscan: avoid false-positive -Wuninitialized warning") Signed-off-by: Arnd Bergmann Signed-off-by: Andrew Morton Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Cc: --- mm/vmscan.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 40d3f1b48a74..4059d1300789 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3223,8 +3223,11 @@ struct ctrl_pos { int gain; }; -static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier_min, - int tier_max, int gain, struct ctrl_pos *pos) +/* + * __noipa works around gcc-16 warning for uninitialized use of pos->refaulted + */ +static void __noipa read_ctrl_pos(struct lruvec *lruvec, int type, int tier_min, + int tier_max, int gain, struct ctrl_pos *pos) { int i; struct lru_gen_folio *lrugen = &lruvec->lrugen; @@ -4873,7 +4876,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec, static int get_tier_idx(struct lruvec *lruvec, int type) { int tier; - struct ctrl_pos sp, pv = {}; + struct ctrl_pos sp, pv; /* * To leave a margin for fluctuations, use a larger gain factor (2:3). @@ -4892,7 +4895,7 @@ static int get_tier_idx(struct lruvec *lruvec, int type) static int get_type_to_scan(struct lruvec *lruvec, int swappiness) { - struct ctrl_pos sp, pv = {}; + struct ctrl_pos sp, pv; if (swappiness <= MIN_SWAPPINESS + 1) return LRU_GEN_FILE; From ae0298a5fbeff32951c0d31398ccec97ec6aa0ea Mon Sep 17 00:00:00 2001 From: Luxiao Xu Date: Thu, 10 Sep 2026 11:42:50 +0800 Subject: [PATCH 252/539] mm/memory: constrain generic_access_phys() to page boundary This patch addresses an issue in generic_access_phys() where accessing memory across page boundaries in PFNMAP VMAs assumes physical pages are contiguous, which can lead to accessing unintended physical memory or exceeding VMA boundaries. generic_access_phys() improperly validates the memory access range: it only validates the start address using follow_pfnmap_start() and passes PAGE_ALIGN(len + offset) to ioremap_prot(). This poses two problems: 1. In PFNMAP VMAs, consecutive virtual pages are not guaranteed to be physically contiguous, and individual PTEs may have different access permissions or writability. 2. The mapping may cross VMA boundaries if len extends beyond vma->vm_end. Constrain the access in generic_access_phys() to at most the current page boundary (PAGE_SIZE - offset) and map only a single PAGE_SIZE via ioremap_prot(). Since the caller __access_remote_vm() already loops over the requested length and handles partial transfers, it will naturally iterate over the remaining pages. Also add a missing (resource_size_t) cast during PFN re-validation to avoid truncation on 32-bit PAE systems. Link: https://lore.kernel.org/e06e28a46c2a176238f03b5740df0913e57c2861.1788842306.git.rakukuip@gmail.com Fixes: 9cb12d7b4cca ("mm/memory.c: actually remap enough memory") Signed-off-by: Luxiao Xu Signed-off-by: Ren Wei Signed-off-by: Andrew Morton Reported-by: Vega Suggested-by: David Hildenbrand Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Grazvydas Ignotas Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: --- mm/memory.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index ec63dd6212ac..926276d41920 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -7131,6 +7131,12 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, bool writable; struct follow_pfnmap_args args = { .vma = vma, .address = addr }; + /* + * Limit access to one page at a time, as that's what follow_pfnmap_start() + * guarantees; expect the caller to retry to read larger ranges. + */ + len = min_t(int, len, PAGE_SIZE - offset); + retry: if (follow_pfnmap_start(&args)) return -EINVAL; @@ -7142,7 +7148,7 @@ retry: if ((write & FOLL_WRITE) && !writable) return -EINVAL; - maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot); + maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); if (!maddr) return -ENOMEM; @@ -7150,7 +7156,7 @@ retry: goto out_unmap; if ((pgprot_val(prot) != pgprot_val(args.pgprot)) || - (phys_addr != (args.pfn << PAGE_SHIFT)) || + (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) || (writable != args.writable)) { follow_pfnmap_end(&args); iounmap(maddr); From 1e1ef39523fc4e6be02ac47bb154042a077acb44 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 10:40:34 +0200 Subject: [PATCH 253/539] docs/mm: ksm: use the renamed ksm structure names The Reference section of ksm.rst asks mm/ksm.c for mm_slot, stable_node and rmap_item. Commit 21fbd59136e0 ("ksm: add the ksm prefix to the names of the ksm private structures") renamed them to ksm_mm_slot, ksm_stable_node and ksm_rmap_item. Since then only ksm_scan is rendered and the three structures are missing from the generated documentation. Use the current names. Link: https://lore.kernel.org/20260905084034.39521-1-kmehltretter@gmail.com Fixes: 21fbd59136e0 ("ksm: add the ksm prefix to the names of the ksm private structures") Signed-off-by: Karl Mehltretter Signed-off-by: Andrew Morton Reviewed-by: Randy Dunlap Acked-by: David Hildenbrand (Arm) Assisted-by: LLM --- Documentation/mm/ksm.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/mm/ksm.rst b/Documentation/mm/ksm.rst index 2b4f72f1f953..5a80573162de 100644 --- a/Documentation/mm/ksm.rst +++ b/Documentation/mm/ksm.rst @@ -78,7 +78,7 @@ The frequency of such scans is defined by Reference --------- .. kernel-doc:: mm/ksm.c - :functions: mm_slot ksm_scan stable_node rmap_item + :functions: ksm_mm_slot ksm_scan ksm_stable_node ksm_rmap_item -- Izik Eidus, From d307e85e4d7d002349723118cdb43da67e2c3447 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 4 Sep 2026 20:05:17 -0700 Subject: [PATCH 254/539] memcg: move per-node objcg to the read-mostly fields Patch series "memcg: group struct fields by access pattern". Every so often we get a memcg performance regression caused by nothing more than a field moving. Someone adds a field, removes one, or puts a few behind a config option. The layout shifts, fields with different access patterns land on the same cache line, and a bot reports a regression. Two examples. commit 98c9daf5ae6b ("mm: memcg: guard memcg1-specific members of struct mem_cgroup_per_node") moved lruvec next to lru_zone_size[] and needed commit f59adcf59332 ("mm: memcg: add cacheline padding after lruvec in mem_cgroup_per_node") to fix it. commit c1afbd5de131 ("mm/memcontrol: avoid false sharing between vmstats and events") had to add ____cacheline_aligned_in_smp for the same reason. Each fix was correct but nothing stops the next field addition from undoing it. This series makes the layout a contract the compiler checks, the same way struct net_device does it. Fields are sorted into named cache line groups by access pattern, and memcg_struct_check() verifies at build time that every field sits in its group. A field added in the wrong place now breaks the build instead of quietly costing a few percent. struct mem_cgroup gets three groups: memcg_write_hot written on the charge, reclaim and socket paths memcg_cold only the cgroup control paths touch these memcg_read_mostly set when the memcg is created, then only read Testing ======= The cgroup selftests give identical results with and without the series. For performance, two identical 30 core Xeon machines each ran both kernels, with the boot order swapped between them so that machine and order effects cancel. The useful tests run two workloads at once in one cgroup, because false sharing only shows up when one side reads a field that the other side writes. slab allocs + page faults, slab side +1.2% page faults + memory.stat readers, fault side +1.3% page faults + memory.stat readers, reader side +0.8% everything else no change No test regressed. The gains are small but the point of the series is the build time contract. This patch (of 6): current_obj_cgroup() reads memcg->nodeinfo[nid]->objcg on every accounted allocation. The field sits at the end of struct mem_cgroup_per_node, on the same cache line as lru_zone_size[] and iter. lru_zone_size[] is written on every LRU add and remove, and iter is written on every reclaim iteration. Move objcg next to the other read-mostly pointers at the start of the struct. No functional change. Link: https://lore.kernel.org/20260905030522.1887837-1-shakeel.butt@linux.dev Link: https://lore.kernel.org/20260905030522.1887837-2-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/memcontrol.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index a03b6e3e5470..cfa1877915d4 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -94,6 +94,7 @@ struct mem_cgroup_per_node { struct lruvec_stats_percpu __percpu *lruvec_stats_percpu; struct lruvec_stats *lruvec_stats; struct shrinker_info __rcu *shrinker_info; + struct obj_cgroup __rcu *objcg; CACHELINE_PADDING(_pad1_); @@ -104,11 +105,9 @@ struct mem_cgroup_per_node { struct mem_cgroup_reclaim_iter iter; /* - * objcg is wiped out as a part of the objcg repaprenting process. * orig_objcg preserves a pointer (and a reference) to the original - * objcg until the end of live of memcg. + * objcg until the end of life of memcg. */ - struct obj_cgroup __rcu *objcg; struct obj_cgroup *orig_objcg; /* list of inherited objcgs, protected by objcg_lock */ struct list_head objcg_list; From 43eae38b7b294e6320a48e3f0a799ec6de3d34f6 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 4 Sep 2026 20:05:18 -0700 Subject: [PATCH 255/539] memcg: split mem_cgroup_private_id into two fields The two members of struct mem_cgroup_private_id have different access patterns. The id is read on every eviction and refault through mem_cgroup_private_id(), and is only written when the memcg is created and destroyed. The ref is written on every swap charge and uncharge. Split them into private_id and private_id_ref so a later patch can put them into different cache line groups. A struct member cannot be split across two groups. No functional change. Link: https://lore.kernel.org/20260905030522.1887837-3-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/memcontrol.h | 10 +++------- mm/memcontrol.c | 18 +++++++++--------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index cfa1877915d4..32a959f1890a 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -66,11 +66,6 @@ struct mem_cgroup_reclaim_cookie { #define MEM_CGROUP_ID_SHIFT 16 -struct mem_cgroup_private_id { - int id; - refcount_t ref; -}; - struct memcg_vmstats_percpu; struct memcg1_events_percpu; struct memcg_vmstats; @@ -189,7 +184,8 @@ struct mem_cgroup { struct cgroup_subsys_state css; /* Private memcg ID. Used to ID objects that outlive the cgroup */ - struct mem_cgroup_private_id id; + int private_id; + refcount_t private_id_ref; /* Accounted resources */ struct page_counter memory; /* Both v1 & v2 */ @@ -810,7 +806,7 @@ static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg) if (mem_cgroup_disabled()) return 0; - return memcg->id.id; + return memcg->private_id; } struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 4cb2db8c0923..2f11a3a79a6b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3773,7 +3773,7 @@ static void memcg_online_kmem(struct mem_cgroup *memcg) static_branch_enable(&memcg_kmem_online_key); - memcg->kmemcg_id = memcg->id.id; + memcg->kmemcg_id = memcg->private_id; } static void memcg_offline_kmem(struct mem_cgroup *memcg) @@ -4032,15 +4032,15 @@ static DEFINE_XARRAY_ALLOC1(mem_cgroup_private_ids); static void mem_cgroup_private_id_remove(struct mem_cgroup *memcg) { - if (memcg->id.id > 0) { - xa_erase(&mem_cgroup_private_ids, memcg->id.id); - memcg->id.id = 0; + if (memcg->private_id > 0) { + xa_erase(&mem_cgroup_private_ids, memcg->private_id); + memcg->private_id = 0; } } static inline void mem_cgroup_private_id_put(struct mem_cgroup *memcg, unsigned int n) { - if (refcount_sub_and_test(n, &memcg->id.ref)) { + if (refcount_sub_and_test(n, &memcg->private_id_ref)) { mem_cgroup_private_id_remove(memcg); /* Memcg ID pins CSS */ @@ -4050,7 +4050,7 @@ static inline void mem_cgroup_private_id_put(struct mem_cgroup *memcg, unsigned struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, unsigned int n) { - while (!refcount_add_not_zero(n, &memcg->id.ref)) { + while (!refcount_add_not_zero(n, &memcg->private_id_ref)) { /* * The root cgroup cannot be destroyed, so it's refcount must * always be >= 1. @@ -4174,7 +4174,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) if (!memcg) return ERR_PTR(-ENOMEM); - error = xa_alloc(&mem_cgroup_private_ids, &memcg->id.id, NULL, + error = xa_alloc(&mem_cgroup_private_ids, &memcg->private_id, NULL, XA_LIMIT(1, MEM_CGROUP_ID_MAX), GFP_KERNEL); if (error) goto fail; @@ -4320,7 +4320,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) lru_gen_online_memcg(memcg); /* Online state pins memcg ID, memcg ID pins CSS */ - refcount_set(&memcg->id.ref, 1); + refcount_set(&memcg->private_id_ref, 1); css_get(css); /* @@ -4333,7 +4333,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) * publish it here at the end of onlining. This matches the * regular ID destruction during offlining. */ - xa_store(&mem_cgroup_private_ids, memcg->id.id, memcg, GFP_KERNEL); + xa_store(&mem_cgroup_private_ids, memcg->private_id, memcg, GFP_KERNEL); return 0; free_objcg: From 0ceeb2c7c543a660b1f58a7ba113c9fa2e269661 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 4 Sep 2026 20:05:19 -0700 Subject: [PATCH 256/539] memcg: group the write-hot fields of struct mem_cgroup These fields are written on the charge, reclaim and socket paths: socket_pressure written by reclaim, read on every socket charge memory_events bumped for this memcg and every ancestor, so a busy child dirties the whole chain memory_events_local vmpressure written on every reclaim iteration private_id_ref written on every swap charge and uncharge kmem_stat high_irq_work, high_work They are spread over the struct today and share cache lines with read-mostly fields. Put them in one cache line group. socket_pressure is kept next to memory_events because mem_cgroup_sk_under_memory_pressure() reads one and bumps the other. Add memcg_struct_check() so the build fails if a field lands outside its group. No functional change. Link: https://lore.kernel.org/20260905030522.1887837-4-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/memcontrol.h | 58 ++++++++++++++++++++++---------------- mm/memcontrol.c | 32 +++++++++++++++++++++ 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 32a959f1890a..66431981f4f7 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -185,7 +185,6 @@ struct mem_cgroup { /* Private memcg ID. Used to ID objects that outlive the cgroup */ int private_id; - refcount_t private_id_ref; /* Accounted resources */ struct page_counter memory; /* Both v1 & v2 */ @@ -195,14 +194,45 @@ struct mem_cgroup { struct page_counter memsw; /* v1 only */ }; + /* Written on the charge, reclaim and socket paths. */ + __cacheline_group_begin_aligned(memcg_write_hot); + /* + * Hint of reclaim pressure for socket memory management. Note + * that this indicator should NOT be used in legacy cgroup mode + * where socket memory is accounted/charged separately. + */ + u64 socket_pressure; +#if BITS_PER_LONG < 64 + seqlock_t socket_pressure_seqlock; +#endif + /* + * memory.events is bumped for this memcg and all its ancestors, so a + * busy child dirties every ancestor. + */ + atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; + atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS]; + + /* vmpressure notifications. Written on every reclaim iteration. */ + struct vmpressure vmpressure; + + /* Written on every swap charge and uncharge. */ + refcount_t private_id_ref; + +#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC + /* MEMCG_KMEM for nmi context */ + atomic_t kmem_stat; +#endif + + /* Range enforcement for interrupt charges */ + struct work_struct high_work; + + __cacheline_group_end_aligned(memcg_write_hot); + /* registered local peak watchers */ struct list_head memory_peaks; struct list_head swap_peaks; spinlock_t peaks_lock; - /* Range enforcement for interrupt charges */ - struct work_struct high_work; - #ifdef CONFIG_ZSWAP unsigned long zswap_max; @@ -213,9 +243,6 @@ struct mem_cgroup { bool zswap_writeback; #endif - /* vmpressure notifications */ - struct vmpressure vmpressure; - /* * Should the OOM killer kill all belonging tasks, had it kill one? */ @@ -231,23 +258,6 @@ struct mem_cgroup { /* memory.stat */ struct memcg_vmstats *vmstats; - /* memory.events */ - atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; - atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS]; - -#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC - /* MEMCG_KMEM for nmi context */ - atomic_t kmem_stat; -#endif - /* - * Hint of reclaim pressure for socket memroy management. Note - * that this indicator should NOT be used in legacy cgroup mode - * where socket memory is accounted/charged separately. - */ - u64 socket_pressure; -#if BITS_PER_LONG < 64 - seqlock_t socket_pressure_seqlock; -#endif int kmemcg_id; #ifdef CONFIG_CGROUP_WRITEBACK diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2f11a3a79a6b..56fc6d8c02a8 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5700,6 +5700,36 @@ __setup("cgroup.memory=", cgroup_memory); * basically everything that doesn't depend on a specific mem_cgroup structure * should be initialized from here. */ +/* + * Fields are grouped by access pattern. Putting a field in the wrong group + * breaks the build here. + */ +static void __init memcg_struct_check(void) +{ + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + socket_pressure); +#if BITS_PER_LONG < 64 + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + socket_pressure_seqlock); +#endif + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + memory_events); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + memory_events_local); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + vmpressure); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + private_id_ref); +#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + kmem_stat); +#endif + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + high_irq_work); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, + high_work); +} + int __init mem_cgroup_init(void) { unsigned int memcg_size; @@ -5713,6 +5743,8 @@ int __init mem_cgroup_init(void) */ BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE); + memcg_struct_check(); + cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, memcg_hotplug_cpu_dead); From 538c63bab205db52397387591ab9899ee03b32af Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 4 Sep 2026 20:05:20 -0700 Subject: [PATCH 257/539] memcg: group the cold fields of struct mem_cgroup These fields are only touched by the cgroup control paths: memory_peaks, swap_peaks, peaks_lock memory.peak open/read/release events_file, events_local_file, swap_events_file cgroup_file_notify() cgwb_list, cgwb_domain, cgwb_frn writeback setup and the foreign dirty slow path mm_list MGLRU mm list They sit in the middle of the struct today. The three cgroup_file members alone are 192 bytes of notify state next to the vmstats pointer. Put them in one cache line group. No functional change. Link: https://lore.kernel.org/20260905030522.1887837-5-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/memcontrol.h | 49 ++++++++++++++++++++++---------------- mm/memcontrol.c | 25 +++++++++++++++++++ 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 66431981f4f7..3a7f80971302 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -228,11 +228,37 @@ struct mem_cgroup { __cacheline_group_end_aligned(memcg_write_hot); + /* + * Off the charge and fault paths. Not write free: cgwb_domain is + * written on every writeout completion and mm_list on fork, exit and + * MGLRU aging. They are grouped here so those writes cannot land on + * a line that the fast paths read. + */ + __cacheline_group_begin_aligned(memcg_cold); /* registered local peak watchers */ struct list_head memory_peaks; struct list_head swap_peaks; spinlock_t peaks_lock; + /* memory.events and memory.events.local */ + struct cgroup_file events_file; + struct cgroup_file events_local_file; + + /* handle for "memory.swap.events" */ + struct cgroup_file swap_events_file; + +#ifdef CONFIG_CGROUP_WRITEBACK + struct list_head cgwb_list; + struct wb_domain cgwb_domain; + struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; +#endif + +#ifdef CONFIG_LRU_GEN_WALKS_MMU + /* per-memcg mm_struct list */ + struct lru_gen_mm_list mm_list; +#endif + __cacheline_group_end_aligned(memcg_cold); + #ifdef CONFIG_ZSWAP unsigned long zswap_max; @@ -248,37 +274,18 @@ struct mem_cgroup { */ bool oom_group; - /* memory.events and memory.events.local */ - struct cgroup_file events_file; - struct cgroup_file events_local_file; - - /* handle for "memory.swap.events" */ - struct cgroup_file swap_events_file; - /* memory.stat */ struct memcg_vmstats *vmstats; int kmemcg_id; -#ifdef CONFIG_CGROUP_WRITEBACK - struct list_head cgwb_list; -#endif - /* Keep the hot per-CPU stats pointer away from memory event counters. */ struct memcg_vmstats_percpu __percpu *vmstats_percpu ____cacheline_aligned_in_smp; -#ifdef CONFIG_CGROUP_WRITEBACK - struct wb_domain cgwb_domain; - struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; -#endif - -#ifdef CONFIG_LRU_GEN_WALKS_MMU - /* per-memcg mm_struct list */ - struct lru_gen_mm_list mm_list; -#endif - #ifdef CONFIG_MEMCG_V1 + /* v1 only. Not grouped: v1 is legacy, sorting it is not worth it. */ + /* Legacy consumer-oriented counters */ struct page_counter kmem; /* v1 only */ struct page_counter tcpmem; /* v1 only */ diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 56fc6d8c02a8..51539265bfa9 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5728,6 +5728,31 @@ static void __init memcg_struct_check(void) high_irq_work); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, high_work); + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + memory_peaks); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + swap_peaks); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + peaks_lock); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + events_file); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + events_local_file); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + swap_events_file); +#ifdef CONFIG_CGROUP_WRITEBACK + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + cgwb_list); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + cgwb_domain); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + cgwb_frn); +#endif +#ifdef CONFIG_LRU_GEN_WALKS_MMU + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, + mm_list); +#endif } int __init mem_cgroup_init(void) From 7580861e13035066b0f345ab319aaad586c3e7c7 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 4 Sep 2026 20:05:21 -0700 Subject: [PATCH 258/539] memcg: group the read-mostly fields of struct mem_cgroup These fields are set when the memcg is created and only read after that: vmstats_percpu read on every stat update vmstats zswap_max, zswap_writeback private_id read on every eviction and refault kmemcg_id read on every list_lru lookup oom_group Put them in one cache line group at the end of the struct, right before nodeinfo[]. nodeinfo[] is read-mostly too but it is a flexible array, so it cannot sit inside a group. The group ends without padding so the two share a line. This also drops the ____cacheline_aligned_in_smp on vmstats_percpu added by commit c1afbd5de131 ("mm/memcontrol: avoid false sharing between vmstats and events"). That only aligned the start of the field. cgwb_domain followed it on the same line and is written on every writeout completion. A group boundary covers both sides. No functional change. Link: https://lore.kernel.org/20260905030522.1887837-6-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/memcontrol.h | 62 +++++++++++++++++++++----------------- mm/memcontrol.c | 17 +++++++++++ 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 3a7f80971302..73594783a2ef 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -183,9 +183,6 @@ struct obj_cgroup { struct mem_cgroup { struct cgroup_subsys_state css; - /* Private memcg ID. Used to ID objects that outlive the cgroup */ - int private_id; - /* Accounted resources */ struct page_counter memory; /* Both v1 & v2 */ @@ -259,30 +256,6 @@ struct mem_cgroup { #endif __cacheline_group_end_aligned(memcg_cold); -#ifdef CONFIG_ZSWAP - unsigned long zswap_max; - - /* - * Prevent pages from this memcg from being written back from zswap to - * swap, and from being swapped out on zswap store failures. - */ - bool zswap_writeback; -#endif - - /* - * Should the OOM killer kill all belonging tasks, had it kill one? - */ - bool oom_group; - - /* memory.stat */ - struct memcg_vmstats *vmstats; - - int kmemcg_id; - - /* Keep the hot per-CPU stats pointer away from memory event counters. */ - struct memcg_vmstats_percpu __percpu *vmstats_percpu - ____cacheline_aligned_in_smp; - #ifdef CONFIG_MEMCG_V1 /* v1 only. Not grouped: v1 is legacy, sorting it is not worth it. */ @@ -322,6 +295,41 @@ struct mem_cgroup { int swappiness; #endif /* CONFIG_MEMCG_V1 */ + /* + * Set when the memcg is created and cleared when it is offlined. + * Never written on a hot path. + */ + __cacheline_group_begin_aligned(memcg_read_mostly); + /* Read on every stat update */ + struct memcg_vmstats_percpu __percpu *vmstats_percpu; + + /* memory.stat */ + struct memcg_vmstats *vmstats; + +#ifdef CONFIG_ZSWAP + unsigned long zswap_max; +#endif + + /* Private memcg ID. Used to ID objects that outlive the cgroup */ + int private_id; + + int kmemcg_id; + + /* + * Should the OOM killer kill all belonging tasks, had it kill one? + */ + bool oom_group; + +#ifdef CONFIG_ZSWAP + /* + * Prevent pages from this memcg from being written back from zswap to + * swap, and from being swapped out on zswap store failures. + */ + bool zswap_writeback; +#endif + /* Not padded: nodeinfo[] is read-mostly too, let it share the line. */ + __cacheline_group_end(memcg_read_mostly); + struct mem_cgroup_per_node *nodeinfo[]; }; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 51539265bfa9..a392f06a5213 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5753,6 +5753,23 @@ static void __init memcg_struct_check(void) CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, mm_list); #endif + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + vmstats_percpu); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + vmstats); +#ifdef CONFIG_ZSWAP + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + zswap_max); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + zswap_writeback); +#endif + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + private_id); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + kmemcg_id); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, + oom_group); } int __init mem_cgroup_init(void) From c1b11d76ea9c9ab641afed9ef7df41c31bdde8e1 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 4 Sep 2026 20:05:22 -0700 Subject: [PATCH 259/539] memcg: group the fields of struct mem_cgroup_per_node Replace the two ad-hoc CACHELINE_PADDING members with named cache line groups: memcg_pn_read_mostly memcg, lruvec_stats_percpu, lruvec_stats, shrinker_info, objcg memcg_pn_lruvec lruvec memcg_pn_write_hot lru_zone_size, iter, nmi slab stats memcg_pn_cold orig_objcg, objcg_list The group markers give the same isolation the padding did, but they are named and the build now checks them. lruvec still gets its own lines. Commit f59adcf59332 ("mm: memcg: add cacheline padding after lruvec in mem_cgroup_per_node") showed why that matters: lru_zone_size[] is written under lru_lock but read without it by lruvec_lru_size(), so it must not share a line with lruvec. Splitting the cold fields out costs one extra cache line per node per memcg. No functional change. Link: https://lore.kernel.org/20260905030522.1887837-7-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Signed-off-by: Andrew Morton Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/memcontrol.h | 32 ++++++++++++++++++++++---------- mm/memcontrol.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 73594783a2ef..c0c9805b6f03 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -82,7 +82,8 @@ struct mem_cgroup_reclaim_iter { * per-node information in memory controller. */ struct mem_cgroup_per_node { - /* Keep the read-only fields at the start */ + /* Set when the memcg is created, then only read. */ + __cacheline_group_begin_aligned(memcg_pn_read_mostly); struct mem_cgroup *memcg; /* Back pointer, we cannot */ /* use container_of */ @@ -91,14 +92,30 @@ struct mem_cgroup_per_node { struct shrinker_info __rcu *shrinker_info; struct obj_cgroup __rcu *objcg; - CACHELINE_PADDING(_pad1_); + __cacheline_group_end_aligned(memcg_pn_read_mostly); - /* Fields which get updated often at the end. */ + /* + * Keep lruvec on its own lines. Sharing them with lru_zone_size[] + * regressed, see commit f59adcf59332 ("mm: memcg: add cacheline + * padding after lruvec in mem_cgroup_per_node"). + */ + __cacheline_group_begin_aligned(memcg_pn_lruvec); struct lruvec lruvec; - CACHELINE_PADDING(_pad2_); + __cacheline_group_end_aligned(memcg_pn_lruvec); + + /* Written on every LRU update and on every reclaim iteration. */ + __cacheline_group_begin_aligned(memcg_pn_write_hot); long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter; +#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC + /* slab stats for nmi context */ + atomic_t slab_reclaimable; + atomic_t slab_unreclaimable; +#endif + __cacheline_group_end_aligned(memcg_pn_write_hot); + /* Touched only when the memcg is reparented or freed. */ + __cacheline_group_begin_aligned(memcg_pn_cold); /* * orig_objcg preserves a pointer (and a reference) to the original * objcg until the end of life of memcg. @@ -106,12 +123,7 @@ struct mem_cgroup_per_node { struct obj_cgroup *orig_objcg; /* list of inherited objcgs, protected by objcg_lock */ struct list_head objcg_list; - -#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC - /* slab stats for nmi context */ - atomic_t slab_reclaimable; - atomic_t slab_unreclaimable; -#endif + __cacheline_group_end_aligned(memcg_pn_cold); }; struct mem_cgroup_threshold { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a392f06a5213..e539881f58f1 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5770,6 +5770,36 @@ static void __init memcg_struct_check(void) kmemcg_id); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, oom_group); + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, memcg); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, lruvec_stats_percpu); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, lruvec_stats); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, shrinker_info); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_read_mostly, objcg); + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_lruvec, lruvec); + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_write_hot, lru_zone_size); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_write_hot, iter); +#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_write_hot, slab_reclaimable); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_write_hot, slab_unreclaimable); +#endif + + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_cold, orig_objcg); + CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, + memcg_pn_cold, objcg_list); } int __init mem_cgroup_init(void) From bc82fb61d4891bcae7fdd4411a36aceb51b7f0ca Mon Sep 17 00:00:00 2001 From: Wilson Felipe Pereira Date: Fri, 4 Sep 2026 23:20:47 +0000 Subject: [PATCH 260/539] mm/zswap: convert zswap_store_page() and zswap_compress() to take a folio Currently, zswap_store() iterates through a folio and extracts individual struct page pointers using folio_page() to pass into zswap_store_page() and zswap_compress(). Pass the folio and the page index directly into these functions instead. This eliminates the need to materialize struct page pointers inside the zswap_store() loop and replaces legacy page-based accessors with their folio equivalents: - sg_set_page() -> sg_set_folio() - kmap_local_page() -> kmap_local_folio() - page_to_nid() -> folio_nid() - page_swap_entry() -> calculated via folio->swap and index Link: https://lore.kernel.org/20260904232108.3034333-1-wfelipe@google.com Signed-off-by: Wilson Felipe Pereira Signed-off-by: Andrew Morton Reviewed-by: Tal Zussman Cc: Chengming Zhou Cc: Johannes Weiner Cc: Nhat Pham --- mm/zswap.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index f3ae3c81e48e..b894de1786fd 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -824,8 +824,8 @@ fail: return ret; } -static bool zswap_compress(struct page *page, struct zswap_entry *entry, - struct zswap_pool *pool) +static bool zswap_compress(struct folio *folio, long index, + struct zswap_entry *entry, struct zswap_pool *pool) { struct crypto_acomp_ctx *acomp_ctx; struct scatterlist input, output; @@ -841,7 +841,7 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry, dst = acomp_ctx->buffer; sg_init_table(&input, 1); - sg_set_page(&input, page, PAGE_SIZE, 0); + sg_set_folio(&input, folio, PAGE_SIZE, index * PAGE_SIZE); sg_init_one(&output, dst, PAGE_SIZE); acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen); @@ -870,8 +870,7 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry, */ if (comp_ret || !dlen || dlen >= PAGE_SIZE) { rcu_read_lock(); - if (!mem_cgroup_zswap_writeback_enabled( - folio_memcg(page_folio(page)))) { + if (!mem_cgroup_zswap_writeback_enabled(folio_memcg(folio))) { rcu_read_unlock(); comp_ret = comp_ret ? comp_ret : -EINVAL; goto unlock; @@ -879,12 +878,12 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry, rcu_read_unlock(); comp_ret = 0; dlen = PAGE_SIZE; - dst = kmap_local_page(page); + dst = kmap_local_folio(folio, index * PAGE_SIZE); mapped = true; } gfp = GFP_NOWAIT | __GFP_NORETRY | __GFP_HIGHMEM | __GFP_MOVABLE; - handle = zs_malloc(pool->zs_pool, dlen, gfp, page_to_nid(page)); + handle = zs_malloc(pool->zs_pool, dlen, gfp, folio_nid(folio)); if (IS_ERR_VALUE(handle)) { alloc_ret = PTR_ERR((void *)handle); goto unlock; @@ -1392,21 +1391,22 @@ resched: * main API **********************************/ -static bool zswap_store_page(struct page *page, +static bool zswap_store_page(struct folio *folio, long index, struct obj_cgroup *objcg, struct zswap_pool *pool) { - swp_entry_t page_swpentry = page_swap_entry(page); + swp_entry_t page_swpentry = swp_entry(swp_type(folio->swap), + swp_offset(folio->swap) + index); struct zswap_entry *entry, *old; /* allocate entry */ - entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page)); + entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio)); if (!entry) { zswap_reject_kmemcache_fail++; return false; } - if (!zswap_compress(page, entry, pool)) + if (!zswap_compress(folio, index, entry, pool)) goto compress_failed; old = xa_store(swap_zswap_tree(page_swpentry), @@ -1515,9 +1515,7 @@ bool zswap_store(struct folio *folio) } for (index = 0; index < nr_pages; ++index) { - struct page *page = folio_page(folio, index); - - if (!zswap_store_page(page, objcg, pool)) + if (!zswap_store_page(folio, index, objcg, pool)) goto put_pool; } From bffa47f1ca00952def6b5cc6feda3889033d1c29 Mon Sep 17 00:00:00 2001 From: David Stevens Date: Fri, 4 Sep 2026 10:31:45 -0700 Subject: [PATCH 261/539] memcg: don't call schedule_work when no spinning is allowed Memcg charging can be done from any context, but calling schedule_work() isn't safe from an NMI. If memory.high is breached from a context where spinning isn't allowed, use irq_work to schedule the reclaim work. Found this via code inspection. I spent a little bit trying to trigger it for real, but the only way I managed was by writing a hacky driver absuing alloc_pages_nolock(). Link: https://lore.kernel.org/20260904173145.2028377-1-stevensd@google.com Fixes: 3ac4638a734a ("memcg: make memcg_rstat_updated nmi safe") Signed-off-by: David Stevens Signed-off-by: Andrew Morton Acked-by: Michal Hocko Reviewed-by: Johannes Weiner Acked-by: Shakeel Butt Cc: Lorenzo Stoakes Cc: Muchun Song Cc: Roman Gushchin --- include/linux/memcontrol.h | 2 ++ mm/memcontrol.c | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index c0c9805b6f03..4f720791a31c 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -23,6 +23,7 @@ #include #include #include +#include struct mem_cgroup; struct obj_cgroup; @@ -233,6 +234,7 @@ struct mem_cgroup { #endif /* Range enforcement for interrupt charges */ + struct irq_work high_irq_work; struct work_struct high_work; __cacheline_group_end_aligned(memcg_write_hot); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e539881f58f1..3e14e0ce9e7e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -62,6 +62,7 @@ #include #include #include +#include #include "internal.h" #include "swap.h" #include "swap_table.h" @@ -2424,6 +2425,11 @@ static void high_work_func(struct work_struct *work) reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); } +static void high_irq_work_func(struct irq_work *work) +{ + schedule_work(&container_of(work, struct mem_cgroup, high_irq_work)->high_work); +} + /* * Clamp the maximum sleep time per allocation batch to 2 seconds. This is * enough to still cause a significant slowdown in most cases, while still @@ -2832,7 +2838,10 @@ done_restock: /* Don't bother a random interrupted task */ if (!in_task()) { if (mem_high) { - schedule_work(&memcg->high_work); + if (allow_spinning) + schedule_work(&memcg->high_work); + else + irq_work_queue(&memcg->high_irq_work); break; } continue; @@ -4207,6 +4216,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) goto fail; INIT_WORK(&memcg->high_work, high_work_func); + init_irq_work(&memcg->high_irq_work, high_irq_work_func); vmpressure_init(&memcg->vmpressure); INIT_LIST_HEAD(&memcg->memory_peaks); INIT_LIST_HEAD(&memcg->swap_peaks); @@ -4415,6 +4425,7 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css) static_branch_dec(&memcg_bpf_enabled_key); vmpressure_cleanup(&memcg->vmpressure); + irq_work_sync(&memcg->high_irq_work); cancel_work_sync(&memcg->high_work); free_shrinker_info(memcg); mem_cgroup_free(memcg); From ba99faaf670c05a96edd1aa7bc6fa19f576697a4 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 3 Sep 2026 14:56:16 -0700 Subject: [PATCH 262/539] mm/memcontrol: skip non-hierarchical memcg-wide stats when v1 is unavailable memcg_vmstats keeps a non-hierarchical copy of every memcg-wide stat item and event alongside the hierarchical one. The only readers however are the legacy memory.stat and memory.numa_stat, and reparenting on offline. All of them live under CONFIG_MEMCG_V1, and their accessors (memcg_page_state_local() and memcg_events_local()) are already compiled out with it. This means on a CONFIG_MEMCG_V1=n kernel, memcg_vmstats's non-hierarchical arrays are written to on every rstat flush, despite their values never being read / accessed. The same holds when the kernel does support v1 but the controller has been blocked from v1 hierarchies with the boot param cgroup_no_v1={memory,all}. A v1 mount is refused in that case, so the legacy memory.stat can never exist and the arrays are just as unread / unaccessed. Compile out the non-hierarchical memcg-wide arrays if CONFIG_MEMCG_V1 is not set. If it is set but cgroup_no_v1= has blocked the controller, skip updates on the arrays. This makes flushes cheaper. mem_cgroup_stat_aggregate() can now skip the read-modify-write of ac->local[i]. Nothing else accesses state_local or events_local, so those cachelines were getting pulled in solely for the writes and they are separate from the ones the loop is already walking. On an 80-cpu x86_64 machine with 500 cgroups each running a workload that dirties anon, file, dirty/writeback, slab, kmem, mlock, and reclaim counters, timing mem_cgroup_css_rstat_flush() in-kernel in TSC ticks per flush showed roughly before after delta memcg-wide aggregation 1231 1180 -4.1% overall flush function 2452 2397 -2.2% These numbers are from taking the median of 70 samples, one per 20s window on each kernel. The 95% intervals observed on the two deltas are [-5.41%, -1.76%] and [-4.53%, -0.14%]. The values above include the timing overhead itself, so only the delta is meaningful here. Counting the items that actually changed, a median of 1.5 of the 77 memcg-wide items (57 state + 20 events) had a non-zero per-cpu delta at each flush, which means the benchmarks above are with one or two fewer cachelines pulled in per flush. The count is low because the benchmark reads memory.stat in a loop to keep the flush rate up. For cases where flushes are triggered only by the 2s periodic worker, more changes will have accumulated between flushes, so more cachelines are skipped and the per-flush saving should be larger. Link: https://lore.kernel.org/20260903215616.1456239-1-joannelkoong@gmail.com Signed-off-by: Joanne Koong Signed-off-by: Andrew Morton Reviewed-by: Johannes Weiner Acked-by: Shakeel Butt Reviewed-by: Yosry Ahmed Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin --- include/linux/cgroup.h | 3 +++ kernel/cgroup/cgroup-internal.h | 1 - mm/memcontrol.c | 48 ++++++++++++++++++++++++++++----- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 5dfa915a630e..2afb4cb2bb4f 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -154,6 +154,9 @@ struct cgroup *cgroup_get_from_path(const char *path); struct cgroup *cgroup_get_from_fd(int fd); struct cgroup *cgroup_v1v2_get_from_fd(int fd); +/* Was this controller blocked from v1 hierarchies by cgroup_no_v1= ? */ +bool cgroup1_ssid_disabled(int ssid); + int cgroup_attach_task_all(struct task_struct *from, struct task_struct *); int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from); diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 58797123b752..7c367c8d0cbe 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -285,7 +285,6 @@ extern struct kernfs_syscall_ops cgroup1_kf_syscall_ops; extern const struct fs_parameter_spec cgroup1_fs_parameters[]; int proc_cgroupstats_show(struct seq_file *m, void *v); -bool cgroup1_ssid_disabled(int ssid); void cgroup1_pidlist_destroy_all(struct cgroup *cgrp); void cgroup1_release_agent(struct work_struct *work); void cgroup1_check_for_release(struct cgroup *cgrp); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 3e14e0ce9e7e..3e0f6edfd350 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -676,9 +676,11 @@ struct memcg_vmstats { long state[MEMCG_VMSTAT_SIZE]; unsigned long events[NR_MEMCG_EVENTS]; +#ifdef CONFIG_MEMCG_V1 /* Non-hierarchical (CPU aggregated) page state & events */ long state_local[MEMCG_VMSTAT_SIZE]; unsigned long events_local[NR_MEMCG_EVENTS]; +#endif /* Pending child counts during tree propagation */ long state_pending[MEMCG_VMSTAT_SIZE]; @@ -688,6 +690,31 @@ struct memcg_vmstats { atomic_long_t stats_updates; }; +/* + * The non-hierarchical memcg-wide counters are read back only by the legacy + * memory.stat and memory.numa_stat, and by reparenting on offline, all of which + * are v1-only. If the kernel is built without CONFIG_MEMCG_V1, or if the boot + * param cgroup_no_v1= has blocked the memory controller from v1 hierarchies, + * then nothing reads them and writers can skip the updates. + */ +static long *memcg_state_local_array(struct mem_cgroup *memcg) +{ +#ifdef CONFIG_MEMCG_V1 + if (!cgroup1_ssid_disabled(memory_cgrp_id)) + return memcg->vmstats->state_local; +#endif + return NULL; +} + +static unsigned long *memcg_events_local_array(struct mem_cgroup *memcg) +{ +#ifdef CONFIG_MEMCG_V1 + if (!cgroup1_ssid_disabled(memory_cgrp_id)) + return memcg->vmstats->events_local; +#endif + return NULL; +} + /* * memcg and lruvec stats flushing * @@ -4469,7 +4496,10 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) struct aggregate_control { /* pointer to the aggregated (CPU and subtree aggregated) counters */ long *aggregate; - /* pointer to the non-hierarchichal (CPU aggregated) counters */ + /* + * pointer to the non-hierarchical (CPU aggregated) counters or NULL to + * skip updating them (see memcg_state_local_array()) + */ long *local; /* pointer to the pending child counters during tree propagation */ long *pending; @@ -4508,7 +4538,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac) } /* Aggregate counts on this level and propagate upwards */ - if (delta_cpu) + if (delta_cpu && ac->local) ac->local[i] += delta_cpu; if (delta) { @@ -4522,6 +4552,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac) #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) { + long *state_local = memcg_state_local_array(memcg); int nid; if (atomic_read(&memcg->kmem_stat)) { @@ -4529,7 +4560,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) int index = memcg_stats_index(MEMCG_KMEM); memcg->vmstats->state[index] += kmem; - memcg->vmstats->state_local[index] += kmem; + if (state_local) + state_local[index] += kmem; if (parent) parent->vmstats->state_pending[index] += kmem; } @@ -4551,7 +4583,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; - memcg->vmstats->state_local[index] += slab; + if (state_local) + state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4564,7 +4597,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; - memcg->vmstats->state_local[index] += slab; + if (state_local) + state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4589,7 +4623,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) ac = (struct aggregate_control) { .aggregate = memcg->vmstats->state, - .local = memcg->vmstats->state_local, + .local = memcg_state_local_array(memcg), .pending = memcg->vmstats->state_pending, .ppending = parent ? parent->vmstats->state_pending : NULL, .cstat = statc->state, @@ -4600,7 +4634,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) ac = (struct aggregate_control) { .aggregate = memcg->vmstats->events, - .local = memcg->vmstats->events_local, + .local = memcg_events_local_array(memcg), .pending = memcg->vmstats->events_pending, .ppending = parent ? parent->vmstats->events_pending : NULL, .cstat = statc->events, From 4cac0fd3128893fb42de3abba334376b829b825c Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Thu, 3 Sep 2026 20:08:39 +0100 Subject: [PATCH 263/539] mm/madvise: swap in CoW'd MAP_PRIVATE-file mappings on MADV_WILLNEED Currently MADV_WILLNEED treats file-backed and pure anonymous mappings entirely separately - using POSIX_FADV_WILLNEED (equivalent of a readahead) for the former and a tree walk and swap in to swap cache for the latter. MAP_PRIVATE-file backed mappings straddle the two and currently get treated as if they were purely file-backed, meaning any swapped out private pages remain swapped out. Resolve the issue by explicitly checking for CoW'd MAP_PRIVATE-file backed mappings and performing both walks in this case. Since the logic checks for vma->anon_vma this means un-CoW'd MAP_PRIVATE-file backed mappings retain only the single file walk. Link: https://lore.kernel.org/aprjOxDy3JCPb2oa@gremlin Reported-by: Mike Kaplinskiy Closes: https://lore.kernel.org/all/CABeknB_S2XJSHFgnHdgnN0rjzHhH4oQJs_APq9fvxHztQ_pgiA@mail.gmail.com/ Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Vlastimil Babka (SUSE) Reviewed-by: Pedro Falcato Cc: David Hildenbrand Cc: Jann Horn Cc: Liam R. Howlett --- mm/madvise.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 73c2901b9adb..963337f93a7a 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -297,11 +297,12 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) loff_t offset; #ifdef CONFIG_SWAP - if (!file) { + if (vma_is_cow_mapping(vma) && vma->anon_vma) { walk_page_range_vma(vma, start, end, &swapin_walk_ops, vma); lru_add_drain(); /* Push any new pages onto the LRU now */ - return 0; } + if (!file) + return 0; if (shmem_mapping(file->f_mapping)) { shmem_swapin_range(vma, start, end, file->f_mapping); From 7ee68fff2ba6740558cab195925b3a7ff7a1480f Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Fri, 11 Sep 2026 16:00:48 +0800 Subject: [PATCH 264/539] mm: memcg: redirect stats updates of dying memcgs for all hierarchies Patch series "mm: workingset: fix the shadow node budget under MGLRU", v5. Commit 7404bd37cfbe ("mm: workingset: use lruvec_lru_size() to get the number of lru pages") broke the workingset shadow node budget under MGLRU: lruvec_lru_size() reads mz->lru_zone_size, which MGLRU never maintains, so count_shadow_nodes() sees the evictable LRU lists as empty and the shadow shrinker reclaims eviction tokens almost as fast as they are created, losing thrashing protection. Patch 1 extends the dying-mcg stat redirection (previously cgroup v1 only) to all hierarchies, addressing the reparenting race that motivated 7404bd37cfbe. Patch 2 then switches count_shadow_nodes() back to lruvec_page_state_local(), which both classic LRU and MGLRU maintain. Patch 3 recovers the performance. Patch 1 added an unconditional rcu_read_lock() to the stat update fast path; patch 3 checks memcg_is_dying() first and takes the RCU lock only on the rare dying path. Patch 4 closes an accounting gap that patch 2 makes visible: on cgroup v2, reparent_state_local() never moves the dying memcg's non-hierarchical lruvec stats to the parent, so the parent receives the uncharges without the matching charges and its state_local underflows. Patch 4 reparents those stats, mirroring cgroup v1. Performance testing =================== The test script and the raw results are available at [1]. Environment: 10-vCPU QEMU guest, 8 GiB RAM, cgroup v2; 7 runs per configuration, medians reported. Workloads: w1-anon-churn: single-threaded anon fault/charge loop in a memcg (MADV_DONTNEED + re-fault, no reclaim). Every touch is a real fault with charge and memcg stat updates, so it stresses exactly the fast path patch 1 changes. This is the meaningful signal: it is not reclaim-bound, so the small fast-path overhead is not drowned out. w2-file-churn: file read loop under memory.high pressure (reclaim-bound; the differences below are within run-to-run noise and are shown for completeness only). w3-reparent: reparent accounting sanity check. w1-anon-churn (pages/s): classic LRU MGLRU base 4393028 4377122 patches 1-2 4385996 (-0.2%) 4352887 (-0.6%) patches 1-3 4381832 (-0.3%) 4377053 (+0.0%) w2-file-churn (MB/s): classic LRU MGLRU base 8277 8226 patches 1-2 8226 (-0.6%) 8123 (-1.3%) patches 1-3 8157 (-1.4%) 8294 (+0.8%) w3-reparent passed on all kernels. The small overhead visible with patches 1-2 comes from the redirection added by patch 1; patch 3 brings w1 back to the base level in both LRU configurations. The w2-file-churn differences are within run-to-run noise: that workload is reclaim-bound and too noisy to expose the small fast-path overhead, so w1-anon-churn is the meaningful signal. Patch 4 only touches the memcg offline path and is not exercised by these workloads. This patch (of 4): get_non_dying_memcg_start() redirects the stat updates of a dying memcg to its closest non-dying ancestor, but only on cgroup v1; on cgroup v2 the stats keep being accounted to the dying memcg itself. A later patch in this series restores lruvec_page_state_local() in count_shadow_nodes() to fix the broken workingset shadow node budget under MGLRU. count_shadow_nodes() is the only reader of those non-hierarchical state_locals on cgroup v2: when a memcg is offlined, its pages are reparented to the ancestor but their stat updates keep being accounted to the dying memcg, so count_shadow_nodes() computes a wrong shadow node budget and workingset thrashing protection is lost. This is user visible as premature reclaim of hot page cache and degraded performance under memory pressure. Apply the redirection to all hierarchies to fix this. Offlining is rare, so the added cost on the stat update fast path is limited to an rcu_read_lock() and a css_is_dying() check; the upward walk happens only while a memcg is dying. Link: https://lore.kernel.org/cover.1789096175.git.zhuhui@kylinos.cn Link: https://lore.kernel.org/c1ef4ef6a84cac479e573f4423b734dc8176f7d5.1789096175.git.zhuhui@kylinos.cn Link: https://gist.github.com/teawater/32f373ec41d185d840455eb167321a5a [1] Fixes: 7404bd37cfbe ("mm: workingset: use lruvec_lru_size() to get the number of lru pages") Signed-off-by: Hui Zhu Signed-off-by: Andrew Morton Acked-by: Shakeel Butt Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Wei Xu Cc: Yuanchu Xie Cc: --- mm/memcontrol.c | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 3e0f6edfd350..8a571e80a103 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -873,20 +873,14 @@ static long memcg_state_val_in_pages(int idx, long val) return val < 0 ? -res : res; } -#ifdef CONFIG_MEMCG_V1 /* - * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race with - * reparenting of non-hierarchical state_locals. + * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race + * with reparenting of non-hierarchical state_locals. Offlining a + * memcg is rare, so do the redirection for all cgroup hierarchies. */ -static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg, - bool *rcu_locked) +static inline struct mem_cgroup * +get_non_dying_memcg_start(struct mem_cgroup *memcg, bool *rcu_locked) { - /* Rebinding can cause this value to be changed at runtime */ - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) { - *rcu_locked = false; - return memcg; - } - rcu_read_lock(); *rcu_locked = true; @@ -898,22 +892,8 @@ static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *me static inline void get_non_dying_memcg_end(bool rcu_locked) { - if (!rcu_locked) - return; - rcu_read_unlock(); } -#else -static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg, - bool *rcu_locked) -{ - return memcg; -} - -static inline void get_non_dying_memcg_end(bool rcu_locked) -{ -} -#endif static void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, long val) From c7a37bac3ccc2732993705d3c3488b278b422ec6 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Fri, 11 Sep 2026 16:00:49 +0800 Subject: [PATCH 265/539] mm: workingset: use lruvec_page_state_local() to count lru pages Commit 7404bd37cfbe ("mm: workingset: use lruvec_lru_size() to get the number of lru pages") switched count_shadow_nodes() to lruvec_lru_size(). With CONFIG_MEMCG enabled, lruvec_lru_size() reads mz->lru_zone_size, which only the classic LRU paths maintain. MGLRU accounts its pages through __update_lru_size(), which skips that array, so with MGLRU on the four evictable LRU lists are always seen as empty. The shadow node budget (pages >> 3) then collapses to slab plus unevictable pages, and the workingset shadow shrinker reclaims eviction tokens almost as fast as they are created, losing thrashing protection. lruvec_page_state_local() reads lruvec_stats->state_local instead, which both classic LRU and MGLRU maintain. Switch back to it. The reparenting race this re-exposes on cgroup v2 is closed by the preceding patch that redirects dying-memcg stat updates for all hierarchies. Link: https://lore.kernel.org/2ed42f96aca124856ea30f774afb55cbe6d8ba58.1789096175.git.zhuhui@kylinos.cn Fixes: 7404bd37cfbe ("mm: workingset: use lruvec_lru_size() to get the number of lru pages") Signed-off-by: Hui Zhu Signed-off-by: Andrew Morton Acked-by: Shakeel Butt Reviewed-by: Barry Song Reviewed-by: Kairui Song Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Wei Xu Cc: Yuanchu Xie Cc: --- mm/workingset.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm/workingset.c b/mm/workingset.c index 7ac2b88c80ae..8412f4840ae3 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -688,10 +688,9 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, mem_cgroup_flush_stats_ratelimited(sc->memcg); lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid)); - for (pages = 0, i = 0; i < NR_LRU_LISTS; i++) - pages += lruvec_lru_size(lruvec, i, MAX_NR_ZONES - 1); - + pages += lruvec_page_state_local(lruvec, + NR_LRU_BASE + i); pages += lruvec_page_state_local( lruvec, NR_SLAB_RECLAIMABLE_B) >> PAGE_SHIFT; pages += lruvec_page_state_local( From 7c0a645068deaf4c3a8c619ddbb8b84c477f9a9a Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Fri, 11 Sep 2026 16:00:50 +0800 Subject: [PATCH 266/539] mm: memcg: skip the RCU lock when the memcg is not dying get_non_dying_memcg_start() takes rcu_read_lock() on every stat update, but the lock only protects the upward walk to a non-dying ancestor, which happens solely while a memcg is being offlined. The dying check itself reads the CSS_DYING flag of a memcg the caller already holds a reference to, so it is safe without the lock. Check memcg_is_dying() first and return immediately when the memcg is alive, taking the RCU lock only on the rare dying path. On an anon fault/charge churn workload in a memcg this recovers the ~0.6% overhead added by the previous patch (4368077 vs 4343159 pages/s before, back to ~4377000 pages/s after). Link: https://lore.kernel.org/9ffdbdfc96312e3e13cb8f056bfe26649492d949.1789096175.git.zhuhui@kylinos.cn Fixes: 7404bd37cfbe ("mm: workingset: use lruvec_lru_size() to get the number of lru pages") Signed-off-by: Hui Zhu Signed-off-by: Andrew Morton Acked-by: Shakeel Butt Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Wei Xu Cc: Yuanchu Xie Cc: --- mm/memcontrol.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 8a571e80a103..307203301a2a 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -881,6 +881,17 @@ static long memcg_state_val_in_pages(int idx, long val) static inline struct mem_cgroup * get_non_dying_memcg_start(struct mem_cgroup *memcg, bool *rcu_locked) { + /* + * Fast path: the caller holds a reference to @memcg, so reading + * its CSS_DYING flag without the RCU lock is safe. The RCU lock + * is only needed to walk up to a non-dying ancestor, which + * happens only while a memcg is actually being offlined. + */ + if (!memcg_is_dying(memcg)) { + *rcu_locked = false; + return memcg; + } + rcu_read_lock(); *rcu_locked = true; @@ -892,6 +903,9 @@ get_non_dying_memcg_start(struct mem_cgroup *memcg, bool *rcu_locked) static inline void get_non_dying_memcg_end(bool rcu_locked) { + if (!rcu_locked) + return; + rcu_read_unlock(); } From 154c16a783b75105347cf5ed7731afc887308120 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Fri, 11 Sep 2026 16:00:51 +0800 Subject: [PATCH 267/539] mm: memcg: reparent non-hierarchical lruvec stats on cgroup v2 On cgroup v2, reparent_state_local() returns early and never moves the dying memcg's non-hierarchical state_local base counts to its parent. Meanwhile memcg_reparent_objcgs() rewrites objcg->memcg to the parent, so when the reparented folios are freed later, the negative deltas land on the parent's lruvec. The parent therefore receives the uncharges without ever having received the matching charges, and its state_local (NR_LRU_BASE + lru, MEMCG_SOCK, NR_SLAB_RECLAIMABLE_B, NR_SLAB_UNRECLAIMABLE_B) permanently underflows. Since lruvec_page_state_local() clamps negative values to zero, the underflow masks the parent's own legitimate pages. count_shadow_nodes() is the only reader of these non-hierarchical state_locals on cgroup v2, so the underflow directly distorts the workingset shadow node budget. Fix this by reparenting the lruvec state_locals on cgroup v2 as well, mirroring what cgroup v1 already does. Only the lruvec stats consumed by count_shadow_nodes() are moved; the memcg-level stats are left alone because on v2 they are exposed through the rstat hierarchical tree and are not read from state_local. Link: https://lore.kernel.org/4a7a64eed2b145ad535fedaea3624f8310c29d5b.1789096175.git.zhuhui@kylinos.cn Fixes: 8285917d6f38 ("mm: memcontrol: prepare for reparenting non-hierarchical stats") Signed-off-by: Hui Zhu Signed-off-by: Andrew Morton Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Cc: --- mm/memcontrol-v1.h | 5 +++-- mm/memcontrol.c | 42 ++++++++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index b9a21f0fd2c3..2cd37e1792d7 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -25,6 +25,9 @@ int memory_stat_show(struct seq_file *m, void *v); struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, unsigned int n); +void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg, + struct mem_cgroup *parent, int idx); + /* Cgroup v1-specific declarations */ #ifdef CONFIG_MEMCG_V1 @@ -67,8 +70,6 @@ void reparent_memcg1_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgr void reparent_memcg_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent, int idx); -void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg, - struct mem_cgroup *parent, int idx); void memcg1_account_kmem(struct mem_cgroup *memcg, int nr_pages); static inline bool memcg1_tcpmem_active(struct mem_cgroup *memcg) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 307203301a2a..cf53d4ac7ecc 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -233,14 +233,29 @@ static inline struct obj_cgroup *__memcg_reparent_objcgs(struct mem_cgroup *memc return objcg; } -#ifdef CONFIG_MEMCG_V1 static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force); +/* + * Reparent the non-hierarchical lruvec stats that count_shadow_nodes() reads + * to approximate the shadow node budget. They are not exposed to userspace + * on cgroup v2, but they must follow the reparented folios; otherwise the + * ancestor would only receive the negative deltas when the folios are freed + * without ever having received the positive base, and its local stats would + * permanently underflow. + */ +static void reparent_v2_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent) +{ + int i; + + for (i = 0; i < NR_LRU_LISTS; i++) + reparent_memcg_lruvec_state_local(memcg, parent, NR_LRU_BASE + i); + + reparent_memcg_lruvec_state_local(memcg, parent, NR_SLAB_RECLAIMABLE_B); + reparent_memcg_lruvec_state_local(memcg, parent, NR_SLAB_UNRECLAIMABLE_B); +} + static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent) { - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) - return; - /* * Reparent stats exposed non-hierarchically. Flush @memcg's stats first * to read its stats accurately , and conservatively flush @parent's @@ -249,17 +264,18 @@ static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgr */ __mem_cgroup_flush_stats(memcg, true); - /* The following counts are all non-hierarchical and need to be reparented. */ - reparent_memcg1_state_local(memcg, parent); - reparent_memcg1_lruvec_state_local(memcg, parent); + if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) { + reparent_v2_lruvec_state_local(memcg, parent); + } else { +#ifdef CONFIG_MEMCG_V1 + /* The following counts are all non-hierarchical and need to be reparented. */ + reparent_memcg1_state_local(memcg, parent); + reparent_memcg1_lruvec_state_local(memcg, parent); +#endif + } __mem_cgroup_flush_stats(parent, true); } -#else -static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent) -{ -} -#endif static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid) { @@ -571,7 +587,6 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec, return x; } -#ifdef CONFIG_MEMCG_V1 static void __mod_memcg_lruvec_state(struct mem_cgroup_per_node *pn, enum node_stat_item idx, long val); @@ -593,7 +608,6 @@ void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg, __mod_memcg_lruvec_state(parent_pn, idx, value); } } -#endif /* Subset of vm_event_item to report for memcg event stats */ static const unsigned int memcg_vm_event_stat[] = { From 52d95a55a0113408acd2d4ce0294a72c527f5092 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 18:49:58 +0300 Subject: [PATCH 268/539] mm/execmem: free ROX cache chunks only when they span an entire vm area Patch series "mm/execmem: fixes and cleanups for the ROX cache". Sashiko review of the ROX cache refill path asked what happens when the PMD_SIZE allocation fails there. Pulling that thread uncovered a bit more than the fallback. Patch 1 fixes a real bug, although rare bug: execmem_cache_clean() can free a chunk that is still partially in use, because a PMD sized and PMD aligned free range in the middle of a larger chunk looks exactly like a chunk that nobody uses. Patch 2 handles maple tree allocation failures in the cache. They are unlikely, but silently dropping an area from the tree is not a great way to deal with them. Patch 3 deals with the fallback that started all this. The cache exists to keep the direct map free of unnecessary splits, and the fallback happily filled it with base page mapped areas that could never be freed from the cache again. Ask vmalloc for a huge mapping or nothing, and serve whatever does not fit outside the cache. Patches 4 and 5 convert the ROX cache to scope based cleanup. The series was tested on x86 with module load/unload cycles, with PTDUMP confirming that the cache is using 2M mappings, and with nohugevmalloc to exercise the uncached fallback. This patch (of 5): When execmem refills the ROX cache, it vmalloc()s multiples of PMD_SIZE aligned to PMD_SIZE. For every such allocation vmalloc creates a vm area. The first part of the vmalloc()ed chunk is returned to the allocation that triggered the cache refill and the remaining part is added to the cache and handed out for subsequent allocations with execmem_alloc(). When only the first part is freed, the entire vm area remains in the ROX cache and can be handed out again. In the case when the first allocation is larger than PMD_SIZE and the second allocation from the freed first part of the chunk is exactly PMD_SIZE, execmem_cache_clean() will free the entire chunk while part of it is still in use. For example: /* * vmalloc(4M), return p0 to the caller * add [p0 + 3M, p0 + 4M) to the cache */ p0 = execmem_alloc(3M); /* return p0 + 3M from the cache to the caller */ p1 = execmem_alloc(1M); /* put [p0, p0 + 3M) back into the cache */ execmem_free(p0); /* return p0 from the cache to the caller */ p2 = execmem_alloc(2M); /* return p0 + 2M from the cache to the caller */ p3 = execmem_alloc(1M); /* bah! execmem_cache_clean() frees the entire 4M chunk */ execmem_free(p2); Make sure that the ranges that execmem_cache_clean() frees cover the entire vm area. Link: https://lore.kernel.org/20260903-execmem-rox-cache-pmd-v1-v1-0-11beb2a3d249@kernel.org Link: https://lore.kernel.org/20260903-execmem-rox-cache-pmd-v1-v1-1-11beb2a3d249@kernel.org Fixes: 2e45474ab14f ("execmem: add support for cache of large ROX pages") Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Assisted-by: copilot:claude-opus-5 Cc: Benjamin Tissoires Cc: Jiri Kosina Cc: Luis Chamberalin Cc: "Uladzislau Rezki (Sony)" Cc: --- mm/execmem.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/execmem.c b/mm/execmem.c index ad07cae9ed58..ba277790e313 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -143,9 +143,11 @@ static void execmem_cache_clean(struct work_struct *work) mutex_lock(mutex); mas_for_each(&mas, area, ULONG_MAX) { + struct vm_struct *vm = find_vm_area(area); size_t size = mas_range_len(&mas); - if (IS_ALIGNED(size, PMD_SIZE) && + if (vm && get_vm_area_size(vm) == size && + IS_ALIGNED(size, PMD_SIZE) && IS_ALIGNED(mas.index, PMD_SIZE)) { mas_store_gfp(&mas, NULL, GFP_KERNEL); vfree(area); From d80773cedde2eb66e5702d383b5eff0190e7693d Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 18:49:59 +0300 Subject: [PATCH 269/539] mm/execmem: handle potential allocation errors in the maple tree execmem_cache_clean() and execmem_cache_alloc_locked() ignore potential allocation failures in mas_store_gfp(). While in practice they are unlikely to happen, it's better to handle those errors and ensure the integrity of the ROX cache. Preallocate the maple tree nodes for the stores that must not fail and order the maple tree updates so that there won't be any failures once a tree has been modified. Link: https://lore.kernel.org/20260903-execmem-rox-cache-pmd-v1-v1-2-11beb2a3d249@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Assisted-by: copilot:claude-opus-5 Cc: Benjamin Tissoires Cc: Jiri Kosina Cc: Luis Chamberalin Cc: "Uladzislau Rezki (Sony)" --- mm/execmem.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/mm/execmem.c b/mm/execmem.c index ba277790e313..00dd6324cae0 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -149,7 +149,15 @@ static void execmem_cache_clean(struct work_struct *work) if (vm && get_vm_area_size(vm) == size && IS_ALIGNED(size, PMD_SIZE) && IS_ALIGNED(mas.index, PMD_SIZE)) { - mas_store_gfp(&mas, NULL, GFP_KERNEL); + /* + * Preallocate to ensure mas_store does not fail + * If there is no memory for the tree update, bail out, + * next execmem_free() might be more lucky + */ + if (mas_preallocate(&mas, NULL, GFP_KERNEL)) + break; + + mas_store_prealloc(&mas, NULL); vfree(area); } } @@ -219,30 +227,34 @@ static void *execmem_cache_alloc_locked(struct execmem_range *range, size_t size addr = mas_free.index; last = mas_free.last; + mas_set_range(&mas_free, addr, addr + size - 1); + if (mas_preallocate(&mas_free, NULL, GFP_KERNEL)) + return NULL; + /* insert allocated size to busy_areas at range [addr, addr + size) */ mas_set_range(&mas_busy, addr, addr + size - 1); err = mas_store_gfp(&mas_busy, (void *)addr, GFP_KERNEL); if (err) - return NULL; + goto err_destroy_mas_free; - mas_store_gfp(&mas_free, NULL, GFP_KERNEL); + mas_store_prealloc(&mas_free, NULL); if (area_size > size) { - void *ptr = (void *)(addr + size); - /* * re-insert remaining free size to free_areas at range * [addr + size, last] + * the range matches an existing entry, so this cannot allocate */ + ptr = (void *)(addr + size); mas_set_range(&mas_free, addr + size, last); - err = mas_store_gfp(&mas_free, ptr, GFP_KERNEL); - if (err) { - mas_store_gfp(&mas_busy, NULL, GFP_KERNEL); - return NULL; - } + mas_store_gfp(&mas_free, ptr, GFP_KERNEL); } ptr = (void *)addr; return ptr; + +err_destroy_mas_free: + mas_destroy(&mas_free); + return NULL; } static void *__execmem_cache_alloc(struct execmem_range *range, size_t size) From 569db54d8b7a4ec463a62c444de0d6f5f10e9dc1 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 18:50:00 +0300 Subject: [PATCH 270/539] mm/execmem: make sure ROX cache always contains multiples of PMD_SIZE The ROX cache relies on its chunks being PMD mapped. For a PMD mapped chunk, set_memory_rox() updates the direct map alias one PMD at a time and the large mappings there survive. When execmem refills the cache, it rounds up the requested allocation size to PMD_SIZE and tries to allocate that with vmalloc(VM_ALLOW_HUGE_VMAP). If that allocation fails, execmem falls back to vmalloc() of the original size. There are two issues with this approach: * If huge pages are not available, __vmalloc_node_range() silently falls back to base pages. The area execmem gets is virtually contiguous, but it is backed by 512 scattered base pages. Permission updates on such areas split large mappings in the direct map that contain those base pages, up to 512 PMD splits in the worst case. * execmem's own fallback adds a small base page mapped area to the cache. This adds the overhead of cache management to these allocations with no benefit of reducing fragmentation either in the vmalloc/modules address space or in the direct map. Worse, these areas are never freed from the cache, because execmem_cache_clean() releases only chunks that are a multiple of PMD_SIZE and aligned to PMD_SIZE, exactly to minimize the number of base page mappings. Add VM_REQUIRE_HUGE_VMAP option to vmalloc that fails if the allocation of huge pages fails or if such an allocation is not possible because huge page allocations in vmalloc were disabled or the architecture does not support them. Use this option when populating the ROX cache. If vmalloc(VM_REQUIRE_HUGE_VMAP) fails or vmalloc of huge pages is unavailable, handle the memory allocation outside the ROX cache with plain vmalloc(). Since the fallback allocation has to return ROX memory, add an execmem_alloc_rox() helper and use it for both populating the ROX cache and dealing with a fallback allocation in a ROX execmem_range. With that, the cache only ever contains PMD aligned chunks sized as a multiple of PMD_SIZE, and the PMD checks in execmem_cache_clean() become a VM_WARN_ON_ONCE() to ensure that the PMD mapping invariant does not change. Link: https://lore.kernel.org/20260903-execmem-rox-cache-pmd-v1-v1-3-11beb2a3d249@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Assisted-by: copilot:claude-opus-5 Cc: Benjamin Tissoires Cc: Jiri Kosina Cc: Luis Chamberalin Cc: "Uladzislau Rezki (Sony)" --- include/linux/vmalloc.h | 1 + mm/execmem.c | 76 +++++++++++++++++++++++++---------------- mm/vmalloc.c | 15 +++++++- 3 files changed, 62 insertions(+), 30 deletions(-) diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index aed121d729b0..6e555e31e622 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -38,6 +38,7 @@ struct iov_iter; /* in uio.h */ #define VM_DEFER_KMEMLEAK 0 #endif #define VM_SPARSE 0x00001000 /* sparse vm_area. not all pages are present. */ +#define VM_REQUIRE_HUGE_VMAP 0x00002000 /* huge page mapping or nothing */ /* bits [20..32] reserved for arch specific ioremap internals */ diff --git a/mm/execmem.c b/mm/execmem.c index 00dd6324cae0..77653b7f163d 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -51,7 +51,8 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, } if (!p) { - pr_warn_ratelimited("unable to allocate memory\n"); + if (!(vm_flags & VM_REQUIRE_HUGE_VMAP)) + pr_warn_ratelimited("unable to allocate memory\n"); return NULL; } @@ -146,9 +147,10 @@ static void execmem_cache_clean(struct work_struct *work) struct vm_struct *vm = find_vm_area(area); size_t size = mas_range_len(&mas); - if (vm && get_vm_area_size(vm) == size && - IS_ALIGNED(size, PMD_SIZE) && - IS_ALIGNED(mas.index, PMD_SIZE)) { + if (vm && get_vm_area_size(vm) == size) { + VM_WARN_ON_ONCE(!IS_ALIGNED(mas.index, PMD_SIZE) || + !IS_ALIGNED(size, PMD_SIZE)); + /* * Preallocate to ensure mas_store does not fail * If there is no memory for the tree update, bail out, @@ -264,38 +266,41 @@ static void *__execmem_cache_alloc(struct execmem_range *range, size_t size) return execmem_cache_alloc_locked(range, size); } -static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) +static void *execmem_vmalloc_rox(struct execmem_range *range, size_t size, + unsigned long vm_flags) { - unsigned long vm_flags = VM_ALLOW_HUGE_VMAP; - struct mutex *mutex = &execmem_cache.mutex; - struct vm_struct *vm; - size_t alloc_size; - int err = -ENOMEM; - void *p; - - alloc_size = round_up(size, PMD_SIZE); - p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags); - if (!p) { - alloc_size = size; - p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags); - } + void *p = execmem_vmalloc(range, size, PAGE_KERNEL, vm_flags); + int err; if (!p) return NULL; - vm = find_vm_area(p); - if (!vm) - goto err_free_mem; - /* fill memory with instructions that will trap */ - execmem_fill_trapping_insns(p, alloc_size); - + execmem_fill_trapping_insns(p, size); set_vm_flush_reset_perms(p); - - err = set_memory_rox((unsigned long)p, vm->nr_pages); + err = set_memory_rox((unsigned long)p, size >> PAGE_SHIFT); if (err) goto err_free_mem; + return p; + +err_free_mem: + vfree(p); + return NULL; +} + +static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) +{ + unsigned long vm_flags = VM_REQUIRE_HUGE_VMAP; + size_t alloc_size = round_up(size, PMD_SIZE); + struct mutex *mutex = &execmem_cache.mutex; + int err; + void *p; + + p = execmem_vmalloc_rox(range, alloc_size, vm_flags); + if (!p) + return NULL; + /* * New memory blocks must be allocated and added to the cache * as an atomic operation, otherwise they may be consumed @@ -317,6 +322,11 @@ err_free_mem: return NULL; } +static void *execmem_alloc_rox(struct execmem_range *range, size_t size) +{ + return execmem_vmalloc_rox(range, size, 0); +} + static void *execmem_cache_alloc(struct execmem_range *range, size_t size) { void *p; @@ -444,6 +454,11 @@ static void *execmem_cache_alloc(struct execmem_range *range, size_t size) return NULL; } +static void *execmem_alloc_rox(struct execmem_range *range, size_t size) +{ + return NULL; +} + static bool execmem_cache_free(void *ptr) { return false; @@ -453,17 +468,20 @@ static bool execmem_cache_free(void *ptr) void *execmem_alloc(enum execmem_type type, size_t size) { struct execmem_range *range = &execmem_info->ranges[type]; - bool use_cache = range->flags & EXECMEM_ROX_CACHE; + bool use_rox_cache = range->flags & EXECMEM_ROX_CACHE; unsigned long vm_flags = VM_FLUSH_RESET_PERMS; pgprot_t pgprot = range->pgprot; void *p = NULL; size = PAGE_ALIGN(size); - if (use_cache) + if (use_rox_cache) { p = execmem_cache_alloc(range, size); - else + if (!p) + p = execmem_alloc_rox(range, size); + } else { p = execmem_vmalloc(range, size, pgprot, vm_flags); + } return kasan_reset_tag(p); } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 395358b363fc..3845c7bd72b4 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4021,6 +4021,12 @@ static gfp_t vmalloc_fix_flags(gfp_t flags) * %__GFP_SKIP_KASAN can be used to skip unpoisoning of mapped pages * (when prot=%PAGE_KERNEL). * + * %VM_ALLOW_HUGE_VMAP allocates huge pages when possible and falls back to + * base pages if huge page allocation fails. + * + * %VM_REQUIRE_HUGE_VMAP implies %VM_ALLOW_HUGE_VMAP and fails instead of + * silently falling back to base pages. + * * Can not be called from interrupt nor NMI contexts. * Return: the address of the area or %NULL on failure */ @@ -4046,6 +4052,10 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, return NULL; } + /* VM_REQUIRE_HUGE_VMAP implies VM_ALLOW_HUGE_VMAP */ + if (vm_flags & VM_REQUIRE_HUGE_VMAP) + vm_flags |= VM_ALLOW_HUGE_VMAP; + if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) { /* * Try huge pages. Only try for PAGE_KERNEL allocations, @@ -4062,6 +4072,9 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, align = max(original_align, 1UL << shift); } + if ((vm_flags & VM_REQUIRE_HUGE_VMAP) && shift == PAGE_SHIFT) + return NULL; + again: area = __get_vm_area_node(size, align, shift, VM_ALLOC | VM_UNINITIALIZED | vm_flags, start, end, node, @@ -4136,7 +4149,7 @@ again: return area->addr; fail: - if (shift > PAGE_SHIFT) { + if (shift > PAGE_SHIFT && !(vm_flags & VM_REQUIRE_HUGE_VMAP)) { shift = PAGE_SHIFT; align = original_align; goto again; From 254e7a129e47084ba5710d85d605dc5bc56a06bf Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 18:50:01 +0300 Subject: [PATCH 271/539] mm/vmalloc: add DEFINE_FREE() for vfree() ... and use it in hid-core, the only place with a cleanup for a vmalloc() allocation. Link: https://lore.kernel.org/20260903-execmem-rox-cache-pmd-v1-v1-4-11beb2a3d249@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Cc: Benjamin Tissoires Cc: Jiri Kosina Cc: Luis Chamberalin Cc: "Uladzislau Rezki (Sony)" --- drivers/hid/hid-core.c | 4 ++-- include/linux/vmalloc.h | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index a3ff0514f9cd..ec7c2860c93e 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -944,7 +944,7 @@ static int hid_scan_report(struct hid_device *hid) hid_parser_reserved }; - struct hid_parser *parser __free(kvfree) = vzalloc(sizeof(*parser)); + struct hid_parser *parser __free(vfree) = vzalloc(sizeof(*parser)); if (!parser) return -ENOMEM; @@ -1265,7 +1265,7 @@ static int hid_parse_collections(struct hid_device *device) hid_parser_reserved }; - struct hid_parser *parser __free(kvfree) = vzalloc(sizeof(*parser)); + struct hid_parser *parser __free(vfree) = vzalloc(sizeof(*parser)); if (!parser) return -ENOMEM; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 6e555e31e622..034a693777ca 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -3,6 +3,7 @@ #define _LINUX_VMALLOC_H #include +#include #include #include #include @@ -215,6 +216,8 @@ void *__must_check vrealloc_node_align_noprof(const void *p, size_t size, extern void vfree(const void *addr); extern void vfree_atomic(const void *addr); +DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T)) + extern void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot); void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot); From c188936fc93ed525f57cba552514782b61c2fa37 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 18:50:02 +0300 Subject: [PATCH 272/539] mm/execmem: use cleanup infrastructure in ROX cache functions After splitting out execmem_alloc_rox() from execmem_cache_populate_alloc(), the error paths of both functions became less complex and can be easily switched to use the cleanup infrastructure. Use __free(vfree) to free allocated memory on the error paths and guard(mutex) for synchronization in ROX cache functions. Link: https://lore.kernel.org/20260903-execmem-rox-cache-pmd-v1-v1-5-11beb2a3d249@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Cc: Benjamin Tissoires Cc: Jiri Kosina Cc: Luis Chamberalin Cc: "Uladzislau Rezki (Sony)" --- mm/execmem.c | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/mm/execmem.c b/mm/execmem.c index 77653b7f163d..349cadd87486 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -138,11 +138,10 @@ int execmem_restore_rox(void *ptr, size_t size) static void execmem_cache_clean(struct work_struct *work) { struct maple_tree *free_areas = &execmem_cache.free_areas; - struct mutex *mutex = &execmem_cache.mutex; MA_STATE(mas, free_areas, 0, ULONG_MAX); void *area; - mutex_lock(mutex); + guard(mutex)(&execmem_cache.mutex); mas_for_each(&mas, area, ULONG_MAX) { struct vm_struct *vm = find_vm_area(area); size_t size = mas_range_len(&mas); @@ -163,7 +162,6 @@ static void execmem_cache_clean(struct work_struct *work) vfree(area); } } - mutex_unlock(mutex); } static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean); @@ -269,7 +267,7 @@ static void *__execmem_cache_alloc(struct execmem_range *range, size_t size) static void *execmem_vmalloc_rox(struct execmem_range *range, size_t size, unsigned long vm_flags) { - void *p = execmem_vmalloc(range, size, PAGE_KERNEL, vm_flags); + void *p __free(vfree) = execmem_vmalloc(range, size, PAGE_KERNEL, vm_flags); int err; if (!p) @@ -280,22 +278,17 @@ static void *execmem_vmalloc_rox(struct execmem_range *range, size_t size, set_vm_flush_reset_perms(p); err = set_memory_rox((unsigned long)p, size >> PAGE_SHIFT); if (err) - goto err_free_mem; + return NULL; - return p; - -err_free_mem: - vfree(p); - return NULL; + return no_free_ptr(p); } static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) { unsigned long vm_flags = VM_REQUIRE_HUGE_VMAP; size_t alloc_size = round_up(size, PMD_SIZE); - struct mutex *mutex = &execmem_cache.mutex; + void *p __free(vfree) = NULL; int err; - void *p; p = execmem_vmalloc_rox(range, alloc_size, vm_flags); if (!p) @@ -306,20 +299,15 @@ static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t si * as an atomic operation, otherwise they may be consumed * by a parallel call to the execmem_cache_alloc function. */ - mutex_lock(mutex); + guard(mutex)(&execmem_cache.mutex); err = execmem_cache_add_locked(p, alloc_size, GFP_KERNEL); - if (!err) - p = execmem_cache_alloc_locked(range, size); - mutex_unlock(mutex); - if (err) - goto err_free_mem; + return NULL; - return p; + /* the chunk belongs to the cache now */ + retain_and_null_ptr(p); -err_free_mem: - vfree(p); - return NULL; + return execmem_cache_alloc_locked(range, size); } static void *execmem_alloc_rox(struct execmem_range *range, size_t size) From 434114544b82a6292cc03d1dcbe8c11bf295958a Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 8 Sep 2026 11:16:31 -0400 Subject: [PATCH 273/539] mm/swap: add folio_swap_entry() and folio_page_swap_entry() Patch series "mm: remove page_swap_entry()", v2. A folio in the swap cache occupies folio_nr_pages() contiguous swap entries starting at folio->swap, so a page's swap entry is just folio->swap plus the page's index in the folio. The swap entry is folio state, but the only helper for it is page-based: page_swap_entry() takes a page and recomputes the folio its callers already hold. Several callers avoid it by open-coding the arithmetic on folio->swap instead. Add folio_swap_entry() and folio_page_swap_entry(), convert all users, and remove page_swap_entry(), with a few other cleanups along the way. This patch (of 8): A folio in the swap cache occupies folio_nr_pages() contiguous swap entries starting at folio->swap, so a page's swap entry is just folio->swap plus the page's index in the folio. page_swap_entry() hides this behind a compound_head() call, and callers that already have the folio sometimes open-code the arithmetic instead. Add folio_swap_entry(), which takes a folio and a page index, and folio_page_swap_entry() for callers that have the page. [tz2294@columbia.edu: adjust folio_swap_entry() kerneldoc summary, per David] Link: https://lore.kernel.org/20260913-folio_swap_entry-doc-fix-1@columbia.edu Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-0-ee6d01dfa5e1@columbia.edu Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-1-ee6d01dfa5e1@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Catalin Marinas Cc: Chengming Zhou Cc: Chris Li Cc: Dev Jain Cc: Harry Yoo Cc: Jann Horn Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Nhat Pham Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan --- include/linux/swap.h | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/include/linux/swap.h b/include/linux/swap.h index 74ce79404247..a59737b72681 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -272,6 +272,39 @@ struct swap_info_struct { const struct swap_ops *ops; }; +/** + * folio_swap_entry - Return the swap entry at a page index within a folio. + * @folio: The folio. + * @idx: The index of the page within the folio. + * + * A folio in the swap cache occupies folio_nr_pages() contiguous swap + * entries starting at folio->swap. The caller must ensure the folio is + * in the swap cache and that @idx is within the folio. + */ +static inline +swp_entry_t folio_swap_entry(const struct folio *folio, unsigned long idx) +{ + swp_entry_t entry = folio->swap; + + VM_WARN_ON_ONCE_FOLIO(idx >= folio_nr_pages(folio), folio); + entry.val += idx; + return entry; +} + +/** + * folio_page_swap_entry - Return the swap entry of a page in a folio. + * @folio: The folio containing @page. + * @page: A page within @folio. + * + * The caller must ensure the folio is in the swap cache and that @page + * is part of @folio. + */ +static inline swp_entry_t folio_page_swap_entry(const struct folio *folio, + const struct page *page) +{ + return folio_swap_entry(folio, folio_page_idx(folio, page)); +} + static inline swp_entry_t page_swap_entry(struct page *page) { struct folio *folio = page_folio(page); From 140c9dbdf6b2d6c87a76b987ecd5bfe0128976d6 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 8 Sep 2026 11:16:32 -0400 Subject: [PATCH 274/539] mm/huge_memory: add a comment to the open-coded swap entry The swap entry of each new folio in __split_folio_to_order() is computed by hand from folio->swap rather than with folio_swap_entry(), because the folio's page count is not valid while it is being split. Add a comment explaining this. Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-2-ee6d01dfa5e1@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Catalin Marinas Cc: Chengming Zhou Cc: Chris Li Cc: Dev Jain Cc: Harry Yoo Cc: Jann Horn Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Nhat Pham Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon --- mm/huge_memory.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index dd66c6ad5af1..009eb3adc2b7 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3762,6 +3762,10 @@ static void __split_folio_to_order(struct folio *folio, int old_order, */ VM_WARN_ON_ONCE_PAGE(new_folio->private, new_head); + /* + * Not all folio fields are valid during a split, so open-code + * the swap entry rather than using folio_swap_entry(). + */ if (folio_test_swapcache(folio)) new_folio->swap.val = folio->swap.val + i; From b9b589111ba3bc27e24e5f3643d21581a6978775 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 8 Sep 2026 11:16:33 -0400 Subject: [PATCH 275/539] mm/rmap: use folio_page_swap_entry() in ttu_anon_swapbacked_folio() We already have the folio here, so use folio_page_swap_entry() instead of going through page_swap_entry(). This saves a call to compound_head(). No functional change. Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-3-ee6d01dfa5e1@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Catalin Marinas Cc: Chengming Zhou Cc: Chris Li Cc: Dev Jain Cc: Harry Yoo Cc: Jann Horn Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Nhat Pham Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan --- mm/rmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/rmap.c b/mm/rmap.c index 0a3952706faf..5332c52909be 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2147,7 +2147,7 @@ static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma, { const bool anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); - swp_entry_t entry = page_swap_entry(page); + swp_entry_t entry = folio_page_swap_entry(folio, page); struct mm_struct *mm = vma->vm_mm; if (folio_dup_swap(folio, page) < 0) From 90a0486c6a123c3ebc5ad367c3a7907ce6ccc8bc Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 8 Sep 2026 11:16:34 -0400 Subject: [PATCH 276/539] mm/zswap: use folio_swap_entry() in zswap_store_page() zswap_store_page() open-codes the swap entry computation from folio->swap and the page index. Use folio_swap_entry() instead. No functional change. Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-4-ee6d01dfa5e1@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Catalin Marinas Cc: Chengming Zhou Cc: Chris Li Cc: Dev Jain Cc: Harry Yoo Cc: Jann Horn Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Nhat Pham Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan --- mm/zswap.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index b894de1786fd..3a6f89017646 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1395,8 +1395,7 @@ static bool zswap_store_page(struct folio *folio, long index, struct obj_cgroup *objcg, struct zswap_pool *pool) { - swp_entry_t page_swpentry = swp_entry(swp_type(folio->swap), - swp_offset(folio->swap) + index); + swp_entry_t page_swpentry = folio_swap_entry(folio, index); struct zswap_entry *entry, *old; /* allocate entry */ From 73b44d0e31e3d31c9c69bdc607aa457e7eb582e5 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 8 Sep 2026 11:16:35 -0400 Subject: [PATCH 277/539] mm/swapfile: use folio_page_swap_entry() folio_dup_swap() and folio_put_swap() open-code the swap entry computation from folio->swap and folio_page_idx(). Use folio_page_swap_entry() instead. No functional change. Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-5-ee6d01dfa5e1@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Catalin Marinas Cc: Chengming Zhou Cc: Chris Li Cc: Dev Jain Cc: Harry Yoo Cc: Jann Horn Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Nhat Pham Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan --- mm/swapfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 01e7b6b046b6..48d3cd40defd 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1822,7 +1822,7 @@ int folio_dup_swap(struct folio *folio, struct page *page) VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); if (page) { - entry.val += folio_page_idx(folio, page); + entry = folio_page_swap_entry(folio, page); nr_pages = 1; } @@ -1849,7 +1849,7 @@ void folio_put_swap(struct folio *folio, struct page *page) VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); if (page) { - entry.val += folio_page_idx(folio, page); + entry = folio_page_swap_entry(folio, page); nr_pages = 1; } From 694e2d63ec5194475a4216284469f03e489c8334 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 8 Sep 2026 11:16:36 -0400 Subject: [PATCH 278/539] arm64: mte: make mte_save_tags() and mte_restore_tags() static mte_save_tags() and mte_restore_tags() are only used by arch_prepare_to_swap() and arch_swap_restore() in mteswap.c. Make them static and remove their declarations from mte.h. No functional change. Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-6-ee6d01dfa5e1@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Catalin Marinas Cc: Chengming Zhou Cc: Chris Li Cc: Dev Jain Cc: Harry Yoo Cc: Jann Horn Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Nhat Pham Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan --- arch/arm64/include/asm/mte.h | 2 -- arch/arm64/mm/mteswap.c | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 7f7b97e09996..83f3b05fc784 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -23,9 +23,7 @@ unsigned long mte_copy_tags_from_user(void *to, const void __user *from, unsigned long n); unsigned long mte_copy_tags_to_user(void __user *to, void *from, unsigned long n); -int mte_save_tags(struct page *page); void mte_save_page_tags(const void *page_addr, void *tag_storage); -void mte_restore_tags(swp_entry_t entry, struct page *page); void mte_restore_page_tags(void *page_addr, const void *tag_storage); void mte_invalidate_tags(int type, pgoff_t offset); void mte_invalidate_tags_area(int type); diff --git a/arch/arm64/mm/mteswap.c b/arch/arm64/mm/mteswap.c index 63e8d72f202a..3c54afc8b758 100644 --- a/arch/arm64/mm/mteswap.c +++ b/arch/arm64/mm/mteswap.c @@ -20,7 +20,7 @@ void mte_free_tag_storage(char *storage) kfree(storage); } -int mte_save_tags(struct page *page) +static int mte_save_tags(struct page *page) { void *tag_storage, *ret; @@ -47,7 +47,7 @@ int mte_save_tags(struct page *page) return 0; } -void mte_restore_tags(swp_entry_t entry, struct page *page) +static void mte_restore_tags(swp_entry_t entry, struct page *page) { void *tags = xa_load(&mte_pages, entry.val); From 748ed3592d0df394c9f47e8a9ca5dec39f17bab8 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 8 Sep 2026 11:16:37 -0400 Subject: [PATCH 279/539] arm64: mte: pass the swap entry to mte_save_tags() arch_prepare_to_swap() derives a page from the folio only for mte_save_tags() to recompute the folio's swap entry from that page. Pass the entry in directly with folio_swap_entry(), matching mte_restore_tags(), and move the page_mte_tagged() check into the caller so we only compute the entry for tagged pages. __mte_invalidate_tags() loses its only user, so remove it and call mte_invalidate_tags() directly in the error path. This removes two calls to compound_head(). No functional change. Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-7-ee6d01dfa5e1@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Catalin Marinas Cc: Chengming Zhou Cc: Chris Li Cc: Dev Jain Cc: Harry Yoo Cc: Jann Horn Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Nhat Pham Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan --- arch/arm64/mm/mteswap.c | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/arch/arm64/mm/mteswap.c b/arch/arm64/mm/mteswap.c index 3c54afc8b758..f64202b69309 100644 --- a/arch/arm64/mm/mteswap.c +++ b/arch/arm64/mm/mteswap.c @@ -20,22 +20,17 @@ void mte_free_tag_storage(char *storage) kfree(storage); } -static int mte_save_tags(struct page *page) +static int mte_save_tags(swp_entry_t entry, struct page *page) { void *tag_storage, *ret; - if (!page_mte_tagged(page)) - return 0; - tag_storage = mte_allocate_tag_storage(); if (!tag_storage) return -ENOMEM; mte_save_page_tags(page_address(page), tag_storage); - /* lookup the swap entry.val from the page */ - ret = xa_store(&mte_pages, page_swap_entry(page).val, tag_storage, - GFP_KERNEL); + ret = xa_store(&mte_pages, entry.val, tag_storage, GFP_KERNEL); if (WARN(xa_is_err(ret), "Failed to store MTE tags")) { mte_free_tag_storage(tag_storage); return xa_err(ret); @@ -68,13 +63,6 @@ void mte_invalidate_tags(int type, pgoff_t offset) mte_free_tag_storage(tags); } -static inline void __mte_invalidate_tags(struct page *page) -{ - swp_entry_t entry = page_swap_entry(page); - - mte_invalidate_tags(swp_type(entry), swp_offset(entry)); -} - void mte_invalidate_tags_area(int type) { swp_entry_t entry = swp_entry(type, 0); @@ -102,15 +90,23 @@ int arch_prepare_to_swap(struct folio *folio) nr = folio_nr_pages(folio); for (i = 0; i < nr; i++) { - err = mte_save_tags(folio_page(folio, i)); + struct page *page = folio_page(folio, i); + + if (!page_mte_tagged(page)) + continue; + + err = mte_save_tags(folio_swap_entry(folio, i), page); if (err) goto out; } return 0; out: - while (i--) - __mte_invalidate_tags(folio_page(folio, i)); + while (i--) { + swp_entry_t swap = folio_swap_entry(folio, i); + + mte_invalidate_tags(swp_type(swap), swp_offset(swap)); + } return err; } From 1bf185edd550c3d88bb037524b7db010f6b860d7 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 8 Sep 2026 11:16:38 -0400 Subject: [PATCH 280/539] mm/swap: remove page_swap_entry() All callers have been converted to folio_swap_entry() and folio_page_swap_entry(), so remove it. Link: https://lore.kernel.org/20260908-folio_swap_entry-v2-8-ee6d01dfa5e1@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Catalin Marinas Cc: Chengming Zhou Cc: Chris Li Cc: Dev Jain Cc: Harry Yoo Cc: Jann Horn Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Nhat Pham Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan --- include/linux/swap.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index a59737b72681..a37ad8375e01 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -305,15 +305,6 @@ static inline swp_entry_t folio_page_swap_entry(const struct folio *folio, return folio_swap_entry(folio, folio_page_idx(folio, page)); } -static inline swp_entry_t page_swap_entry(struct page *page) -{ - struct folio *folio = page_folio(page); - swp_entry_t entry = folio->swap; - - entry.val += folio_page_idx(folio, page); - return entry; -} - /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; From d214267cd55d216d525db1e26dcc3b3cb308e2c0 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Tue, 8 Sep 2026 06:52:55 -0700 Subject: [PATCH 281/539] Docs/mm/damon/design: fix broken :ref: usage and a typo The Statistics section refers readers to the DAMON sysfs interface documentation for how to read the statistics, using a ':ref:' role. However, the role name has a superfluous 's' appended (':ref:s' instead of ':ref:'). Moreover, its target label 'sysfs_stats' does not exist; the correct label is 'sysfs_schemes_stats'. Fix both. Also fix a typo, 'frequenceis' -> 'frequencies', in the Adaptive Regions Adjustment section. Link: https://lore.kernel.org/20260908135257.97523-1-sj@kernel.org Signed-off-by: Zenghui Yu (Huawei) Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- Documentation/mm/damon/design.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index d036340dae8a..aac84de261aa 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -249,7 +249,7 @@ For each ``aggregation interval``, it compares the access frequencies sum of the two regions' sizes is smaller than the size of total regions divided by the ``minimum number of regions``, DAMON merges the two regions. If the resulting number of total regions is still higher than ``maximum number of -regions``, it repeats the merging with increasing access frequenceis difference +regions``, it repeats the merging with increasing access frequencies difference threshold until the upper-limit of the number of regions is met, or the threshold becomes higher than possible maximum value (``aggregation interval`` divided by ``sampling interval``). Then, after it reports and clears the @@ -889,7 +889,7 @@ the scheme is deactivated. Note that, unlike watermarks, even if a scheme's ``nr_snapshots`` reaches ``max_nr_snapshots``, monitoring will not stop. To know how user-space can read the stats via :ref:`DAMON sysfs interface -`, refer to :ref:s`stats ` part of the +`, refer to :ref:`stats ` part of the documentation. Regions Walking From 3453f37fed8e9588fa8f2f04fc20deed76814c1c Mon Sep 17 00:00:00 2001 From: Krishna Iyer Date: Tue, 8 Sep 2026 06:51:53 -0700 Subject: [PATCH 282/539] mm/damon: move damon_hugetlb_mkold() from vaddr to ops-common Patch series "mm/damon: support access monitoring of hugetlb-backed memory", v3. On virtualization hosts, most system memory is often backed by hugetlbfs. On our production hosts, for example, ~95% of RAM is 1 GiB hugetlb pages backing guest memory. DAMON's physical address space monitoring is blind to such memory: every access check starts at damon_get_folio(), which rejects folios that are not on the LRU lists, and hugetlb folios are managed outside of the LRU by design. As a result, all hugetlb-backed memory is silently reported as never accessed. In testing on a 1 TiB host, an hour of 4-thread random access over 842 GiB inside a guest was statistically indistinguishable from an idle host. The first patch moves damon_hugetlb_mkold() from vaddr to ops-common as a preparation. The second patch teaches the folio mkold/young rmap walkers to handle hugetlb folios, aging the huge PTE and notifying secondary MMUs across the whole huge page size; the secondary MMU notification is what surfaces guest-side (e.g., KVM/EPT) accessed bits. The third patch adds damon_get_monitor_folio() and uses it from the paddr monitoring primitives only. DAMOS action appliers such as DAMON_RECLAIM and DAMON_LRU_SORT keep the LRU-only lookup and are behaviorally unchanged. This series is the first half of an earlier six-patch series [1], split out as SJ suggested [2]. The second half (the 'aging_flush' TLB-flush-assisted aging) is deferred: we will gather more quantitative data on the gap it addresses, including the workload-side impact of the flushes and the working set measurement details SJ asked about, and post it separately once the data is in hand, aligned with the ongoing monitoring preparation actions work. Per Documentation/process/generated-content.rst, this series was developed with the assistance of an AI coding assistant (Anthropic Claude, via Claude Code). The assistant helped draft the code and changelogs, and applied the v1 review feedback. All changes were reviewed by the human submitter, who takes full responsibility for the contribution. The series as posted here was regression-tested on its base commit with a full x86_64 kernel build (no W=1 warnings in mm/damon), the DAMON kunit suite (41/41 passing) and the DAMON selftests (15/15 passing) on a kernel booted with virtme-ng. This patch (of 3): damon_hugetlb_mkold() clears the accessed bit of a hugetlb-mapping huge PTE and propagates the aging to secondary MMUs via mmu_notifier_clear_young(), spanning the whole huge page size. It currently lives in vaddr.c, and is thus usable only by the virtual address space monitoring operations set. The physical address space monitoring operations set will need the same logic, to support access monitoring of hugetlb-backed memory. Move the function to ops-common as-is, with no behavioral change. A follow-up change will use it from the folio-granular rmap walkers. Link: https://lore.kernel.org/20260908135156.97481-1-sj@kernel.org Link: https://lore.kernel.org/20260902025700.17975-2-kiyer@crusoe.ai Link: https://lore.kernel.org/20260908135156.97481-2-sj@kernel.org Signed-off-by: Krishna Iyer Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton Assisted-by: Claude:claude-fable-5 --- mm/damon/ops-common.c | 37 +++++++++++++++++++++++++++++++++++++ mm/damon/ops-common.h | 9 +++++++++ mm/damon/vaddr.c | 34 ---------------------------------- 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 7219c608b195..995cc1f3b9f3 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -3,6 +3,7 @@ * Common Code for Data Access Monitoring */ +#include #include #include #include @@ -103,6 +104,42 @@ void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ } +#ifdef CONFIG_HUGETLB_PAGE +static bool damon_hugetlb_ptep_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, pte_t *entry) +{ + unsigned long psize = huge_page_size(hstate_vma(vma)); + + if (!pte_young(*entry)) + return false; + *entry = huge_ptep_get_and_clear(mm, addr, pte, psize); + *entry = pte_mkold(*entry); + set_huge_pte_at(mm, addr, pte, *entry, psize); + return true; +} + +void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + bool referenced = false; + pte_t entry = huge_ptep_get(mm, addr, pte); + struct folio *folio = pfn_folio(pte_pfn(entry)); + + folio_get(folio); + + referenced = damon_hugetlb_ptep_mkold(pte, mm, vma, addr, &entry); + if (mmu_notifier_clear_young(mm, addr, + addr + huge_page_size(hstate_vma(vma)))) + referenced = true; + + if (referenced) + folio_set_young(folio); + + folio_set_idle(folio); + folio_put(folio); +} +#endif /* CONFIG_HUGETLB_PAGE */ + #define DAMON_MAX_SUBSCORE (100) #define DAMON_MAX_AGE_IN_LOG (32) diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h index 38d295488fa1..f7811c9c7a02 100644 --- a/mm/damon/ops-common.h +++ b/mm/damon/ops-common.h @@ -9,6 +9,15 @@ struct folio *damon_get_folio(unsigned long pfn); void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr); void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr); +#ifdef CONFIG_HUGETLB_PAGE +void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr); +#else +static inline void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ +} +#endif /* CONFIG_HUGETLB_PAGE */ void damon_folio_mkold(struct folio *folio); bool damon_folio_young(struct folio *folio); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 91a0d441c1f9..af9e1b82454c 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -283,40 +283,6 @@ out: } #ifdef CONFIG_HUGETLB_PAGE -static bool damon_hugetlb_ptep_mkold(pte_t *pte, struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long addr, pte_t *entry) -{ - unsigned long psize = huge_page_size(hstate_vma(vma)); - - if (!pte_young(*entry)) - return false; - *entry = huge_ptep_get_and_clear(mm, addr, pte, psize); - *entry = pte_mkold(*entry); - set_huge_pte_at(mm, addr, pte, *entry, psize); - return true; -} - -static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long addr) -{ - bool referenced = false; - pte_t entry = huge_ptep_get(mm, addr, pte); - struct folio *folio = pfn_folio(pte_pfn(entry)); - - folio_get(folio); - - referenced = damon_hugetlb_ptep_mkold(pte, mm, vma, addr, &entry); - if (mmu_notifier_clear_young(mm, addr, - addr + huge_page_size(hstate_vma(vma)))) - referenced = true; - - if (referenced) - folio_set_young(folio); - - folio_set_idle(folio); - folio_put(folio); -} - static int damon_mkold_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long end, struct mm_walk *walk) From ce1db7ec545b47210f431dc64018ea6ce3802a13 Mon Sep 17 00:00:00 2001 From: Krishna Iyer Date: Tue, 8 Sep 2026 06:51:54 -0700 Subject: [PATCH 283/539] mm/damon/ops-common: handle hugetlb folios in folio mkold/young rmap walkers damon_folio_mkold_one() and damon_folio_young_one() assume the folios they walk are mapped by normal PTEs or THP PMDs. When the folio is a hugetlb folio, page_vma_mapped_walk() returns the huge PTE in pvmw.pte with its page table lock held, but the walkers treat it as a normal PTE: they read and age it with PAGE_SIZE-granularity helpers, which is wrong for huge PTEs (up to PUD level), and notify secondary MMUs for only PAGE_SIZE of the mapping. Add hugetlb branches to both walkers. The mkold walker reuses damon_hugetlb_mkold(), which the virtual address space operations set has been using for hugetlb aging: it clears the young bit of the huge PTE via set_huge_pte_at() and calls mmu_notifier_clear_young() spanning the whole huge page size. The young walker gets an equivalent new helper, damon_hugetlb_young(), which reads the huge PTE with huge_ptep_get() and consults the page idle flag and mmu_notifier_test_young() like the existing PTE branch. Locking mirrors what page_vma_mapped_walk() provides: the huge PTE's page table lock is held inside the walk, and for shared hugetlb mappings (the only ones subject to huge PMD sharing), rmap_walk_file() already holds i_mmap_rwsem, satisfying hugetlb_walk()'s locking requirements. This is currently dead code: both rmap walkers are only reachable through damon_get_folio(), which rejects hugetlb folios since they are not on the LRU lists. A following commit will let the physical address space monitoring primitives opt in to hugetlb folios. Link: https://lore.kernel.org/20260902025700.17975-3-kiyer@crusoe.ai Link: https://lore.kernel.org/20260908135156.97481-3-sj@kernel.org Signed-off-by: Krishna Iyer Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton Assisted-by: Claude:claude-fable-5 --- mm/damon/ops-common.c | 61 +++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 995cc1f3b9f3..349e1604cc1b 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -205,10 +205,15 @@ static bool damon_folio_mkold_one(struct folio *folio, while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; - if (pvmw.pte) - damon_ptep_mkold(pvmw.pte, vma, addr); - else + if (pvmw.pte) { + if (folio_test_hugetlb(folio)) + damon_hugetlb_mkold(pvmw.pte, vma->vm_mm, vma, + addr); + else + damon_ptep_mkold(pvmw.pte, vma, addr); + } else { damon_pmdp_mkold(pvmw.pmd, vma, addr); + } } return true; } @@ -233,27 +238,55 @@ void damon_folio_mkold(struct folio *folio) } +#ifdef CONFIG_HUGETLB_PAGE +static bool damon_hugetlb_young(pte_t *pte, struct vm_area_struct *vma, + unsigned long addr, struct folio *folio) +{ + pte_t entry = huge_ptep_get(vma->vm_mm, addr, pte); + + return (pte_present(entry) && pte_young(entry)) || + !folio_test_idle(folio) || + mmu_notifier_test_young(vma->vm_mm, addr); +} +#else +static bool damon_hugetlb_young(pte_t *pte, struct vm_area_struct *vma, + unsigned long addr, struct folio *folio) +{ + return false; +} +#endif /* CONFIG_HUGETLB_PAGE */ + +static bool damon_pte_young(pte_t *pte, struct vm_area_struct *vma, + unsigned long addr, struct folio *folio) +{ + pte_t entry = ptep_get(pte); + + /* + * PFN swap PTEs, such as device-exclusive ones, that actually map + * pages are "old" from a CPU perspective. The MMU notifier takes care + * of any device aspects. + */ + return (pte_present(entry) && pte_young(entry)) || + !folio_test_idle(folio) || + mmu_notifier_test_young(vma->vm_mm, addr); +} + static bool damon_folio_young_one(struct folio *folio, struct vm_area_struct *vma, unsigned long addr, void *arg) { bool *accessed = arg; DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); - pte_t pte; *accessed = false; while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; if (pvmw.pte) { - pte = ptep_get(pvmw.pte); - - /* - * PFN swap PTEs, such as device-exclusive ones, that - * actually map pages are "old" from a CPU perspective. - * The MMU notifier takes care of any device aspects. - */ - *accessed = (pte_present(pte) && pte_young(pte)) || - !folio_test_idle(folio) || - mmu_notifier_test_young(vma->vm_mm, addr); + if (folio_test_hugetlb(folio)) + *accessed = damon_hugetlb_young(pvmw.pte, vma, + addr, folio); + else + *accessed = damon_pte_young(pvmw.pte, vma, + addr, folio); } else { #ifdef CONFIG_TRANSPARENT_HUGEPAGE pmd_t pmd = pmdp_get(pvmw.pmd); From e67684454bbedd7bb622cf1187b211223a7ebc65 Mon Sep 17 00:00:00 2001 From: Krishna Iyer Date: Tue, 8 Sep 2026 06:51:55 -0700 Subject: [PATCH 284/539] mm/damon/paddr: support hugetlb folios in access monitoring DAMON's physical address space monitoring is blind to hugetlb-backed memory. Every access check starts at damon_get_folio(), which rejects folios that are not on the LRU lists. Hugetlb folios are managed outside of the LRU by design, so every sampling attempt on hugetlb-backed memory silently fails and the pages are reported as never accessed. This is a significant blind spot on virtualization hosts. Cloud hypervisor hosts commonly back guest memory with 1 GiB hugetlbfs pages, covering the vast majority of the machine's memory. On such hosts, modules like DAMON_STAT observe only the host-side remainder (page cache, daemons) and report all guest working sets as permanently idle, defeating the purpose of host-level access monitoring. In testing on a 1 TiB host, an hour of 4-thread random access over 842 GiB inside a guest was statistically indistinguishable from an idle host, while a 40x smaller host-side workload produced a quantitatively correct response. Add damon_get_monitor_folio(), which additionally accepts hugetlb folios, and use it in the two paddr access monitoring primitives, damon_pa_mkold() and damon_pa_young(). With the previous commit teaching the folio-granular rmap walkers to age huge PTEs and to call the mmu notifiers spanning the whole huge page, this makes guest accesses visible through secondary MMU (e.g. KVM/EPT) young bits. Free hugetlb pool folios have a zero refcount, so folio_try_get() naturally keeps rejecting them. The DAMOS action appliers (damon_pa_pageout(), damon_pa_mark_accessed_or_deactivate(), damon_pa_migrate(), damon_pa_stat()) keep using damon_get_folio(): reclaim, LRU manipulation and migration cannot act on hugetlb folios, so their behavior is unchanged. Note that the access check granularity for hugetlb-backed memory is the huge page size: one touched byte reports the whole (up to 1 GiB) page as accessed. Also, DAMON now consumes secondary MMU young bits that KVM's own aging uses; at DAMON's sampling rate (one page per region per sampling interval) the interference is negligible. Link: https://lore.kernel.org/20260902025700.17975-4-kiyer@crusoe.ai Link: https://lore.kernel.org/20260908135156.97481-4-sj@kernel.org Signed-off-by: Krishna Iyer Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton Assisted-by: Claude:claude-fable-5 --- mm/damon/ops-common.c | 25 +++++++++++++++++++++---- mm/damon/ops-common.h | 1 + mm/damon/paddr.c | 4 ++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 349e1604cc1b..acf8f216c51c 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -15,14 +15,20 @@ #include "../internal.h" #include "ops-common.h" +static bool damon_folio_acceptable(struct folio *folio, bool monitor) +{ + return folio_test_lru(folio) || + (monitor && folio_test_hugetlb(folio)); +} + /* - * Get an online page for a pfn if it's in the LRU list. Otherwise, returns - * NULL. + * Get an online page for a pfn if it's in the LRU list, or a hugetlb folio if + * @monitor is set. Otherwise, returns NULL. * * The body of this function is stolen from the 'page_idle_get_folio()'. We * steal rather than reuse it because the code is quite simple. */ -struct folio *damon_get_folio(unsigned long pfn) +static struct folio *__damon_get_folio(unsigned long pfn, bool monitor) { struct page *page = pfn_to_online_page(pfn); struct folio *folio; @@ -33,13 +39,24 @@ struct folio *damon_get_folio(unsigned long pfn) folio = page_folio(page); if (!folio_try_get(folio)) return NULL; - if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) { + if (unlikely(page_folio(page) != folio) || + !damon_folio_acceptable(folio, monitor)) { folio_put(folio); folio = NULL; } return folio; } +struct folio *damon_get_folio(unsigned long pfn) +{ + return __damon_get_folio(pfn, false); +} + +struct folio *damon_get_monitor_folio(unsigned long pfn) +{ + return __damon_get_folio(pfn, true); +} + void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr) { pte_t pteval = ptep_get(pte); diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h index f7811c9c7a02..172f0f17c4a8 100644 --- a/mm/damon/ops-common.h +++ b/mm/damon/ops-common.h @@ -6,6 +6,7 @@ #include struct folio *damon_get_folio(unsigned long pfn); +struct folio *damon_get_monitor_folio(unsigned long pfn); void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr); void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index c1e7d7a4f40d..d2173a448d0b 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -37,7 +37,7 @@ static unsigned long damon_pa_core_addr( static void damon_pa_mkold(phys_addr_t paddr) { - struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); + struct folio *folio = damon_get_monitor_folio(PHYS_PFN(paddr)); if (!folio) return; @@ -67,7 +67,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) static bool damon_pa_young(phys_addr_t paddr) { - struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); + struct folio *folio = damon_get_monitor_folio(PHYS_PFN(paddr)); bool accessed; if (!folio) From 0aec9c957d2af48579af74ebab2be072b110ffbe Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Tue, 8 Sep 2026 21:41:15 +0800 Subject: [PATCH 285/539] selftests/mm: fix size truncation in pagemap_ioctl test Patch series "selftests/mm: pagemap_ioctl test fixes and cleanups", v2. This series fixes a size truncation bug in the pagemap_ioctl test that breaks it on arm64 systems with 64K base pages, and applies two small cleanups suggested during the review of the fix. This patch (of 3): On arm64 with 64K base pages, the huge page size is 512 MiB, and hpage_unit_tests() builds a 5 GiB range (10 * 512 MiB) for its tests. This exceeds the range of the int size parameters of gethugepage(), wp_addr_range() and pagemap_ioctl(). The implicit truncation to 1 GiB makes gethugepage() allocate a too small buffer, while the callers keep operating on the original 5 GiB range, resulting in spurious failures or SIGSEGV. Fix the truncation by changing those size parameters to size_t, and for consistency, also convert the remaining size-related parameters and variables that use int, long or unsigned long long to size_t. Link: https://lore.kernel.org/20260908134117.84405-1-zenghui.yu@linux.dev Link: https://lore.kernel.org/20260908134117.84405-2-zenghui.yu@linux.dev Fixes: 46fd75d4a3c9 ("selftests: mm: add pagemap ioctl tests") Signed-off-by: Zenghui Yu (Huawei) Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand (Arm) Acked-by: David Hildenbrand (Arm) Assisted-by: GLM-5.3 OpenCode Cc: Gregory Price Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/pagemap_ioctl.c | 55 ++++++++++++---------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index eadc7159ca5b..3665530eda76 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -44,7 +44,7 @@ const char *progname; #define LEN(region) ((region.end - region.start)/page_size) -static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag, +static long pagemap_ioctl(void *start, size_t len, void *vec, size_t vec_len, int flag, int max_pages, long required_mask, long anyof_mask, long excluded_mask, long return_mask) { @@ -65,7 +65,7 @@ static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); } -static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag, +static long pagemap_ioc(void *start, size_t len, void *vec, size_t vec_len, int flag, int max_pages, long required_mask, long anyof_mask, long excluded_mask, long return_mask, long *walk_end) { @@ -116,7 +116,7 @@ int init_uffd(void) return 0; } -int wp_init(void *addr, long size) +int wp_init(void *addr, size_t size) { struct uffdio_register uffdio_register; struct uffdio_writeprotect wp; @@ -140,7 +140,7 @@ int wp_init(void *addr, long size) return 0; } -int wp_free(void *addr, long size) +int wp_free(void *addr, size_t size) { struct uffdio_register uffdio_register; @@ -152,7 +152,7 @@ int wp_free(void *addr, long size) return 0; } -int wp_addr_range(void *addr, int size) +int wp_addr_range(void *addr, size_t size) { if (pagemap_ioctl(addr, size, NULL, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, @@ -162,7 +162,7 @@ int wp_addr_range(void *addr, int size) return 0; } -void *gethugetlb_mem(int size, int *shmid) +void *gethugetlb_mem(size_t size, int *shmid) { char *mem; @@ -188,7 +188,8 @@ void *gethugetlb_mem(int size, int *shmid) int userfaultfd_tests(void) { - long mem_size, vec_size, written, num_pages = 16; + size_t mem_size, vec_size, num_pages = 16; + long written; char *mem, *vec; mem_size = num_pages * page_size; @@ -229,9 +230,10 @@ int userfaultfd_tests(void) return 0; } -int get_reads(struct page_region *vec, int vec_size) +int get_reads(struct page_region *vec, size_t vec_size) { - int i, sum = 0; + size_t i; + int sum = 0; for (i = 0; i < vec_size; i++) sum += LEN(vec[i]); @@ -241,7 +243,7 @@ int get_reads(struct page_region *vec, int vec_size) int sanity_tests_sd(void) { - unsigned long long mem_size, vec_size, i, total_pages = 0; + size_t mem_size, vec_size, i, total_pages = 0; long ret, ret2, ret3; int num_pages = 1000; int total_writes, total_reads, reads, count; @@ -331,7 +333,7 @@ int sanity_tests_sd(void) if (ret < 0) ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno)); - ksft_test_result((unsigned long long)ret == mem_size/(page_size * 2), + ksft_test_result((size_t)ret == mem_size/(page_size * 2), "%s Repeated pattern of written and non-written pages\n", __func__); /* 4. Repeated pattern of written and non-written pages in parts */ @@ -682,9 +684,9 @@ int sanity_tests_sd(void) return 0; } -int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip) +int base_tests(char *prefix, char *mem, size_t mem_size, int skip) { - unsigned long long vec_size; + size_t vec_size; int written; struct page_region *vec, *vec2; @@ -787,7 +789,7 @@ int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip) return 0; } -void *gethugepage(int map_size) +void *gethugepage(size_t map_size) { int ret; char *map; @@ -810,8 +812,8 @@ int hpage_unit_tests(void) char *map; int ret, ret2; size_t num_pages = 10; - unsigned long long map_size = hpage_size * num_pages; - unsigned long long vec_size = map_size/page_size; + size_t map_size = hpage_size * num_pages; + size_t vec_size = map_size/page_size; struct page_region *vec, *vec2; vec = calloc(vec_size, sizeof(struct page_region)); @@ -1002,8 +1004,9 @@ int hpage_unit_tests(void) int unmapped_region_tests(void) { void *start = (void *)0x10000000; - int written, len = 0x00040000; - long vec_size = len / page_size; + int written; + size_t len = 0x00040000; + size_t vec_size = len / page_size; struct page_region *vec = calloc(vec_size, sizeof(struct page_region)); if (!vec) ksft_exit_fail_msg("error nomem\n"); @@ -1072,7 +1075,7 @@ static void test_simple(void) * with no page table, exercising pagemap_scan_pte_hole(); a base-page range * leaves pte_none entries. */ -static void unpopulated_written_test(const char *name, char *mem, long size, +static void unpopulated_written_test(const char *name, char *mem, size_t size, bool use_thp) { long npages = size / page_size, fast = 0, slow = 0, ret; @@ -1115,7 +1118,7 @@ out: static void unpopulated_scan_test(void) { - long mem_size = 16 * page_size; + size_t mem_size = 16 * page_size; char *mem; mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, @@ -1157,8 +1160,8 @@ static void unpopulated_thp_scan_test(void) int sanity_tests(void) { - unsigned long long mem_size, vec_size; - long ret, fd, i, buf_size, nr_pages; + size_t mem_size, vec_size, i, buf_size; + long ret, fd, nr_pages; struct page_region *vec; char *mem, *fmem; struct stat sbuf; @@ -1582,9 +1585,9 @@ static void transact_test(int page_size) void zeropfn_tests(void) { - unsigned long long mem_size; + size_t mem_size, i; struct page_region vec; - int i, ret; + int ret; char *mmap_mem, *mem; /* Test with normal memory */ @@ -1642,8 +1645,8 @@ void zeropfn_tests(void) int main(int __attribute__((unused)) argc, char *argv[]) { - int shmid, buf_size, fd, i, ret; - unsigned long long mem_size; + int shmid, fd, ret; + size_t mem_size, buf_size, i; char *mem, *map, *fmem; struct stat sbuf; From c4f08a284fec300525eac7a9fd0e718d75eddfb9 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Tue, 8 Sep 2026 21:43:14 +0800 Subject: [PATCH 286/539] selftests/mm: mark file-local symbols of pagemap_ioctl.c static The file-scope variables (pagemap_fd, uffd, page_size, hpage_size and progname) and most functions of the pagemap_ioctl test are only used locally, but lack the static storage class. Mark them static so that the compiler can catch accidental outer references. Link: https://lore.kernel.org/20260908134315.84431-1-zenghui.yu@linux.dev Signed-off-by: Zenghui Yu (Huawei) Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand (Arm) Acked-by: David Hildenbrand (Arm) Cc: Gregory Price Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/pagemap_ioctl.c | 43 +++++++++++----------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 3665530eda76..1a87b7483316 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -36,11 +36,11 @@ #define TEST_ITERATIONS 100 #define PAGEMAP "/proc/self/pagemap" -int pagemap_fd; -int uffd; -size_t page_size; -size_t hpage_size; -const char *progname; +static int pagemap_fd; +static int uffd; +static size_t page_size; +static size_t hpage_size; +static const char *progname; #define LEN(region) ((region.end - region.start)/page_size) @@ -92,8 +92,7 @@ static long pagemap_ioc(void *start, size_t len, void *vec, size_t vec_len, int return ret; } - -int init_uffd(void) +static int init_uffd(void) { struct uffdio_api uffdio_api; @@ -116,7 +115,7 @@ int init_uffd(void) return 0; } -int wp_init(void *addr, size_t size) +static int wp_init(void *addr, size_t size) { struct uffdio_register uffdio_register; struct uffdio_writeprotect wp; @@ -140,7 +139,7 @@ int wp_init(void *addr, size_t size) return 0; } -int wp_free(void *addr, size_t size) +static int wp_free(void *addr, size_t size) { struct uffdio_register uffdio_register; @@ -152,7 +151,7 @@ int wp_free(void *addr, size_t size) return 0; } -int wp_addr_range(void *addr, size_t size) +static int wp_addr_range(void *addr, size_t size) { if (pagemap_ioctl(addr, size, NULL, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, @@ -162,7 +161,7 @@ int wp_addr_range(void *addr, size_t size) return 0; } -void *gethugetlb_mem(size_t size, int *shmid) +static void *gethugetlb_mem(size_t size, int *shmid) { char *mem; @@ -186,7 +185,7 @@ void *gethugetlb_mem(size_t size, int *shmid) return mem; } -int userfaultfd_tests(void) +static int userfaultfd_tests(void) { size_t mem_size, vec_size, num_pages = 16; long written; @@ -230,7 +229,7 @@ int userfaultfd_tests(void) return 0; } -int get_reads(struct page_region *vec, size_t vec_size) +static int get_reads(struct page_region *vec, size_t vec_size) { size_t i; int sum = 0; @@ -241,7 +240,7 @@ int get_reads(struct page_region *vec, size_t vec_size) return sum; } -int sanity_tests_sd(void) +static int sanity_tests_sd(void) { size_t mem_size, vec_size, i, total_pages = 0; long ret, ret2, ret3; @@ -684,7 +683,7 @@ int sanity_tests_sd(void) return 0; } -int base_tests(char *prefix, char *mem, size_t mem_size, int skip) +static int base_tests(char *prefix, char *mem, size_t mem_size, int skip) { size_t vec_size; int written; @@ -789,7 +788,7 @@ int base_tests(char *prefix, char *mem, size_t mem_size, int skip) return 0; } -void *gethugepage(size_t map_size) +static void *gethugepage(size_t map_size) { int ret; char *map; @@ -807,7 +806,7 @@ void *gethugepage(size_t map_size) return map; } -int hpage_unit_tests(void) +static int hpage_unit_tests(void) { char *map; int ret, ret2; @@ -1001,7 +1000,7 @@ int hpage_unit_tests(void) return 0; } -int unmapped_region_tests(void) +static int unmapped_region_tests(void) { void *start = (void *)0x10000000; int written; @@ -1158,7 +1157,7 @@ static void unpopulated_thp_scan_test(void) munmap(area, 2 * hpage_size); } -int sanity_tests(void) +static int sanity_tests(void) { size_t mem_size, vec_size, i, buf_size; long ret, fd, nr_pages; @@ -1330,7 +1329,7 @@ int sanity_tests(void) return 0; } -int mprotect_tests(void) +static int mprotect_tests(void) { int ret; char *mem, *mem2; @@ -1450,7 +1449,7 @@ static ssize_t get_dirty_pages_reset(char *mem, unsigned int count, return cnt; } -void *thread_proc(void *mem) +static void *thread_proc(void *mem) { int *m = mem; long curr_faults, faults; @@ -1583,7 +1582,7 @@ static void transact_test(int page_size) extra_thread_faults); } -void zeropfn_tests(void) +static void zeropfn_tests(void) { size_t mem_size, i; struct page_region vec; From 59350eba932433e998803876a2bd59c32b270a6e Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Tue, 8 Sep 2026 21:44:05 +0800 Subject: [PATCH 287/539] selftests/mm: init page sizes early in pagemap_ioctl test Initialize page_size and hpage_size before calling init_uffd(), hugetlb_setup_default(), etc. That won't fix anything, but it is safer and saner to get these globals set up before doing other things. While at it, drop the page_size parameter of transact_test(), which is actually unnecessary. Link: https://lore.kernel.org/20260908134405.84448-1-zenghui.yu@linux.dev Signed-off-by: Zenghui Yu (Huawei) Signed-off-by: Andrew Morton Suggested-by: Andrew Morton Link: https://lore.kernel.org/20260628111329.9cfcd9c67925869307020aba@linux-foundation.org/ Cc: David Hildenbrand (Arm) Cc: Gregory Price Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/pagemap_ioctl.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 1a87b7483316..d9a4fb782ecf 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -1489,7 +1489,7 @@ static void *thread_proc(void *mem) return NULL; } -static void transact_test(int page_size) +static void transact_test(void) { unsigned int i, count, extra_pages; unsigned int c; @@ -1653,6 +1653,9 @@ int main(int __attribute__((unused)) argc, char *argv[]) ksft_print_header(); + page_size = getpagesize(); + hpage_size = read_pmd_pagesize(); + if (init_uffd()) ksft_exit_skip("Failed to initialize userfaultfd\n"); @@ -1661,9 +1664,6 @@ int main(int __attribute__((unused)) argc, char *argv[]) ksft_set_plan(119); - page_size = getpagesize(); - hpage_size = read_pmd_pagesize(); - pagemap_fd = open(PAGEMAP, O_RDONLY); if (pagemap_fd < 0) ksft_exit_fail_msg("Failed to open " PAGEMAP "\n"); @@ -1823,7 +1823,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) mprotect_tests(); /* 13. Transact test */ - transact_test(page_size); + transact_test(); /* 14. Sanity testing */ sanity_tests(); From 18767a7c475f7727833a51d8dcecfbeefeba6dbf Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Tue, 8 Sep 2026 14:28:21 +0100 Subject: [PATCH 288/539] mm/huge_memory: add folio_reset_partially_mapped() __folio_unqueue_deferred_split() and __folio_freeze_and_split_unmapped() both clear PG_partially_mapped and take the folio out of MTHP_STAT_NR_ANON_PARTIALLY_MAPPED with the same five lines. Move the block into folio_reset_partially_mapped() and call it from both places. The helper asserts what both callers rely on: the folio is frozen, so deferred_split_folio() cannot set the flag again under it, and the folio is already off the deferred split queue. The list check sits behind the flag test because order-1 folios have no _deferred_list. folio_order() is safe to use at this point in the split process: it still shows the pre-split order. Link: https://lore.kernel.org/20260908132821.1517475-1-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand (Arm) Acked-by: David Hildenbrand (Arm) Acked-by: Balbir Singh Reviewed-by: Zi Yan Reviewed-by: Baolin Wang Reviewed-by: Lorenzo Stoakes (ARM) Assisted-by: LLM --- mm/huge_memory.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 009eb3adc2b7..194188c292af 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3976,6 +3976,25 @@ static unsigned int folio_cache_ref_count(const struct folio *folio) return folio_nr_pages(folio); } +static void folio_reset_partially_mapped(struct folio *folio) +{ + /* Folio must be frozen. */ + VM_WARN_ON_FOLIO(folio_ref_count(folio), folio); + + if (!folio_test_partially_mapped(folio)) + return; + + /* + * Order-1 folios have no _deferred_list. The flag is only ever set + * on folios that do, so the list can be checked after the flag. + */ + VM_WARN_ON_FOLIO(!list_empty(&folio->_deferred_list), folio); + + folio_clear_partially_mapped(folio); + mod_mthp_stat(folio_order(folio), + MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); +} + static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int new_order, struct page *split_at, struct xa_state *xas, struct address_space *mapping, bool do_lru, @@ -3984,7 +4003,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n { struct folio *end_folio = folio_next(folio); struct folio *new_folio, *next; - int old_order = folio_order(folio); int ret = 0; VM_WARN_ON_ONCE(!mapping && end); @@ -4002,11 +4020,7 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n * leaves PG_partially_mapped set. * Clear it here: the flag does not survive the split. */ - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(old_order, - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } + folio_reset_partially_mapped(folio); if (mapping) { int nr = folio_nr_pages(folio); @@ -4520,11 +4534,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio) memcg = folio_memcg(folio); lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags); if (__list_lru_del(&deferred_split_lru, lru, &folio->_deferred_list, nid)) { - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(folio_order(folio), - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } + folio_reset_partially_mapped(folio); unqueued = true; } list_lru_unlock_irqrestore(lru, &flags); From eb823057e092132fe01bdbfc61d44343cd103777 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:36 +0100 Subject: [PATCH 289/539] mm/khugepaged: deposit a newly allocated page table on collapse Patch series "mm: make userland page table freeing RCU-safe", v5. The majority of architectures in the kernel defer page table freeing until an RCU grace period has elapsed, this series converts all remaining architectures to do so too and eliminates CONFIG_MMU_GATHER_RCU_TABLE_FREE altogether. This is important because it enables safe lockless page table walking under RCU alone. Doing so allows for reduced lock contention, avoids lock ordering concerns and enables fast, efficient and correct page table walking as a result. Additionally it removes a bunch of code and architecture-specific behaviour which is always a beneficial thing to do. There has been much recent work on this: * In 2023 Hugh Dickins RCU-deferred khugepaged page table retraction in commit 13cf577e6b66 ("mm/pgtable: add pte_free_defer() for pgtable as page"). * Qi Zheng has done most of the work that made this possible starting with the critical commit 718b13861d22 ("x86: mm: free page table pages by RCU instead of semi RCU"). * Qi then went on to convert a large number of architectures in commit e3ecf7c7d082 ("mm: pgtable: convert some architectures to use tlb_remove_ptdesc()"), commit 44b079583f7d ("alpha: mm: enable MMU_GATHER_RCU_TABLE_FREE") and the series to which it belongs. * Qi then introduced the important CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE option in commit 086498aed3f6 ("mm: convert __HAVE_ARCH_TLB_REMOVE_TABLE to CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE config"). * Finally, and critically, Lance Yang then converted the batch allocation fallback case to be RCU-safe in commit 1fb3d8c20bfa ("mm/mmu_gather: replace IPI with synchronize_rcu() when batch allocation fails"). The work I do here is only possible due to the work Hugh, Qi, Lance and others have done previously. An initial task this series addresses is to deposit a freshly allocated PTE page table and RCU-free the existing PTE page table. Not doing so is currently safe, but for page table walkers relying on RCU alone, it would not be. Additionally, it makes it possible to implement lockless RCU-only page table walkers which otherwise would have required the PTE PTL. The changes are largely mechanical - the majority of arches already have the machinery required to support CONFIG_MMU_GATHER_RCU_TABLE_FREE and simply needed configuration changes or small implementation changes to switch over. However some arches required extra attention - sh-X2, m68k-motorola and sparc32. sh-X2 allocates PMDs from the slab allocator and PTEs as normal. Therefore CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE is set to customise page table freeing and the LSB is used to encode which page table level is used, with __tlb_remove_table() doing the right thing depending on this. This pattern is repeated for m68k-motorola and sparc32 to account for different page table levels. In each case, the page tables are aligned such that sufficient bits are available in each case for encoding this information. m68k-motorola required the biggest change - since RCU page table freeing uses call_rcu(), this means page table freeing can arise from softirq context. This was fixed with a BH-disabling spin lock used in both get_pointer_table() and free_pointer_table() (softirq being the only asynchronous context in which the lock is taken). As part of this change, the logic for allocation of a new pointer table was separated out into add_pointer_table() to make the locking more obviously correct. Finally, sparc32 was similar to m68k-motorola in that locking was required, however this was already implemented via a spinlock, and only had to be updated to be IRQ-safe. Additionally, the nocache pool's bit_map lock was updated to be IRQ-safe for softirq frees. Separately, the PTE path can't take mm->page_table_lock from softirq (no mm there), which is fine because the page reference count transitions are atomic and fully ordered. The series finally removes CONFIG_MMU_GATHER_RCU_TABLE_FREE and all related configurations and code that supported !CONFIG_MMU_GATHER_RCU_TABLE_FREE. As a result, page table walks can now be performed safely under RCU without any risk of page tables being freed underneath a walker. However, this is the only guarantee that this work provides - page table walkers must still ensure that page table entries are as expected throughout. All changes have been build tested. As most of the conversions are simply utilising existing mechanics that are known to work, this suffices for most cases. However those arches where significant changes have been made - m68k-motorola, sparc32 and sh-X2 - have been tested further. For each of these a boot test and stress test has been performed - fork 400 children, each mmap()'ing 2 MiB and touching every page then partially munmap()'ing then exiting to trigger as much page table freeing as possible. All were found to be working correctly. (Note that sparc32 LEON SMP is not possible to emulate.) This patch (of 12): collapse_huge_page() deposits a PTE page table on PMD collapse in order that it can be utilised for subsequent split operations, meaning that those operations do not need to perform an allocation (as they are in a context where it might be unwise). However the PTE page table which is deposited is the one which is currently mapped by the PMD entry that is in the process of being collapsed. Once deposited, the PTE page table may be used in a split of any other unrelated PMD entry. This is currently not an issue as this operation is performed with VMA/mmap write lock + anon rmap locks held, so ordinary page table walkers will never accidentally end up walking the wrong thing, and GUP-fast is protected by an IPI via tlb_remove_table_sync_one(). However, the series to which this commit belongs implements RCU-safe page table traversal, at which point this becomes problematic. This can be resolved by using pte_offset_map_lock() which gates on a PTE PTL and a pmd_same() check, but lockless walks are unsafe as things stand. Resolve this by simply allocating a new, zeroed, PTE page table to deposit at the point of collapse. This path is already costly and an allocation has already been performed for the huge folio, so this allocation is statistical noise in terms of performance and memory usage at this point. With this PTE page table deposited, RCU-free the existing PTE page table so it is safe for page table walkers to traverse within a grace period. This also brings this deposit case in line with all other page table deposit logic which deposit a fresh page table. Additionally, this was the only place in the kernel that displaced a page table like this, so eliminating it also helps consistency. Since khugepaged runs as a kernel thread, do a little dance in alloc_deposit_pte_table() to correctly charge the allocation. This is already done for the folio allocation via alloc_charge_folio() but no such wrapper exists for a page table allocation. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-0-31e91065fea4@kernel.org Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-1-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Cc: Zi Yan Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Kiryl Shutsemau Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- mm/khugepaged.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 792166950bba..85ea095906fb 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1278,6 +1278,23 @@ static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_stru return SCAN_SUCCEED; } +static pgtable_t alloc_deposit_pte_table(struct mm_struct *mm) +{ + /* + * khugepaged is run from a kernel thread, so need to manually set the + * correct memcg so the allocation gets charged correctly. + */ + struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); + struct mem_cgroup *old_memcg = set_active_memcg(memcg); + pgtable_t pgtable; + + pgtable = pte_alloc_one(mm); + + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); + return pgtable; +} + /* * collapse_huge_page() expects the mmap_lock to be unlocked before entering and * will always return with the lock unlocked, to avoid holding the mmap_lock @@ -1293,7 +1310,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s LIST_HEAD(compound_pagelist); pmd_t *pmd, _pmd; pte_t *pte = NULL; - pgtable_t pgtable; + pgtable_t pgtable = NULL; struct folio *folio; spinlock_t *pmd_ptl, *pte_ptl; enum scan_result result = SCAN_FAIL; @@ -1310,6 +1327,14 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s goto out_nolock; } + if (is_pmd_order(order)) { + pgtable = alloc_deposit_pte_table(mm); + if (!pgtable) { + result = SCAN_ALLOC_HUGE_PAGE_FAIL; + goto out_nolock; + } + } + mmap_read_lock(mm); result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true, &vma, cc, order); @@ -1433,8 +1458,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s spin_lock(pmd_ptl); VM_WARN_ON_ONCE(!pmd_none(*pmd)); if (is_pmd_order(order)) { - pgtable = pmd_pgtable(_pmd); pgtable_trans_huge_deposit(mm, pmd, pgtable); + pgtable = NULL; map_anon_folio_pmd_nopf(folio, pmd, vma, pmd_addr); } else { /* @@ -1453,6 +1478,9 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s } spin_unlock(pmd_ptl); + if (is_pmd_order(order)) + pte_free_defer(mm, pmd_pgtable(_pmd)); + folio = NULL; result = SCAN_SUCCEED; @@ -1463,6 +1491,8 @@ out_up_write: anon_vma_unlock_write(vma->anon_vma); mmap_write_unlock(mm); out_nolock: + if (pgtable) + pte_free(mm, pgtable); if (folio) folio_put(folio); trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result, order); From d22254fd57f03d9a67db7e10702e19920e8d9fa5 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:37 +0100 Subject: [PATCH 290/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for most 2-level architectures Commit e3ecf7c7d082 ("mm: pgtable: convert some architectures to use tlb_remove_ptdesc()") updated a number of architectures from using pagetable_dtor() + tlb_remove_page_ptdesc() to using tlb_remove_ptdesc() in __pte_free_tlb(). This is meaningful as tlb_remove_ptdesc() allows for RCU page table freeing if CONFIG_MMU_GATHER_RCU_TABLE_FREE is specified. The csky, hexagon, nios2, openrisc, sh (except X2) and m68k-sun3 architectures all have 2 levels of page tables, so the only page tables ever freed by mmu_gather are PTEs, so this update suffices to ensure that every page table freed by the mmu_gather mechanism is freed under RCU. Therefore, update all of these architectures to select CONFIG_MMU_GATHER_RCU_TABLE_FREE. This forms part of an overall effort to switch every architecture to this mode. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-2-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Reviewed-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/csky/Kconfig | 1 + arch/hexagon/Kconfig | 1 + arch/m68k/Kconfig | 1 + arch/nios2/Kconfig | 1 + arch/openrisc/Kconfig | 1 + arch/sh/Kconfig | 1 + 6 files changed, 6 insertions(+) diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 4331313a42ff..80f89ef1d962 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -96,6 +96,7 @@ config CSKY select HAVE_SYSCALL_TRACEPOINTS select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU select LOCK_MM_AND_FIND_VMA + select MMU_GATHER_RCU_TABLE_FREE select MAY_HAVE_SPARSE_IRQ select MODULES_USE_ELF_RELA if MODULES select OF diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index b48491140013..d9b3fb86556b 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -23,6 +23,7 @@ config HEXAGON # select HAVE_CLK select GENERIC_ATOMIC64 select HAVE_PERF_EVENTS + select MMU_GATHER_RCU_TABLE_FREE # GENERIC_ALLOCATOR is used by dma_alloc_coherent() select GENERIC_ALLOCATOR select GENERIC_IRQ_PROBE diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 11835eb59d94..e29610fd1240 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -36,6 +36,7 @@ config M68K select HAVE_MOD_ARCH_SPECIFIC select HAVE_UID16 select MMU_GATHER_NO_RANGE if MMU + select MMU_GATHER_RCU_TABLE_FREE if MMU && SUN3 select MODULES_USE_ELF_REL select MODULES_USE_ELF_RELA select NO_DMA if !MMU && !COLDFIRE diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index 9c0e6eaeb005..b0ccfc3b7a7e 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -19,6 +19,7 @@ config NIOS2 select HAVE_PAGE_SIZE_4KB select IRQ_DOMAIN select LOCK_MM_AND_FIND_VMA + select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select OF select OF_EARLY_FLATTREE diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index 5eb995c13074..d90b24dd3bce 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -35,6 +35,7 @@ config OPENRISC select GENERIC_ATOMIC64 select GENERIC_CLOCKEVENTS_BROADCAST select GENERIC_SMP_IDLE_THREAD + select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select HAVE_DEBUG_STACKOVERFLOW select OR1K_PIC diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index d60f1d5a94c0..204f64912f0e 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -61,6 +61,7 @@ config SUPERH select HAVE_SYSCALL_TRACEPOINTS select IRQ_FORCED_THREADING select LOCK_MM_AND_FIND_VMA + select MMU_GATHER_RCU_TABLE_FREE if MMU && !X2TLB select MODULES_USE_ELF_RELA select NEED_SG_DMA_LENGTH select NO_DMA if !MMU && !DMA_COHERENT From 5ae75414416f35c92ebdfeb3854b5c110da95e09 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:38 +0100 Subject: [PATCH 291/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for MMU riscv Currently riscv gates MMU_GATHER_RCU_TABLE_FREE on CONFIG_SMP and CONFIG_MMU. Commit 69be3fb111e7 ("riscv: enable MMU_GATHER_RCU_TABLE_FREE for SMP && MMU") enabled CONFIG_MMU_GATHER_RCU_TABLE_FREE for CONFIG_SMP, CONFIG_MMU riscv builds. This is expressly for the safety of GUP-fast walkers (CONFIG_HAVE_GUP_FAST is enabled if CONFIG_MMU is enabled). Naturally a single core system does not encounter issues with software page table walkers being correctly synchronised across cores, as there is only a single core. However, CONFIG_PREEMPT_RCU is still available on a riscv UP system, so for a future RCU-only page table walker, this guarantee is required to prevent concurrent page table teardown. All page table freeing is already done via tlb_remove_ptdesc() so the conditions of CONFIG_MMU_GATHER_RCU_TABLE_FREE are already met. This forms part of an overall effort to switch every architecture to this mode. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-3-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Acked-by: Lance Yang Tested-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/riscv/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 5965666194b0..c13ef9cd688d 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -208,7 +208,7 @@ config RISCV select IRQ_FORCED_THREADING select KASAN_VMALLOC if KASAN select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE if SMP && MMU + select MMU_GATHER_RCU_TABLE_FREE if MMU select MODULES_USE_ELF_RELA if MODULES select OF select OF_EARLY_FLATTREE From 8e645d98fb7887944cfbad1a7de63925cd94f8a5 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:39 +0100 Subject: [PATCH 292/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for MMU arm Commit a0ad5496b2b3 ("arm: mm: enable HAVE_RCU_TABLE_FREE logic") enabled CONFIG_MMU_GATHER_RCU_TABLE_FREE (then named HAVE_RCU_TABLE_FREE) for SMP arm architectures with LPAE enabled. Regardless of whether CONFIG_ARM_LPAE is enabled or not, the same page table freeing functions __pte_free_tlb() and __pmd_free_tlb() are used. Non-LPAE PMD page tables are folded into the PGD and freed by pgd_free() (PGD freeing is not part of mmu_gather page table freeing in any case), so this is a noop in this case. Since commit 358d1c39c82a ("arm: convert various functions to use ptdescs") both LPAE and non-LPAE PTE page table freeing uses tlb_remove_ptdesc(). Thus all page table freeing is performed under RCU with CONFIG_MMU_GATHER_RCU_TABLE_FREE enabled for LPAE and non-LPAE and thus it need not be gated on LPAE. A UP arm system can set CONFIG_PREEMPT_RCU, so a future pure RCU page table walker requires MMU_GATHER_RCU_TABLE_FREE to be enabled on UP as well, even if concurrent GUP fast is not possible there. Therefore, it is both safe and desirable to set CONFIG_MMU_GATHER_RCU_TABLE_FREE for all MMU arm architectures (nommu does not perform mmu_gather operations). This forms part of an overall effort to switch every architecture to this mode. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-4-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Tested-by: Lance Yang Acked-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 408aa58a2a5b..72b9afc6ae10 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -134,7 +134,7 @@ config ARM select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE + select MMU_GATHER_RCU_TABLE_FREE if MMU select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ select HAVE_RUST if CPU_LITTLE_ENDIAN && CPU_32v7 && !KASAN From 799bd3c4be5f95d5a514d5a28ce8dbec7717ee23 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:40 +0100 Subject: [PATCH 293/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for arc, microblaze, xtensa Each of these architectures directly free page tables without routing these changes through tlb_remove_ptdesc(). The use of tlb_remove_ptdesc() is required for CONFIG_MMU_GATHER_RCU_TABLE_FREE to correctly free page tables under RCU, so simply update these architectures to use these functions. Since none of the architectures share page tables or do anything unusual, nothing complicated is required here. Therefore this is simply a mechanical change - convert __pud_free_tlb(), __pmd_free_tlb() and __pte_free_tlb() to use tlb_remove_ptdesc() as required. At the point this is in place, all mmu_gather page table freeing is performed under RCU, and thus MMU_GATHER_RCU_TABLE_FREE is selected for each architecture. Note that CONFIG_MMU_GATHER_RCU_TABLE_FREE is dependent on CONFIG_MMU for xtensa to reflect the fact that nommu does not implement page table gathering. This forms part of an overall effort to switch every architecture to this mode. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-5-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Acked-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/arc/Kconfig | 1 + arch/arc/include/asm/pgalloc.h | 6 +++--- arch/microblaze/Kconfig | 1 + arch/microblaze/include/asm/pgalloc.h | 2 +- arch/xtensa/Kconfig | 1 + arch/xtensa/include/asm/tlb.h | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 80e61175bf52..0c1679f4d386 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -47,6 +47,7 @@ config ARC select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select LOCK_MM_AND_FIND_VMA + select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select OF select OF_EARLY_FLATTREE diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h index dfae070fe8d5..9b6c37f92e97 100644 --- a/arch/arc/include/asm/pgalloc.h +++ b/arch/arc/include/asm/pgalloc.h @@ -72,7 +72,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) set_p4d(p4dp, __p4d((unsigned long)pudp)); } -#define __pud_free_tlb(tlb, pmd, addr) pud_free((tlb)->mm, pmd) +#define __pud_free_tlb(tlb, pmd, addr) tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd)) #endif @@ -83,10 +83,10 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) set_pud(pudp, __pud((unsigned long)pmdp)); } -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +#define __pmd_free_tlb(tlb, pmd, addr) tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd)) #endif -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) +#define __pte_free_tlb(tlb, pte, addr) tlb_remove_ptdesc((tlb), page_ptdesc(pte)) #endif /* _ASM_ARC_PGALLOC_H */ diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 484ebb3baedf..af7e821e96c1 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -41,6 +41,7 @@ config MICROBLAZE select PCI_SYSCALL if PCI select CPU_NO_EFFICIENT_FFS select MMU_GATHER_NO_RANGE + select MMU_GATHER_RCU_TABLE_FREE select SPARSE_IRQ select ZONE_DMA select TRACE_IRQFLAGS_SUPPORT diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h index 084a8a0dc239..ffee6a009219 100644 --- a/arch/microblaze/include/asm/pgalloc.h +++ b/arch/microblaze/include/asm/pgalloc.h @@ -25,7 +25,7 @@ extern void __bad_pte(pmd_t *pmd); extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm); -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, (pte)) +#define __pte_free_tlb(tlb, pte, addr) tlb_remove_ptdesc((tlb), page_ptdesc(pte)) #define pmd_populate(mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)page_address(pte)) diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index f2f9cd9cde50..33c4caee30e2 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -55,6 +55,7 @@ config XTENSA select HAVE_VIRT_CPU_ACCOUNTING_GEN select IRQ_DOMAIN select LOCK_MM_AND_FIND_VMA + select MMU_GATHER_RCU_TABLE_FREE if MMU select MODULES_USE_ELF_RELA select PERF_USE_VMALLOC select TRACE_IRQFLAGS_SUPPORT diff --git a/arch/xtensa/include/asm/tlb.h b/arch/xtensa/include/asm/tlb.h index 8c3ceb427018..6fb7b78154f6 100644 --- a/arch/xtensa/include/asm/tlb.h +++ b/arch/xtensa/include/asm/tlb.h @@ -16,7 +16,7 @@ #include -#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) +#define __pte_free_tlb(tlb, pte, address) tlb_remove_ptdesc((tlb), page_ptdesc(pte)) void check_tlb_sanity(void); From 461a24a8121598f6dea41d49065231a1a824d8da Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:41 +0100 Subject: [PATCH 294/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for sparc64 Commit 4a0100f7546f ("sparc64: use RCU page table freeing") enabled CONFIG_MMU_GATHER_RCU_TABLE_FREE for SMP sparc64 architectures, expressly for GUP-fast page table walkers. Naturally, UP systems do not have to worry about concurrent GUP fast operations. However, CONFIG_PREEMPT_RCU is also available even on a UP system, so a future pure-RCU page table walker requires MMU_GATHER_RCU_TABLE_FREE to be enabled on UP, even if concurrent GUP fast is not possible there. To enable future pure-RCU page table walkers, enable MMU_GATHER_RCU_TABLE_FREE unconditionally. With this change, it is no longer necessary to have !CONFIG_SMP pgtable_free_tlb(), so also remove this now dead code. This forms part of an overall effort to switch every architecture to this mode. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-6-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Tested-by: Lance Yang Acked-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/sparc/Kconfig | 4 ++-- arch/sparc/include/asm/pgalloc_64.h | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index ab77d3f2536e..8d42ebc6d302 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -75,8 +75,8 @@ config SPARC64 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_KRETPROBES select HAVE_KPROBES - select MMU_GATHER_RCU_TABLE_FREE if SMP - select HAVE_ARCH_TLB_REMOVE_TABLE if SMP + select MMU_GATHER_RCU_TABLE_FREE + select HAVE_ARCH_TLB_REMOVE_TABLE select MMU_GATHER_MERGE_VMAS select MMU_GATHER_NO_FLUSH_CACHE select HAVE_ARCH_TRANSPARENT_HUGEPAGE diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index caa7632be4c2..b5055d259b74 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -74,8 +74,6 @@ void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable); void pgtable_free(void *table, bool is_page); -#ifdef CONFIG_SMP - struct mmu_gather; void tlb_remove_table(struct mmu_gather *, void *); @@ -96,12 +94,6 @@ static inline void __tlb_remove_table(void *_table) is_page = true; pgtable_free(table, is_page); } -#else /* CONFIG_SMP */ -static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) -{ - pgtable_free(table, is_page); -} -#endif /* !CONFIG_SMP */ static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte, unsigned long address) From ccdaaf59166467297e17354cc9ec32a91dcd1f7a Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:42 +0100 Subject: [PATCH 295/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for m68k-coldfire Similar to sun3, the coldfire variant of m68k uses 2-level page tables. Update its __pte_free_tlb() function to use tlb_remove_ptdesc() in order that, with CONFIG_MMU_GATHER_RCU_TABLE_FREE, page tables are freed under RCU. The page tables occupy a page each and have no odd semantics, so this change suffices to allow enabling of CONFIG_MMU_GATHER_RCU_TABLE_FREE for m68k-coldfire, so do so. This forms part of an overall effort to switch every architecture to this mode. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-7-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Acked-by: Greg Ungerer Tested-by: Greg Ungerer Acked-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/m68k/Kconfig | 2 +- arch/m68k/include/asm/mcf_pgalloc.h | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index e29610fd1240..6b8ec67c86fd 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -36,7 +36,7 @@ config M68K select HAVE_MOD_ARCH_SPECIFIC select HAVE_UID16 select MMU_GATHER_NO_RANGE if MMU - select MMU_GATHER_RCU_TABLE_FREE if MMU && SUN3 + select MMU_GATHER_RCU_TABLE_FREE if MMU && (SUN3 || COLDFIRE) select MODULES_USE_ELF_REL select MODULES_USE_ELF_RELA select NO_DMA if !MMU && !COLDFIRE diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h index fc5454d37da3..b53ff0950db2 100644 --- a/arch/m68k/include/asm/mcf_pgalloc.h +++ b/arch/m68k/include/asm/mcf_pgalloc.h @@ -39,10 +39,7 @@ extern inline pmd_t *pmd_alloc_kernel(pgd_t *pgd, unsigned long address) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, unsigned long address) { - struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); - - pagetable_dtor(ptdesc); - pagetable_free(ptdesc); + tlb_remove_ptdesc(tlb, virt_to_ptdesc(pgtable)); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm) From a72df945113ad9bdffa159afd3667538e24f112a Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:43 +0100 Subject: [PATCH 296/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for sh-X2 Currently, non-x2 sh specifies CONFIG_MMU_GATHER_RCU_TABLE_FREE allowing RCU page table freeing. sh-X2 is problematic because it utilises slab-allocated PMD page tables, and thus tlb_remove_ptdesc() cannot be used in these cases. All other sh variants are fine as commit e3ecf7c7d082 ("mm: pgtable: convert some architectures to use tlb_remove_ptdesc()") already converted page table freeing to use tlb_remove_ptdesc(), which does so after an RCU grace period when CONFIG_MMU_GATHER_RCU_TABLE_FREE is specified. Resolve this issue by firstly specifying CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE for sh-X2, so the arch can provide its own __tlb_remove_table() implementation (called after the RCU grace period). Then, convert __pmd_free_tlb() to tag the pointer to the PMD, and have __tlb_remove_table() check this tag to determine whether to free via the slab or to use pagetable_dtor_free(). This follows the pattern used by sparc64 as implemented in commit 4a0100f7546f ("sparc64: use RCU page table freeing"). Previously __pmd_free_tlb() freed PMD page tables immediately, before any TLB flush IPI. This seems to be a pre-existing bug, which this change also resolves. CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE is only specified for sh-X2, as setting it disables CONFIG_PT_RECLAIM and causes __tlb_remove_table_one() to call tlb_remove_table_sync_rcu() and synchronize_rcu() in turn, and this is not necessary for other sh variants. This forms part of an overall effort to switch every architecture to this mode. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-8-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Acked-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/sh/Kconfig | 3 ++- arch/sh/include/asm/pgalloc.h | 6 +++++- arch/sh/mm/pgtable.c | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 204f64912f0e..75236bef6f16 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -33,6 +33,7 @@ config SUPERH select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_KGDB select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TLB_REMOVE_TABLE if X2TLB select HAVE_ARCH_TRACEHOOK select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK @@ -61,7 +62,7 @@ config SUPERH select HAVE_SYSCALL_TRACEPOINTS select IRQ_FORCED_THREADING select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE if MMU && !X2TLB + select MMU_GATHER_RCU_TABLE_FREE if MMU select MODULES_USE_ELF_RELA select NEED_SG_DMA_LENGTH select NO_DMA if !MMU && !DMA_COHERENT diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h index 6fe7123d38fa..67ce7fa23fa1 100644 --- a/arch/sh/include/asm/pgalloc.h +++ b/arch/sh/include/asm/pgalloc.h @@ -17,7 +17,11 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address); extern void pmd_free(struct mm_struct *mm, pmd_t *pmd); -#define __pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, (pmdp)) +extern void __tlb_remove_table(void *table); + +/* PMDs are slab-allocated, tag so they are freed correctly. */ +#define __pmd_free_tlb(tlb, pmdp, addr) \ + tlb_remove_table((tlb), (void *)((unsigned long)(pmdp) | 1)) #endif static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, diff --git a/arch/sh/mm/pgtable.c b/arch/sh/mm/pgtable.c index 3a4085ea0161..f6184b86b89c 100644 --- a/arch/sh/mm/pgtable.c +++ b/arch/sh/mm/pgtable.c @@ -56,4 +56,24 @@ void pmd_free(struct mm_struct *mm, pmd_t *pmd) { kmem_cache_free(pmd_cachep, pmd); } + +static void __tlb_remove_table_slab(void *table) +{ + kmem_cache_free(pmd_cachep, table); +} + +static void __tlb_remove_table_pgtable(void *table) +{ + pagetable_dtor_free(table); +} + +void __tlb_remove_table(void *table) +{ + const unsigned long addr = (unsigned long)table; + + if (addr & 1) + __tlb_remove_table_slab((void *)(addr & ~1UL)); + else + __tlb_remove_table_pgtable(table); +} #endif /* PAGETABLE_LEVELS > 2 */ From 0f5cbeebe00eeda81a8ef26b4a0c159c2240ca9a Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:44 +0100 Subject: [PATCH 297/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for m68k-motorola sun3 and coldfire are already supported, however motorola requires a little more care. Here, custom table removal logic is required, so CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE is enabled for m68k-motorola. Firstly as part of this change, the page table level must be communicated to the underlying __tlb_remove_table() implementation. Take advantage of the fact that page tables are aligned by more than enough to permit setting TABLE_PTE or TABLE_PMD in the low bits of the pointer, and store this there. Then update __pte_free_tlb() and __pmd_free_tlb() to pass this through, then have __tlb_remove_table() decode this and pass it to free_pointer_table(). The page table freeing is performed via call_rcu(), so free_pointer_table() now will be invoked from softirq context, and as such may be re-entrant. Introduce a spinlock to handle this and hold it over the time a given ptable entry is being referenced in both get_pointer_table() and free_pointer_table(). As softirq is the only asynchronous context in which the lock is taken, it suffices to disable bottom halves while holding it. In order to make things a little easier in this respect, separate out the logic for adding a new ptable entry into add_pointer_table() and only hold the lock during ptable entry insertion in this case. Note that original list_add_tail(new, dp) added new prior to dp, which is ptable_list[type].next, i.e. after ptable_list[type]. The equivalent therefore is list_add(new, &ptable_list[type]), which adds new after ptable_list[type], only without needing to make reference to dp. Note that, as m68k-motorola specifies CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE, it does not enable CONFIG_PT_RECLAIM. This isn't meaningfully impactful. With this applied, all of m68k implements CONFIG_MMU_GATHER_RCU_TABLE_FREE. This forms part of an overall effort to switch every architecture to this mode. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-9-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Acked-by: Lance Yang Tested-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/m68k/Kconfig | 3 +- arch/m68k/include/asm/motorola_pgalloc.h | 9 +- arch/m68k/mm/motorola.c | 119 +++++++++++++++-------- 3 files changed, 84 insertions(+), 47 deletions(-) diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 6b8ec67c86fd..fa5d39549da9 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -29,6 +29,7 @@ config M68K select HAVE_ARCH_LIBGCC_H select HAVE_ARCH_SECCOMP select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TLB_REMOVE_TABLE if MMU_MOTOROLA select HAVE_ASM_MODVERSIONS select HAVE_DEBUG_BUGVERBOSE select HAVE_EFFICIENT_UNALIGNED_ACCESS if !CPU_HAS_NO_UNALIGNED @@ -36,7 +37,7 @@ config M68K select HAVE_MOD_ARCH_SPECIFIC select HAVE_UID16 select MMU_GATHER_NO_RANGE if MMU - select MMU_GATHER_RCU_TABLE_FREE if MMU && (SUN3 || COLDFIRE) + select MMU_GATHER_RCU_TABLE_FREE if MMU select MODULES_USE_ELF_REL select MODULES_USE_ELF_RELA select NO_DMA if !MMU && !COLDFIRE diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h index 1091fb0affbe..dcde40e8b5c6 100644 --- a/arch/m68k/include/asm/motorola_pgalloc.h +++ b/arch/m68k/include/asm/motorola_pgalloc.h @@ -17,6 +17,7 @@ enum m68k_table_types { extern void init_pointer_table(void *table, int type); extern void *get_pointer_table(struct mm_struct *mm, int type); extern int free_pointer_table(void *table, int type); +extern void __tlb_remove_table(void *table); /* * Allocate and free page tables. The xxx_kernel() versions are @@ -47,7 +48,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, unsigned long address) { - free_pointer_table(pgtable, TABLE_PTE); + tlb_remove_table(tlb, (void *)((unsigned long)pgtable | TABLE_PTE)); } @@ -61,10 +62,10 @@ static inline int pmd_free(struct mm_struct *mm, pmd_t *pmd) return free_pointer_table(pmd, TABLE_PMD); } -static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, - unsigned long address) +static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long address) { - return free_pointer_table(pmd, TABLE_PMD); + tlb_remove_table(tlb, (void *)((unsigned long)pmd | TABLE_PMD)); } diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c index b30aa69a73a6..f3efa0d13963 100644 --- a/arch/m68k/mm/motorola.c +++ b/arch/m68k/mm/motorola.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -103,6 +104,8 @@ static struct list_head ptable_list[3] = { LIST_HEAD_INIT(ptable_list[2]), }; +static DEFINE_SPINLOCK(ptable_lock); + #define PD_PTABLE(ptdesc) ((ptable_desc *)&(virt_to_ptdesc((void *)(ptdesc))->pt_list)) #define PD_PTDESC(ptable) (list_entry(ptable, struct ptdesc, pt_list)) #define PD_MARKBITS(dp) (*(unsigned int *)&PD_PTDESC(dp)->pt_index) @@ -139,52 +142,65 @@ void __init init_pointer_table(void *table, int type) return; } +/* + * For a pointer table for a user process address space, a + * table is taken from a ptdesc allocated for the purpose. Each + * ptdesc can hold 8 pointer tables. The ptdesc is remapped in + * virtual address space to be noncacheable. + */ +static void *add_pointer_table(struct mm_struct *mm, int type) +{ + struct ptdesc *ptdesc; + ptable_desc *new; + void *pt_addr; + + ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); + if (!ptdesc) + return NULL; + + pt_addr = ptdesc_address(ptdesc); + + switch (type) { + case TABLE_PTE: + /* + * m68k doesn't have SPLIT_PTE_PTLOCKS for not having + * SMP. + */ + pagetable_pte_ctor(mm, ptdesc); + break; + case TABLE_PMD: + pagetable_pmd_ctor(mm, ptdesc); + break; + case TABLE_PGD: + pagetable_pgd_ctor(ptdesc); + break; + } + + mmu_page_ctor(pt_addr); + + new = PD_PTABLE(pt_addr); + + PD_MARKBITS(new) = ptable_mask(type) - 1; + scoped_guard(spinlock_bh, &ptable_lock) + list_add(new, &ptable_list[type]); + + return (pmd_t *)pt_addr; +} + void *get_pointer_table(struct mm_struct *mm, int type) { - ptable_desc *dp = ptable_list[type].next; - unsigned int mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp); unsigned int tmp, off; + unsigned long mask; + ptable_desc *dp; + void *ret; + + spin_lock_bh(&ptable_lock); + dp = ptable_list[type].next; + mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp); - /* - * For a pointer table for a user process address space, a - * table is taken from a ptdesc allocated for the purpose. Each - * ptdesc can hold 8 pointer tables. The ptdesc is remapped in - * virtual address space to be noncacheable. - */ if (mask == 0) { - struct ptdesc *ptdesc; - ptable_desc *new; - void *pt_addr; - - ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); - if (!ptdesc) - return NULL; - - pt_addr = ptdesc_address(ptdesc); - - switch (type) { - case TABLE_PTE: - /* - * m68k doesn't have SPLIT_PTE_PTLOCKS for not having - * SMP. - */ - pagetable_pte_ctor(mm, ptdesc); - break; - case TABLE_PMD: - pagetable_pmd_ctor(mm, ptdesc); - break; - case TABLE_PGD: - pagetable_pgd_ctor(ptdesc); - break; - } - - mmu_page_ctor(pt_addr); - - new = PD_PTABLE(pt_addr); - PD_MARKBITS(new) = ptable_mask(type) - 1; - list_add_tail(new, dp); - - return (pmd_t *)pt_addr; + spin_unlock_bh(&ptable_lock); + return add_pointer_table(mm, type); } for (tmp = 1, off = 0; (mask & tmp) == 0; tmp <<= 1, off += ptable_size(type)) @@ -194,7 +210,10 @@ void *get_pointer_table(struct mm_struct *mm, int type) /* move to end of list */ list_move_tail(dp, &ptable_list[type]); } - return ptdesc_address(PD_PTDESC(dp)) + off; + + ret = ptdesc_address(PD_PTDESC(dp)) + off; + spin_unlock_bh(&ptable_lock); + return ret; } int free_pointer_table(void *table, int type) @@ -204,6 +223,8 @@ int free_pointer_table(void *table, int type) unsigned long pt_addr = ptable & PAGE_MASK; unsigned int mask = 1U << ((ptable - pt_addr)/ptable_size(type)); + spin_lock_bh(&ptable_lock); + dp = PD_PTABLE(pt_addr); if (PD_MARKBITS (dp) & mask) panic ("table already free!"); @@ -213,6 +234,8 @@ int free_pointer_table(void *table, int type) if (PD_MARKBITS(dp) == ptable_mask(type)) { /* all tables in ptdesc are free, free ptdesc */ list_del(dp); + spin_unlock_bh(&ptable_lock); + mmu_page_dtor((void *)pt_addr); pagetable_dtor_free(virt_to_ptdesc((void *)pt_addr)); return 1; @@ -223,9 +246,21 @@ int free_pointer_table(void *table, int type) */ list_move(dp, &ptable_list[type]); } + + spin_unlock_bh(&ptable_lock); return 0; } +void __tlb_remove_table(void *table) +{ + /* The bottom 2 bits are used to encode page table type. */ + const unsigned long encoded = (unsigned long)table; + void *addr = (void *)(encoded & ~3UL); + const int type = encoded & 3; + + free_pointer_table(addr, type); +} + /* size of memory already mapped in head.S */ extern __initdata unsigned long m68k_init_mapped_size; From 35dd856770f31abc103e96b96c31050a4a83ac0d Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:45 +0100 Subject: [PATCH 298/539] mm: enable MMU_GATHER_RCU_TABLE_FREE for sparc32 Careful handling is required for sparc32 which implements page tables as part of a shared backing page. To support this, a custom __tlb_remove_table() function is required, as specified by CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE. This allows __pte_free_tlb() and __pmd_free_tlb() to specify which page table level is being freed, which is transmitted to __tlb_remove_table() through setting the lowest bit of the page table to 1 for a PMD and 0 for a PTE (the page tables are 256-byte aligned so this is safe to do). Next, since the page table freeing is done via RCU callback, and thus might be executed in softirq context, update the spin locks to IRQ save/restore. Then, in __tlb_remove_table(), figure out whether to free a PMD page table via free_pmd_fast() or a PTE via the newly introduced __pte_free() function, using the lower bit encoded in __pte_free_tlb() or __pmd_free_tlb() to determine which to call. As part of this change use this spin lock rather than mm->page_table_lock for all shared page table exclusion, as RCU freeing means that page tables can be freed from soft IRQ context so both don't have an mm and also mm->page_table_lock is not IRQ-safe. Note that the specification of CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE disables CONFIG_PT_RECLAIM for sparc32, which mirrors sparc64. This forms part of an overall effort to switch every architecture to this mode, and with it complete, means every architecture now supports CONFIG_MMU_GATHER_RCU_TABLE_FREE. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-10-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Tested-by: Lance Yang Acked-by: Lance Yang Cc: David Hildenbrand Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/sparc/Kconfig | 2 ++ arch/sparc/include/asm/pgalloc_32.h | 7 +++++-- arch/sparc/lib/bitext.c | 14 ++++++------- arch/sparc/mm/srmmu.c | 32 ++++++++++++++++++++++++----- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 8d42ebc6d302..79c09d6ee466 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -64,6 +64,8 @@ config SPARC32 select HAVE_UID16 select HAVE_PAGE_SIZE_4KB select LOCK_MM_AND_FIND_VMA + select MMU_GATHER_RCU_TABLE_FREE + select HAVE_ARCH_TLB_REMOVE_TABLE select OLD_SIGACTION select ZONE_DMA diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h index 4f73e87b22a3..36010852ba0c 100644 --- a/arch/sparc/include/asm/pgalloc_32.h +++ b/arch/sparc/include/asm/pgalloc_32.h @@ -48,7 +48,9 @@ static inline void free_pmd_fast(pmd_t * pmd) } #define pmd_free(mm, pmd) free_pmd_fast(pmd) -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) + +#define __pmd_free_tlb(tlb, pmd, addr) \ + tlb_remove_table((tlb), (void *)((unsigned long)(pmd) | 1UL)) #define pmd_populate(mm, pmd, pte) pmd_set(pmd, pte) @@ -72,6 +74,7 @@ static inline void free_pte_fast(pte_t *pte) #define pte_free_kernel(mm, pte) free_pte_fast(pte) void pte_free(struct mm_struct * mm, pgtable_t pte); -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) +void __tlb_remove_table(void *table); +#define __pte_free_tlb(tlb, pte, addr) tlb_remove_table((tlb), (void *)(pte)) #endif /* _SPARC_PGALLOC_H */ diff --git a/arch/sparc/lib/bitext.c b/arch/sparc/lib/bitext.c index 32a5c1d9459c..c309e27973ce 100644 --- a/arch/sparc/lib/bitext.c +++ b/arch/sparc/lib/bitext.c @@ -22,8 +22,6 @@ * @align: requested alignment * * Returns offset in the map or -1 if out of space. - * - * Not safe to call from an interrupt (uses spin_lock). */ int bit_map_string_get(struct bit_map *t, int len, int align) { @@ -31,6 +29,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align) int off_new; int align1; int i, color; + unsigned long flags; if (t->num_colors) { /* align is overloaded to be the page color */ @@ -50,7 +49,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align) BUG(); color &= align1; - spin_lock(&t->lock); + spin_lock_irqsave(&t->lock, flags); if (len < t->last_size) offset = t->first_free; else @@ -64,7 +63,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align) if (offset >= t->size) offset = 0; if (count + len > t->size) { - spin_unlock(&t->lock); + spin_unlock_irqrestore(&t->lock, flags); /* P3 */ printk(KERN_ERR "bitmap out: size %d used %d off %d len %d align %d count %d\n", t->size, t->used, offset, len, align, count); @@ -90,7 +89,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align) t->last_off = 0; t->used += len; t->last_size = len; - spin_unlock(&t->lock); + spin_unlock_irqrestore(&t->lock, flags); return offset; } } @@ -103,10 +102,11 @@ int bit_map_string_get(struct bit_map *t, int len, int align) void bit_map_clear(struct bit_map *t, int offset, int len) { int i; + unsigned long flags; if (t->used < len) BUG(); /* Much too late to do any good, but alas... */ - spin_lock(&t->lock); + spin_lock_irqsave(&t->lock, flags); for (i = 0; i < len; i++) { if (test_bit(offset + i, t->map) == 0) BUG(); @@ -115,7 +115,7 @@ void bit_map_clear(struct bit_map *t, int offset, int len) if (offset < t->first_free) t->first_free = offset; t->used -= len; - spin_unlock(&t->lock); + spin_unlock_irqrestore(&t->lock, flags); } void bit_map_init(struct bit_map *t, unsigned long *map, int size) diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index 9a74902ad181..1c277ab3cdb8 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -340,38 +340,60 @@ pgd_t *get_pgd_fast(void) * Alignments up to the page size are the same for physical and virtual * addresses of the nocache area. */ + +static DEFINE_SPINLOCK(pte_page_lock); + pgtable_t pte_alloc_one(struct mm_struct *mm) { + unsigned long flags; pte_t *ptep; struct page *page; if (!(ptep = pte_alloc_one_kernel(mm))) return NULL; page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); - spin_lock(&mm->page_table_lock); + spin_lock_irqsave(&pte_page_lock, flags); if (page_ref_inc_return(page) == 2 && !pagetable_pte_ctor(mm, page_ptdesc(page))) { page_ref_dec(page); ptep = NULL; } - spin_unlock(&mm->page_table_lock); + spin_unlock_irqrestore(&pte_page_lock, flags); return ptep; } -void pte_free(struct mm_struct *mm, pgtable_t ptep) +static void __pte_free(pgtable_t ptep) { struct page *page; + unsigned long flags; page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); - spin_lock(&mm->page_table_lock); + spin_lock_irqsave(&pte_page_lock, flags); if (page_ref_dec_return(page) == 1) pagetable_dtor(page_ptdesc(page)); - spin_unlock(&mm->page_table_lock); + spin_unlock_irqrestore(&pte_page_lock, flags); srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE); } +void pte_free(struct mm_struct *mm, pgtable_t ptep) +{ + __pte_free(ptep); +} + +void __tlb_remove_table(void *table) +{ + const unsigned long encoded = (unsigned long)table; + const unsigned long addr = encoded & ~1UL; + const bool is_pmd = encoded & 1; + + if (is_pmd) + free_pmd_fast((pmd_t *)addr); + else + __pte_free((pgtable_t)addr); +} + /* context handling - a dynamically sized pool is used */ #define NO_CONTEXT -1 From 35e340a47888e3574ee4e8ef7da3f73a32034e58 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:46 +0100 Subject: [PATCH 299/539] mm: userland pgtable freeing is RCU-safe now, remove leftover bits Now every architecture has been converted to support CONFIG_MMU_GATHER_RCU_TABLE_FREE, this configuration option no longer makes any sense to keep around. Therefore remove it, and remove all the dead code that existed for !CONFIG_MMU_GATHER_RCU_TABLE_FREE architectures previously. Additionally, CONFIG_MMU_GATHER_TABLE_FREE is no longer necessary, as all architectures instead use CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE when a custom __tlb_remove_table() is required, so remove this too. A number of architectures only enabled CONFIG_MMU_GATHER_RCU_TABLE_FREE if CONFIG_MMU was set, however the mmu_gather logic only actually does something meaningful if CONFIG_MMU is set (mmu_gather.c is only compiled in this case, for instance). As a result, there's no need to gate any of this logic on CONFIG_MMU explicitly. CONFIG_PT_RECLAIM however does have a strict dependency on CONFIG_MMU, so make this dependency explicit. Additionally, correct comments to remove references to non-RCU page table gathering and make it clear that this is not 'semi-RCU', nor has it been since commit 1fb3d8c20bfa ("mm/mmu_gather: replace IPI with synchronize_rcu() when batch allocation fails"). With this change in place the kernel policy is now that userspace page tables are freed after an RCU grace period, and thus it is now safe to unconditionally perform page table walks under RCU, safe in the knowledge that page tables will not be freed underneath the walker. This is all that is guaranteed, however, so naturally it is still incumbent upon page table walkers to ensure that the page table entries are as expected. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-11-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Reviewed-by: Lance Yang Acked-by: David Hildenbrand (Arm) Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- arch/Kconfig | 8 ---- arch/alpha/Kconfig | 1 - arch/arc/Kconfig | 1 - arch/arm/Kconfig | 1 - arch/arm64/Kconfig | 1 - arch/csky/Kconfig | 1 - arch/hexagon/Kconfig | 1 - arch/loongarch/Kconfig | 1 - arch/m68k/Kconfig | 1 - arch/microblaze/Kconfig | 1 - arch/mips/Kconfig | 1 - arch/nios2/Kconfig | 1 - arch/openrisc/Kconfig | 1 - arch/parisc/Kconfig | 1 - arch/powerpc/Kconfig | 1 - arch/riscv/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/sh/Kconfig | 1 - arch/sparc/Kconfig | 2 - arch/sparc/include/asm/tlb_64.h | 2 - arch/um/Kconfig | 1 - arch/x86/Kconfig | 1 - arch/xtensa/Kconfig | 1 - include/asm-generic/tlb.h | 66 +++++---------------------------- mm/Kconfig | 2 +- mm/gup.c | 5 ++- mm/mmu_gather.c | 30 +++------------ 27 files changed, 18 insertions(+), 117 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 45c657772362..6f7516916797 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -526,13 +526,6 @@ config HAVE_ARCH_JUMP_LABEL config HAVE_ARCH_JUMP_LABEL_RELATIVE bool -config MMU_GATHER_TABLE_FREE - bool - -config MMU_GATHER_RCU_TABLE_FREE - bool - select MMU_GATHER_TABLE_FREE - config MMU_GATHER_PAGE_SIZE bool @@ -548,7 +541,6 @@ config MMU_GATHER_MERGE_VMAS config MMU_GATHER_NO_GATHER bool - depends on MMU_GATHER_TABLE_FREE config ARCH_WANT_IRQS_OFF_ACTIVATE_MM bool diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index e53ef2d88463..9063c7bda4e4 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -42,7 +42,6 @@ config ALPHA select ARCH_STACKWALK select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67 select MMU_GATHER_NO_RANGE - select MMU_GATHER_RCU_TABLE_FREE select SPARSEMEM_EXTREME if SPARSEMEM select ZONE_DMA select TRACE_IRQFLAGS_SUPPORT diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 0c1679f4d386..80e61175bf52 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -47,7 +47,6 @@ config ARC select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select OF select OF_EARLY_FLATTREE diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 72b9afc6ae10..0cc289a7184a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -134,7 +134,6 @@ config ARM select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select MMU_GATHER_RCU_TABLE_FREE if MMU select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ select HAVE_RUST if CPU_LITTLE_ENDIAN && CPU_32v7 && !KASAN diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 2bbeded33da0..b6c2dd8b2612 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -221,7 +221,6 @@ config ARM64 select HAVE_RELIABLE_STACKTRACE select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_FUNCTION_ARG_ACCESS_API - select MMU_GATHER_RCU_TABLE_FREE select HAVE_RSEQ select HAVE_RUST if RUSTC_SUPPORTS_ARM64 select HAVE_STACKPROTECTOR diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 80f89ef1d962..4331313a42ff 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -96,7 +96,6 @@ config CSKY select HAVE_SYSCALL_TRACEPOINTS select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE select MAY_HAVE_SPARSE_IRQ select MODULES_USE_ELF_RELA if MODULES select OF diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index d9b3fb86556b..b48491140013 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -23,7 +23,6 @@ config HEXAGON # select HAVE_CLK select GENERIC_ATOMIC64 select HAVE_PERF_EVENTS - select MMU_GATHER_RCU_TABLE_FREE # GENERIC_ALLOCATOR is used by dma_alloc_coherent() select GENERIC_ALLOCATOR select GENERIC_IRQ_PROBE diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 1d8fb1e456d6..0bd8503fd5c4 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -188,7 +188,6 @@ config LOONGARCH select IRQ_LOONGARCH_CPU select LOCK_MM_AND_FIND_VMA select MMU_GATHER_MERGE_VMAS if MMU - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA if MODULES select NEED_PER_CPU_EMBED_FIRST_CHUNK select NEED_PER_CPU_PAGE_FIRST_CHUNK diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index fa5d39549da9..eb84c3af92c0 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -37,7 +37,6 @@ config M68K select HAVE_MOD_ARCH_SPECIFIC select HAVE_UID16 select MMU_GATHER_NO_RANGE if MMU - select MMU_GATHER_RCU_TABLE_FREE if MMU select MODULES_USE_ELF_REL select MODULES_USE_ELF_RELA select NO_DMA if !MMU && !COLDFIRE diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index af7e821e96c1..484ebb3baedf 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -41,7 +41,6 @@ config MICROBLAZE select PCI_SYSCALL if PCI select CPU_NO_EFFICIENT_FFS select MMU_GATHER_NO_RANGE - select MMU_GATHER_RCU_TABLE_FREE select SPARSE_IRQ select ZONE_DMA select TRACE_IRQFLAGS_SUPPORT diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index d7c67cebe065..ed17c9b1624c 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -97,7 +97,6 @@ config MIPS select IRQ_FORCED_THREADING select ISA if EISA select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_REL if MODULES select MODULES_USE_ELF_RELA if MODULES && 64BIT select PERF_USE_VMALLOC diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index b0ccfc3b7a7e..9c0e6eaeb005 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -19,7 +19,6 @@ config NIOS2 select HAVE_PAGE_SIZE_4KB select IRQ_DOMAIN select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select OF select OF_EARLY_FLATTREE diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index d90b24dd3bce..5eb995c13074 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -35,7 +35,6 @@ config OPENRISC select GENERIC_ATOMIC64 select GENERIC_CLOCKEVENTS_BROADCAST select GENERIC_SMP_IDLE_THREAD - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select HAVE_DEBUG_STACKOVERFLOW select OR1K_PIC diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index d3afac2f0d9b..77f67028ad89 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -80,7 +80,6 @@ config PARISC select GENERIC_CLOCKEVENTS select CPU_NO_EFFICIENT_FFS select THREAD_INFO_IN_TASK - select MMU_GATHER_RCU_TABLE_FREE select NEED_DMA_MAP_STATE select NEED_SG_DMA_LENGTH select HAVE_ARCH_KGDB diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 2580e27e4328..0767cfcbaa42 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -307,7 +307,6 @@ config PPC select KASAN_VMALLOC if KASAN && EXECMEM select LOCK_MM_AND_FIND_VMA select MMU_GATHER_PAGE_SIZE - select MMU_GATHER_RCU_TABLE_FREE select HAVE_ARCH_TLB_REMOVE_TABLE select MMU_GATHER_MERGE_VMAS select MMU_LAZY_TLB_SHOOTDOWN if PPC_BOOK3S_64 diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c13ef9cd688d..0db108ea1466 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -208,7 +208,6 @@ config RISCV select IRQ_FORCED_THREADING select KASAN_VMALLOC if KASAN select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE if MMU select MODULES_USE_ELF_RELA if MODULES select OF select OF_EARLY_FLATTREE diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index b88b85042136..a34376c05f6e 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -267,7 +267,6 @@ config S390 select LOCK_MM_AND_FIND_VMA select MMU_GATHER_MERGE_VMAS select MMU_GATHER_NO_GATHER - select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE if PCI select NEED_PER_CPU_EMBED_FIRST_CHUNK diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 75236bef6f16..fe859def918c 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -62,7 +62,6 @@ config SUPERH select HAVE_SYSCALL_TRACEPOINTS select IRQ_FORCED_THREADING select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE if MMU select MODULES_USE_ELF_RELA select NEED_SG_DMA_LENGTH select NO_DMA if !MMU && !DMA_COHERENT diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 79c09d6ee466..742ffff8c37f 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -64,7 +64,6 @@ config SPARC32 select HAVE_UID16 select HAVE_PAGE_SIZE_4KB select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE select HAVE_ARCH_TLB_REMOVE_TABLE select OLD_SIGACTION select ZONE_DMA @@ -77,7 +76,6 @@ config SPARC64 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_KRETPROBES select HAVE_KPROBES - select MMU_GATHER_RCU_TABLE_FREE select HAVE_ARCH_TLB_REMOVE_TABLE select MMU_GATHER_MERGE_VMAS select MMU_GATHER_NO_FLUSH_CACHE diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h index 3037187482db..f5f9631685d5 100644 --- a/arch/sparc/include/asm/tlb_64.h +++ b/arch/sparc/include/asm/tlb_64.h @@ -29,9 +29,7 @@ void flush_tlb_pending(void); * and therefore we don't need a TLBI when freeing page-table pages. */ -#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE #define tlb_needs_table_invalidate() (false) -#endif #include diff --git a/arch/um/Kconfig b/arch/um/Kconfig index d9541d13d9eb..94b8ff70f578 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -44,7 +44,6 @@ config UML select HAVE_SYSCALL_TRACEPOINTS select THREAD_INFO_IN_TASK select SPARSE_IRQ - select MMU_GATHER_RCU_TABLE_FREE config MMU bool diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index a8c3b3d31a27..6e5e462ec059 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -283,7 +283,6 @@ config X86 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select ASYNC_KERNEL_PGTABLE_FREE if IOMMU_SVA - select MMU_GATHER_RCU_TABLE_FREE select MMU_GATHER_MERGE_VMAS select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_REGS_AND_STACK_ACCESS_API diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 33c4caee30e2..f2f9cd9cde50 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -55,7 +55,6 @@ config XTENSA select HAVE_VIRT_CPU_ACCOUNTING_GEN select IRQ_DOMAIN select LOCK_MM_AND_FIND_VMA - select MMU_GATHER_RCU_TABLE_FREE if MMU select MODULES_USE_ELF_RELA select PERF_USE_VMALLOC select TRACE_IRQFLAGS_SUPPORT diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index bdcc2778ac64..9d827076db19 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -67,11 +67,8 @@ * - tlb_remove_table() * * tlb_remove_table() is the basic primitive to free page-table directories - * (__p*_free_tlb()). In it's most primitive form it is an alias for - * tlb_remove_page() below, for when page directories are pages and have no - * additional constraints. - * - * See also MMU_GATHER_TABLE_FREE and MMU_GATHER_RCU_TABLE_FREE. + * (__p*_free_tlb()). Page directories are freed after an RCU grace + * period - see the comment in mm/mmu_gather.c. * * - tlb_remove_page() / tlb_remove_page_size() * - __tlb_remove_folio_pages() / __tlb_remove_page_size() @@ -151,24 +148,15 @@ * This might be useful if your architecture has size specific TLB * invalidation instructions. * - * MMU_GATHER_TABLE_FREE + * Page directories (__p*_free_tlb()) are always freed via tlb_remove_table(), + * after an RCU grace period (see mm/mmu_gather.c). * - * This provides tlb_remove_table(), to be used instead of tlb_remove_page() - * for page directores (__p*_free_tlb()). + * This serialises against software page-table walkers, including architectures + * which do not use IPIs for remote TLB invalidates. * - * Useful if your architecture has non-page page directories. - * - * When used, an architecture is expected to provide __tlb_remove_table() or - * use the generic __tlb_remove_table(), which does the actual freeing of these - * pages. - * - * MMU_GATHER_RCU_TABLE_FREE - * - * Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see - * comment below). - * - * Useful if your architecture doesn't use IPIs for remote TLB invalidates - * and therefore doesn't naturally serialize with software page-table walkers. + * An architecture is expected to provide __tlb_remove_table() (see + * HAVE_ARCH_TLB_REMOVE_TABLE) or use the generic __tlb_remove_table(), which + * does the actual freeing of these pages. * * MMU_GATHER_NO_FLUSH_CACHE * @@ -200,12 +188,8 @@ * various ptep_get_and_clear() functions. */ -#ifdef CONFIG_MMU_GATHER_TABLE_FREE - struct mmu_table_batch { -#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE struct rcu_head rcu; -#endif unsigned int nr; void *tables[]; }; @@ -224,23 +208,6 @@ static inline void __tlb_remove_table(void *table) extern void tlb_remove_table(struct mmu_gather *tlb, void *table); -#else /* !CONFIG_MMU_GATHER_TABLE_FREE */ - -static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page); -/* - * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based - * page directories and we can use the normal page batching to free them. - */ -static inline void tlb_remove_table(struct mmu_gather *tlb, void *table) -{ - struct ptdesc *ptdesc = (struct ptdesc *)table; - - pagetable_dtor(ptdesc); - tlb_remove_page(tlb, ptdesc_page(ptdesc)); -} -#endif /* CONFIG_MMU_GATHER_TABLE_FREE */ - -#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE /* * This allows an architecture that does not use the linux page-tables for * hardware to skip the TLBI when freeing page tables. @@ -253,19 +220,6 @@ void tlb_remove_table_sync_one(void); void tlb_remove_table_sync_rcu(void); -#else - -#ifdef tlb_needs_table_invalidate -#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE -#endif - -static inline void tlb_remove_table_sync_one(void) { } - -static inline void tlb_remove_table_sync_rcu(void) { } - -#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */ - - #ifndef CONFIG_MMU_GATHER_NO_GATHER /* * If we can't allocate a page to make a big batch of page pointers @@ -325,9 +279,7 @@ static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct struct mmu_gather { struct mm_struct *mm; -#ifdef CONFIG_MMU_GATHER_TABLE_FREE struct mmu_table_batch *batch; -#endif unsigned long start; unsigned long end; diff --git a/mm/Kconfig b/mm/Kconfig index c1ddf59c0d71..bc7befafb47b 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1465,7 +1465,7 @@ config HAVE_ARCH_TLB_REMOVE_TABLE config PT_RECLAIM def_bool y - depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE + depends on MMU && !HAVE_ARCH_TLB_REMOVE_TABLE help Try to reclaim empty user page table pages in paths other than munmap and exit_mmap path. diff --git a/mm/gup.c b/mm/gup.c index a4036c02e213..c2dfcb4744bc 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2700,8 +2700,9 @@ EXPORT_SYMBOL(get_user_pages_unlocked); * Before activating this code, please be aware that the following assumptions * are currently made: * - * *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to - * free pages containing page tables or TLB flushing requires IPI broadcast. + * *) tlb_remove_table() is used to free pages containing page tables, with + * the free deferred until an RCU grace period has elapsed (see + * mm/mmu_gather.c). * * *) ptes can be read atomically by the architecture. * diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 3985d856de7f..2a72a9686773 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -218,8 +218,6 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_ #endif /* MMU_GATHER_NO_GATHER */ -#ifdef CONFIG_MMU_GATHER_TABLE_FREE - static void __tlb_remove_table_free(struct mmu_table_batch *batch) { int i; @@ -230,10 +228,8 @@ static void __tlb_remove_table_free(struct mmu_table_batch *batch) free_page((unsigned long)batch); } -#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE - /* - * Semi RCU freeing of the page directories. + * RCU freeing of the page directories. * * This is needed by some architectures to implement software pagetable walkers. * @@ -259,13 +255,13 @@ static void __tlb_remove_table_free(struct mmu_table_batch *batch) * means. * * What we do is batch the freed directory pages (tables) and RCU free them. - * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling - * holds off grace periods. + * Disabling IRQs or preemption holds off RCU grace periods, so this protects + * both rcu_read_lock() and IRQ-disabling walkers. * * However, in order to batch these pages we need to allocate storage, this * allocation is deep inside the MM code and can thus easily fail on memory - * pressure. To guarantee progress we fall back to single table freeing, see - * the implementation of tlb_remove_table_one(). + * pressure. To guarantee progress we fall back to single table freeing, which + * is also RCU-deferred - see the implementation of tlb_remove_table_one(). * */ @@ -315,15 +311,6 @@ void tlb_remove_table_sync_rcu(void) synchronize_rcu(); } -#else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */ - -static void tlb_remove_table_free(struct mmu_table_batch *batch) -{ - __tlb_remove_table_free(batch); -} - -#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */ - /* * If we want tlb_remove_table() to imply TLB invalidates. */ @@ -403,13 +390,6 @@ static inline void tlb_table_init(struct mmu_gather *tlb) tlb->batch = NULL; } -#else /* !CONFIG_MMU_GATHER_TABLE_FREE */ - -static inline void tlb_table_flush(struct mmu_gather *tlb) { } -static inline void tlb_table_init(struct mmu_gather *tlb) { } - -#endif /* CONFIG_MMU_GATHER_TABLE_FREE */ - static void tlb_flush_mmu_free(struct mmu_gather *tlb) { tlb_table_flush(tlb); From a5c0a39d0e573740b124cfabebef6f5f3573bf24 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 25 Sep 2026 21:09:47 +0100 Subject: [PATCH 300/539] mm: change the contract for free_pgtables(), update docs Now that page tables are freed after an RCU grace period, it is safe for read-only page table walkers to walk page table ranges that are being concurrently torn down, provided the mm is kept alive via mmgrab(). It is however unsafe for writers to do so, as they must obtain an appropriate lock to do so safely. Update the pte_offset_map_lock()'s comment block to reflect this. Similarly update the process addresses documentation. Link: https://lore.kernel.org/20260925-rcu-pagetable-freeing-v5-12-31e91065fea4@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Cc: Zi Yan Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Lance Yang Cc: Usama Arif Cc: Guo Ren Cc: Brian Cain Cc: Geert Uytterhoeven Cc: Dinh Nguyen Cc: Simon Schuster Cc: Jonas Bonn Cc: Stefan Kristiansson Cc: Stafford Horne Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Alexandre Ghiti Cc: Russell King Cc: Vineet Gupta Cc: Michal Simek Cc: Chris Zankel Cc: Max Filippov Cc: Will Deacon Cc: Aneesh Kumar K.V Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: David S. Miller Cc: Andreas Larsson Cc: Richard Henderson Cc: Matt Turner Cc: Magnus Lindholm Cc: Catalin Marinas Cc: Mark Rutland Cc: Huacai Chen Cc: WANG Xuerui Cc: Thomas Bogendoerfer Cc: James Bottomley Cc: Helge Deller Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Christophe Leroy Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: H. Peter Anvin Cc: Arnd Bergmann Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Yoshinori Sato Cc: Shakeel Butt Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Hugh Dickins Cc: Qi Zheng --- Documentation/mm/process_addrs.rst | 11 +++++++++++ mm/pgtable-generic.c | 15 +++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Documentation/mm/process_addrs.rst b/Documentation/mm/process_addrs.rst index a7296f251799..78231995e490 100644 --- a/Documentation/mm/process_addrs.rst +++ b/Documentation/mm/process_addrs.rst @@ -537,6 +537,17 @@ We establish basic locking rules when interacting with page tables: * When changing a page table entry the page table lock for that page table **must** be held, except if you can safely assume nobody can access the page tables concurrently (such as on invocation of :c:func:`!free_pgtables`). +* Page tables may be *walked* under RCU alone, as page tables are freed only + after an RCU grace period has elapsed. However, any entry found must be + revalidated after the page table lock is taken (such as the + :c:func:`!pmd_same` recheck performed by :c:func:`!pte_offset_map_lock`) + before it is acted upon. Changing an entry requires the page table lock + and one of the locks that excludes teardown (any one of the mmap, VMA or + rmap locks). +* When traversing page tables under RCU alone it is important to take care + when operating upon leaf entries - if the value is operated upon (for + instance getting the folio associated with a PTE) an appropriate lock must + be taken to prevent concurrent modification. * Reads from and writes to page table entries must be *appropriately* atomic. See the section on atomicity below for details. * Populating previously empty entries requires that the mmap or VMA locks are diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index b45e891d1193..26643d76bfb0 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -397,10 +397,17 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, * Note: "RO" / "RW" expresses the intended semantics, not that the *kmap* will * be read-only/read-write protected. * - * Note that free_pgtables(), used after unmapping detached vmas, or when - * exiting the whole mm, does not take page table lock before freeing a page - * table, and may not use RCU at all: "outsiders" like khugepaged should avoid - * pte_offset_map() and co once the vma is detached from mm or mm_users is zero. + * Note that free_pgtables(), used after unmapping detached vmas or when exiting + * the whole mm, does not take a page table lock before freeing a page table. + * + * As page table freeing itself is RCU-safe, page table readers can safely run + * concurrently with page table teardown. + * + * However, writers CANNOT as, without a lock being held, nothing prevents + * concurrent teardown. + * + * Also note that the PGD itself is freed at mmdrop() time, not under RCU - so + * the walker must keep the mm alive either by pinning the mm or the VMA. */ pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp) From 74d9aa0d8e98adde5a56e80ab68184244a97f66b Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Tue, 8 Sep 2026 14:26:49 +0800 Subject: [PATCH 301/539] mm: vmscan: avoid anon scanning for GFP_NOIO with low swapcache We have observed some cases where memory is allocated with GFP_NOIO, so we cannot reclaim any anon folios unless they are in swapcache. We can end up spending more than 150 ms looping in `shrink_folio_list()` scanning non-swapcache folios without reclaiming a single folio. This is pure overhead. This is particularly true on systems using zRAM, where swapcache is relatively rare. So let's check whether anon reclaim is allowed by GFP_IO and whether there is enough swapcache to make it worthwhile. If the swapcache is extremely low, we're essentially searching for a needle in a haystack, so let's avoid scanning anon in the first place. On Android this is triggered by dm-verity hash-block reads through dm-bufio, which legitimately use GFP_NOIO because they run underneath the IO path: verity_verify_io -> verity_hash_for_block -> verity_verify_level -> dm_bufio_read_with_ioprio -> new_read -> __bufio_new -> alloc_buffer gfp: GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN Such a reclaimer can land on a memcg with a large, unswapped anon LRU and a tiny file LRU (e.g. inactive_anon ~335 MB vs inactive_file ~4 MB, with negligible swapcache). shrink_lruvec() then keeps feeding that huge anon list into shrink_folio_list() - ~2400 shrink_folio_list() calls, ~93,000 anon folios scanned - where every folio is kept because it needs IO. The 150+ ms above is one such single shrink_lruvec() pass (not accumulated across a reclaim cycle), and it reclaims nothing; the actual progress comes entirely from the file side. Aging anon alongside file does have some value for a later __GFP_IO reclaimer, so it is not strictly pure overhead. But that aging is only deferred, not lost: kswapd and other __GFP_IO reclaimers still walk and age anon. Spending ~168 ms aging memory that this context cannot reclaim is not a worthwhile trade-off in a latency-sensitive path. To stay conservative, this only skips anon when the swapcache is really tiny - below 1/64 of the anon LRU - i.e. when essentially no anon on the list can be reclaimed without IO. Whenever there is a meaningful amount of swapcached anon, the normal path is used and anon is scanned and aged as before. Note this only addresses the traditional active/inactive LRU. MGLRU selects anon vs file scanning in its own path and is not covered here; fixing the MGLRU case is left as a TODO. Link: https://lore.kernel.org/20260908062649.1045883-1-zhangbo56@xiaomi.com Signed-off-by: Bo Zhang Signed-off-by: Andrew Morton Reviewed-by: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Shakeel Butt --- mm/vmscan.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 4059d1300789..8cbf562e6252 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -362,20 +362,72 @@ static bool can_demote(int nid, struct scan_control *sc, return !nodes_empty(allowed_mask); } +#ifdef CONFIG_SWAP +static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg, + int nid, struct scan_control *sc) +{ + pg_data_t *pgdat = NODE_DATA(nid); + unsigned long anon_pages, swapcache; + + /* + * A GFP_NOIO reclaimer can only reclaim anon that is already in the + * swapcache (adding anon to the swapcache needs IO). When swapcache is + * far below the anon LRU, scanning anon reclaims nothing and only burns + * CPU. The 1/64 threshold keeps this to the case where anon is + * effectively unreclaimable. + */ + if (!sc || (sc->gfp_mask & __GFP_IO)) + return false; + + /* + * FIXME: MGLRU doesn't fully respect can_reclaim_anon_pages() for the + * scanning type, so only apply this to the traditional LRU for now. + */ + if (lru_gen_enabled()) + return false; + + if (memcg) { + struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); + + anon_pages = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + + lruvec_page_state(lruvec, NR_ACTIVE_ANON); + swapcache = lruvec_page_state(lruvec, NR_SWAPCACHE); + } else { + anon_pages = node_page_state(pgdat, NR_INACTIVE_ANON) + + node_page_state(pgdat, NR_ACTIVE_ANON); + swapcache = node_page_state(pgdat, NR_SWAPCACHE); + } + + return swapcache < (anon_pages >> 6); +} +#else +static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg, + int nid, struct scan_control *sc) +{ + return true; +} +#endif /* CONFIG_SWAP */ + static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, int nid, struct scan_control *sc) { if (memcg == NULL) { /* - * For non-memcg reclaim, is there - * space in any swap device? + * For non-memcg reclaim, is there space in any swap device? + * And under GFP_NOIO, is there enough swapcached anon to make + * scanning anon worthwhile? */ - if (get_nr_swap_pages() > 0) + if (get_nr_swap_pages() > 0 && + !reclaimable_anon_is_low(memcg, nid, sc)) return true; } else { - /* Is the memcg below its swap limit? */ - if (mem_cgroup_get_nr_swap_pages(memcg) > 0) + /* + * Is the memcg below its swap limit, and under GFP_NOIO does + * it have enough swapcached anon to make scanning worthwhile? + */ + if (mem_cgroup_get_nr_swap_pages(memcg) > 0 && + !reclaimable_anon_is_low(memcg, nid, sc)) return true; } From 306170e7ea66d1f261b03ce5531f7b54f624e367 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Mon, 14 Sep 2026 19:29:22 +0800 Subject: [PATCH 302/539] mm: mglru: clear the reference counter for rejected folios As per the comment on LRU_REFS_FLAGS, when accessed folios are promoted to a new generation, LRU_REFS_FLAGS should be cleared so that the reference counter can start over. For folios rejected by shrink_folio_list(), we clear LRU_REFS_FLAGS and set the PG_active flag when lru_gen_folio_seq() would place them in the oldest generation. That's fine. But for rejected folios where lru_gen_folio_seq() returns a generation other than the oldest one (which can be treated as a promotion), we do not clear LRU_REFS_FLAGS. This can violate the promotion mechanism. And this means the rejected folio enters the new generation with stale, inflated tier bits, which can inflate reference counts and distort eviction statistics for these rejected folios. Fix this by clearing LRU_REFS_FLAGS for rejected folios. Of course, I need to evaluate the impact of the changes, which mainly falls into 3 cases: 1. When lru_gen_folio_seq() returns the oldest generation for rejected folios, there are no logic changes, and they will be put back into the 2nd youngest generation. 2. For rejected folios with PG_active set by shrink_folio_list(), we only clear the LRU_REFS_FLAGS and do not change the generation. 3. For rejected folios with PG_referenced set, the original code would put them back into the 2nd oldest generation. After this patch, we will put them back into the 2nd youngest generation. I think case 3 is also reasonable, before commmit 6cbdd9726fb5 ("mm/mglru: use folio_mark_accessed to replace folio_set_active"), a rejected referenced folio was also put back to the 2nd youngest gen. Meanwhile, I didn't see any noticeable performance impact on my 32-core Arm machine when running 'make -j32' to build the kernel inside a 3G-limited memcg with either zram or NVMe swap. Link: https://lore.kernel.org/7384df363c12e4acdaa2e0428420cd8eed320ee7.1789384831.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Reviewed-by: Kairui Song Reviewed-by: Barry Song Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie --- mm/vmscan.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 8cbf562e6252..a49da82ed479 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -5075,11 +5075,15 @@ retry: continue; } - /* don't add rejected folios to the oldest generation */ - if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) { - folio_set_lru_refs(folio, 0); + /* + * See the comments on LRU_REFS_FLAGS. + * + * The rejected folios are never added to the oldest generation, + * so this effectively promotes them by at least one generation. + */ + folio_set_lru_refs(folio, 0); + if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) folio_set_active(folio); - } } move_folios_to_lru(&list); From 51a8de81fd2edbf87d8046e352060329bb216fad Mon Sep 17 00:00:00 2001 From: Anastasios Papagiannis Date: Wed, 9 Sep 2026 09:42:31 +0300 Subject: [PATCH 303/539] mm/nommu: reject wrapping ranges in access_remote_vm() The NOMMU implementation of access_process_vm() rejects address ranges whose end wraps around, but access_remote_vm() bypasses this check even though both functions delegate to __access_remote_vm(). Move the wraparound check into __access_remote_vm() so it applies to both entry points. This is originally reported in [1]. Link: https://lore.kernel.org/20260909064231.18693-1-tasos.papagiannnis@gmail.com Fixes: f55f199b7d76 ("NOMMU: implement access_remote_vm") Signed-off-by: Anastasios Papagiannis Signed-off-by: Andrew Morton Closes: https://lore.kernel.org/bpf/4ef240a5bea36ff84df9589671367832860795159386a4c8fba546a0fa8b786f@mail.kernel.org/ [1] Reviewed-by: Lorenzo Stoakes (ARM) Cc: Liam R. Howlett Cc: Hajime Tazaki Cc: --- mm/nommu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/nommu.c b/mm/nommu.c index 498e01ee40b0..ed44510e3770 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1674,6 +1674,9 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, struct vm_area_struct *vma; int write = gup_flags & FOLL_WRITE; + if (addr + len < addr) + return 0; + if (mmap_read_lock_killable(mm)) return 0; @@ -1727,9 +1730,6 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in { struct mm_struct *mm; - if (addr + len < addr) - return 0; - mm = get_task_mm(tsk); if (!mm) return 0; From cb4936da6b37a6d374807f5ac9b5d96b7bba0e10 Mon Sep 17 00:00:00 2001 From: Youngjun Park Date: Thu, 10 Sep 2026 01:15:51 +0900 Subject: [PATCH 304/539] mm/swap: fix stale comment on swap_info_struct::cluster_info Patch series "mm/swap: skip empty clusters in the swapoff scan", v4. Speed up swapoff and reduce scanning stalls on large, mostly empty swap devices by skipping empty clusters. Reduce swapoff time on a 1 TiB device from 158 ms to 94 ms after filling 128 GiB, and from 392 ms to 73 ms after filling 512 GiB; expect little benefit when substantial swap data remains. find_next_to_unuse() walks a swap device one offset at a time. Slot state now lives in a per cluster swap table, so patch 2 dismisses an empty cluster with one counter read instead of SWAPFILE_CLUSTER table reads. Patch 1 is an unrelated one line comment fix noticed on the way. A debug test confirmed the skip path runs, and swapoff completed under load with no DEBUG_VM or lockdep splats. This patch (of 2): setup_swap_clusters_info() allocates cluster_info for every swap area, not only for SSDs. Link: https://lore.kernel.org/20260909161552.2335971-1-youngjun.park@lge.com Link: https://lore.kernel.org/20260909161552.2335971-2-youngjun.park@lge.com Signed-off-by: Youngjun Park Signed-off-by: Andrew Morton Acked-by: Kairui Song Reviewed-by: Barry Song Cc: Baoquan He Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham --- include/linux/swap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index a37ad8375e01..61005501888c 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -240,7 +240,7 @@ struct swap_info_struct { struct plist_node list; /* entry in swap_active_head */ signed char type; /* strange name for an index */ unsigned int max; /* size of this swap device */ - struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */ + struct swap_cluster_info *cluster_info; /* array, one entry per cluster */ struct list_head free_clusters; /* free clusters list */ struct list_head full_clusters; /* full clusters list */ struct list_head nonfull_clusters[SWAP_NR_ORDERS]; From 5b3d04dece09a0c79832770505c6a85439a7a1c2 Mon Sep 17 00:00:00 2001 From: Youngjun Park Date: Thu, 10 Sep 2026 01:15:52 +0900 Subject: [PATCH 305/539] mm/swap: scan by cluster in find_next_to_unuse() find_next_to_unuse() walks every offset from 0 to si->max, and swapoff restarts that walk on each retry, so the cost scales with the size of the device rather than with the few slots the shmem and mmlist passes could not free. It has caused stalls before. The flat walk predates the swap table. Slot state now lives in a per cluster table, and wait_for_allocation() stops all allocation before try_to_unuse() runs, so a cluster that holds no slot in use stays that way. Skip such a cluster instead of reading all of its entries. Fill a 1 TiB swap up to some amount, then swapoff. What is left sits at the top of what was filled, so every slot below it is free. Medians over 11 pairs at 32 and 128 GiB, 3 pairs at 256 and 512. filled swapoff old new 32 GiB 92.4ms 66.3ms 128 GiB 157.6ms 94.4ms 256 GiB 209.7ms 63.8ms 512 GiB 391.8ms 73.4ms old grows with how much was filled, new does not. In the ordinary case swap still holds real data and swapoff spends its time reading it back. it is tested 4 GiB on an 8 GiB device, where the scan is 1.4% of try_to_unuse(), and there is no difference either way. Commit dc644a073769 ("mm: add three more cond_resched() in swapoff") answered those stalls with a cond_resched() every 256 offsets. A walk bounded by one cluster no longer needs that counter. The loop now runs at most SWAPFILE_CLUSTER times before it returns or reschedules, the same bound swap_reclaim_full_clusters() already scans between cond_resched() calls. The scan end is clamped to si->max, so the walk stops there rather than running into the masked tail of the last cluster. ci->count is read without ci->lock, so READ_ONCE() marks the read for KCSAN. Allocation is already stopped, so the count can only drop, and a slot stops being counted only after its folio has left the swap cache. An empty cluster therefore holds nothing for try_to_unuse() to act on. Link: https://lore.kernel.org/20260909161552.2335971-3-youngjun.park@lge.com Signed-off-by: Youngjun Park Signed-off-by: Andrew Morton Reviewed-by: Barry Song Acked-by: Kairui Song Reviewed-by: Baoquan He Reviewed-by: Nhat Pham Cc: Chris Li Cc: Kemeng Shi --- mm/swapfile.c | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 48d3cd40defd..a8118f095f4d 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -370,8 +370,6 @@ static void discard_swap_cluster(struct swap_info_struct *si, } } -#define LATENCY_LIMIT 256 - static inline bool cluster_is_empty(struct swap_cluster_info *info) { return info->count == 0; @@ -2726,7 +2724,9 @@ unlock: static unsigned int find_next_to_unuse(struct swap_info_struct *si, unsigned int prev) { - unsigned int i; + struct swap_cluster_info *ci; + unsigned long i, end; + unsigned int ci_off; unsigned long swp_tb; /* @@ -2735,19 +2735,36 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si, * hits are okay, and sys_swapoff() has already prevented new * allocations from this area (while holding swap_lock). */ - for (i = prev + 1; i < si->max; i++) { - swp_tb = swap_table_get(__swap_offset_to_cluster(si, i), - i % SWAPFILE_CLUSTER); - if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb)) - break; - if ((i % LATENCY_LIMIT) == 0) + i = prev + 1; + while (i < si->max) { + ci = __swap_offset_to_cluster(si, i); + end = min_t(unsigned long, + ALIGN_DOWN(i, SWAPFILE_CLUSTER) + SWAPFILE_CLUSTER, + si->max); + + /* + * An empty cluster has no slot in use, so skip it whole. + * A slot is uncounted only after its folio left the swap + * cache, so there is nothing here for try_to_unuse() to act on. + * Count only drops here, so a READ_ONCE() without ci->lock is + * enough, unlike in every other cluster_is_empty() caller. + */ + if (!READ_ONCE(ci->count)) { + i = end; cond_resched(); + continue; + } + + ci_off = i % SWAPFILE_CLUSTER; + for (; i < end; ci_off++, i++) { + swp_tb = swap_table_get(ci, ci_off); + if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb)) + return i; + } + cond_resched(); } - if (i == si->max) - i = 0; - - return i; + return 0; } static int try_to_unuse(unsigned int type) From df8060e66580555be78fe50b286d06da5e7fad14 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 9 Sep 2026 07:04:03 -0700 Subject: [PATCH 306/539] mm/damon/vaddr: support prep_probes Patch series "mm/damon/vaddr: support {prep,apply}_probes". DAMON supports data attributes monitoring. However, only the physical address space operation set (paddr) is supporting it. Add the support to the virtual address space operation set (vaddr). Patch 1 adds prep_probes support to vaddr. Patch 2 moves probe filter handling code in paddr.c that can be reused by vaddr to ops-common.c. Patch 3 adds minimum apply_probes support to vaddr. Patch 4 extends the support for hugetlb. Patch 5 extends the support for pgidle_unset filter. Test ==== I confirmed it can capture ~48 mb working set of masim in vaddr mode, like below. First, start masim [1] to access ~48 mb memory at a time, in the background. $ ./masim/masim.py run \ --config_file ./masim/configs/stairs-50mb.cfg \ --repeat 10 --quiet & Note that the config says the working set is 50mb. It is 50 million bytes, so ~48 MiB. Start traditional access monitoring of masim's virtual address space using damo [2]. $ sudo ./damo/damo start $(pidof masim) Confirm it can capture the ~48 MiB working set as the 4-th region on the snapshot. $ sudo ./damo/damo report access heatmap: 11111111334[...]3000000000000000000000000000000000000000489999997777777743333333555556[...]8 # min/max temperatures: -1,150,000,000, 80,009,499, column size: 6.963 MiB intervals: sample 5 ms aggr 100 ms (max access hz 200) 0 addr 85.355 TiB size 55.703 MiB access 0 hz age 9.500 s 1 addr 85.355 TiB size 18.984 MiB access 0 hz age 6.700 s 2 addr 127.183 TiB size 278.516 MiB access 0 hz age 11.500 s 3 addr 127.183 TiB size 7.570 MiB access 0 hz age 900 ms 4 addr 127.183 TiB size 48.133 MiB access 190 hz age 800 ms 5 addr 127.183 TiB size 54.977 MiB access 0 hz age 1.700 s 6 addr 127.183 TiB size 55.113 MiB access 0 hz age 6.700 s 7 addr 127.183 TiB size 37.902 MiB access 0 hz age 3.900 s 8 addr 127.990 TiB size 120.000 KiB access 0 hz age 11.400 s 9 addr 127.990 TiB size 8.000 KiB access 70 hz age 0 ns 10 addr 127.990 TiB size 4.000 KiB access 0 hz age 11.200 s memory bw estimate: 8.931 GiB per second total size: 557.027 MiB record DAMON intervals: sample 5 ms, aggr 100 ms Stop access monitoring and start probe-only mode access monitoring. $ sudo ./damo/damo stop $ sudo ./damo/damo start $(pidof masim) --probe_prep set_pgidle \ --probe_filter allow pgidle_unset --probe_weight 1 Confirm it can also capture the ~48 MiB working set as the 11-th region on the snapshot. $ sudo ./damo/damo report attrs heatmap: 00000000113[...]40000000000000000000000000000000000000000000000001111114999999533333336[...]6 # min/max temperatures: -840,000,000, 330,002,000, column size: 6.961 MiB probe prep: set_pgidle, filter: allow pgidle_unset (weight: 1) intervals: sample 5 ms aggr 100 ms (max probe hits 20) # size address age probe_hits 0 8.000 KiB 127.990 TiB 8.700 s 0 1 120.000 KiB 127.990 TiB 8.600 s 0 2 55.543 MiB 127.183 TiB 8.400 s 0 3 110.008 MiB 127.183 TiB 8.300 s 0 4 55.352 MiB 127.183 TiB 8.100 s 0 5 55.605 MiB 127.183 TiB 8 s 0 6 55.691 MiB 85.355 TiB 7.900 s 0 7 54.430 MiB 127.183 TiB 7.900 s 0 8 50.391 MiB 127.183 TiB 6.900 s 0 9 18.879 MiB 85.355 TiB 6 s 0 10 52.844 MiB 127.183 TiB 3.500 s 0 11 48.039 MiB 127.183 TiB 3.300 s 20 12 4.000 KiB 127.990 TiB 8.500 s 20 memory bw estimate: 0 B per second total size: 556.910 MiB record DAMON intervals: sample 5 ms, aggr 100 ms This patch (of 5): DAMON virtual address space operation set (vaddr) is not supporting prep_probes. Add the support. Link: https://lore.kernel.org/20260909140408.104699-2-sj@kernel.org Link: https://github.com/sjp38/masim [1] Link: https://github.com/damonitor/damo [2] Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Gutierrez Asier --- mm/damon/vaddr.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index af9e1b82454c..20f4784f1781 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -483,6 +483,45 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) return max_nr_accesses; } +static void damon_va_prep_probe_region(struct damon_ctx *ctx, + struct mm_struct *mm, struct damon_region *r, + struct damon_probe *probe) +{ + struct damon_prep *p; + + damon_for_each_prep(p, probe) { + switch (p->action) { + case DAMON_PREP_SET_PGIDLE: + damon_va_mkold(mm, r->sampling_addr); + break; + default: + break; + } + } +} + +static void damon_va_prep_probes(struct damon_ctx *ctx, bool set_samples) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + struct damon_probe *p; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + if (!mm) + continue; + damon_for_each_region(r, t) { + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); + damon_for_each_probe(p, ctx) + damon_va_prep_probe_region(ctx, mm, r, p); + } + mmput(mm); + } +} + static bool damos_va_filter_young_match(struct damos_filter *filter, struct folio *folio, struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pmd_t *pmdp) @@ -911,6 +950,7 @@ static int __init damon_va_initcall(void) .update = damon_va_update, .prepare_access_checks = damon_va_prepare_access_checks, .check_accesses = damon_va_check_accesses, + .prep_probes = damon_va_prep_probes, .target_valid = damon_va_target_valid, .cleanup_target = damon_va_cleanup_target, .apply_scheme = damon_va_apply_scheme, From ed1ddfa3e809ff39304f42ccb3e66530e2d88705 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 9 Sep 2026 07:04:04 -0700 Subject: [PATCH 307/539] mm/damon/paddr: move probe filter handling to ops-common Probe filter matching logic for anon and memcg type filters in DAMON's physical address space operation set (paddr) can be reused by virtual address space operation set (vaddr) in future. Prepare the future by moving the code to ops-common.c Link: https://lore.kernel.org/20260909140408.104699-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Gutierrez Asier --- mm/damon/ops-common.c | 32 ++++++++++++++++++++++++++++++++ mm/damon/ops-common.h | 2 ++ mm/damon/paddr.c | 23 +---------------------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index acf8f216c51c..c36cc39cd2c7 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -531,3 +531,35 @@ bool damos_ops_has_filter(struct damos *s) return true; return false; } + +bool damon_ops_filter_match(struct damon_filter *filter, struct folio *folio) +{ + bool matched = false; + struct mem_cgroup *memcg; + + switch (filter->type) { + case DAMON_FILTER_TYPE_ANON: + if (!folio) { + matched = false; + break; + } + matched = folio_test_anon(folio); + break; + case DAMON_FILTER_TYPE_MEMCG: + if (!folio) { + matched = false; + break; + } + rcu_read_lock(); + memcg = folio_memcg_check(folio); + if (!memcg) + matched = false; + else + matched = filter->memcg_id == mem_cgroup_id(memcg); + rcu_read_unlock(); + break; + default: + break; + } + return matched == filter->matching; +} diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h index 172f0f17c4a8..ee2705825100 100644 --- a/mm/damon/ops-common.h +++ b/mm/damon/ops-common.h @@ -31,3 +31,5 @@ bool damos_folio_filter_match(struct damos_filter *filter, struct folio *folio); unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid); bool damos_ops_has_filter(struct damos *s); + +bool damon_ops_filter_match(struct damon_filter *filter, struct folio *folio); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index d2173a448d0b..d7c81829445b 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -143,29 +143,8 @@ static bool damon_pa_filter_match(struct damon_filter *filter, struct folio *folio) { bool matched = false; - struct mem_cgroup *memcg; switch (filter->type) { - case DAMON_FILTER_TYPE_ANON: - if (!folio) { - matched = false; - break; - } - matched = folio_test_anon(folio); - break; - case DAMON_FILTER_TYPE_MEMCG: - if (!folio) { - matched = false; - break; - } - rcu_read_lock(); - memcg = folio_memcg_check(folio); - if (!memcg) - matched = false; - else - matched = filter->memcg_id == mem_cgroup_id(memcg); - rcu_read_unlock(); - break; case DAMON_FILTER_TYPE_PGIDLE_UNSET: if (!folio) matched = false; @@ -173,7 +152,7 @@ static bool damon_pa_filter_match(struct damon_filter *filter, matched = damon_folio_young(folio); break; default: - break; + return damon_ops_filter_match(filter, folio); } return matched == filter->matching; } From ca74560ad16db56c84bccb6488d2d979fbcb0924 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 9 Sep 2026 07:04:05 -0700 Subject: [PATCH 308/539] mm/damon/vaddr: support apply_probe DAMON virtual address space operation set (vaddr) is not supporting apply_probe. Add a minimum support. Do not support hugetlb pages and PGIDLE_UNSET filter type for simplicity. Link: https://lore.kernel.org/20260909140408.104699-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Gutierrez Asier --- mm/damon/vaddr.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 20f4784f1781..7c548ec0cf6b 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -522,6 +522,128 @@ static void damon_va_prep_probes(struct damon_ctx *ctx, bool set_samples) } } +static bool damon_va_filter_pass(struct folio *folio, struct damon_probe *p) +{ + struct damon_filter *f; + bool pass = true; + + damon_for_each_filter(f, p) { + if (damon_ops_filter_match(f, folio)) { + pass = f->allow; + break; + } + pass = !f->allow; + } + return pass; +} + +struct damon_va_probe_walk_private { + struct damon_ctx *ctx; + struct damon_region *r; +}; + +static void damon_va_probe_folio(struct damon_ctx *ctx, + struct damon_region *r, struct folio *folio) +{ + struct damon_probe *probe; + int i = 0; + + damon_for_each_probe(probe, ctx) { + if (damon_va_filter_pass(folio, probe)) + r->probe_hits[i]++; + i++; + } +} + +static int damon_va_probe_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t *pte; + pte_t ptent; + spinlock_t *ptl; + struct folio *folio; + struct damon_va_probe_walk_private *priv = walk->private; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + ptl = pmd_trans_huge_lock(pmd, walk->vma); + if (ptl) { + pmd_t pmde = pmdp_get(pmd); + + if (!pmd_present(pmde)) + goto huge_out; + folio = vm_normal_folio_pmd(walk->vma, addr, pmde); + if (!folio) + goto huge_out; + damon_va_probe_folio(priv->ctx, priv->r, folio); + +huge_out: + spin_unlock(ptl); + return 0; + } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + + pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); + if (!pte) + return 0; + ptent = ptep_get(pte); + if (!pte_present(ptent)) + goto out; + folio = vm_normal_folio(walk->vma, addr, ptent); + if (!folio) + goto out; + damon_va_probe_folio(priv->ctx, priv->r, folio); + +out: + pte_unmap_unlock(pte, ptl); + return 0; +} + +static void __damon_va_apply_probes(struct damon_ctx *ctx, + struct mm_struct *mm, struct damon_region *r) +{ + struct damon_va_probe_walk_private arg = { + .ctx = ctx, + .r = r, + }; + struct mm_walk_ops damon_probe_walk_ops = { + .pmd_entry = damon_va_probe_pmd_entry, + .hugetlb_entry = NULL, + }; + unsigned long addr = r->sampling_addr; + + if (!mm) + return; + + damon_va_walk_page_range(mm, addr, addr + 1, &damon_probe_walk_ops, + &arg); +} + +static unsigned int damon_va_apply_probes(struct damon_ctx *ctx, + bool set_samples, bool return_max_wsum) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + unsigned int max_wsum = 0; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + damon_for_each_region(r, t) { + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); + __damon_va_apply_probes(ctx, mm, r); + if (return_max_wsum) + max_wsum = max(damon_probe_hits_wsum(r, false, + ctx), max_wsum); + } + if (mm) + mmput(mm); + } + + return max_wsum; +} + static bool damos_va_filter_young_match(struct damos_filter *filter, struct folio *folio, struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pmd_t *pmdp) @@ -951,6 +1073,7 @@ static int __init damon_va_initcall(void) .prepare_access_checks = damon_va_prepare_access_checks, .check_accesses = damon_va_check_accesses, .prep_probes = damon_va_prep_probes, + .apply_probes = damon_va_apply_probes, .target_valid = damon_va_target_valid, .cleanup_target = damon_va_cleanup_target, .apply_scheme = damon_va_apply_scheme, From f72c1f220d3d1c108422b4a9a979f5c052c4cf8c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 9 Sep 2026 07:04:06 -0700 Subject: [PATCH 309/539] mm/damon/vaddr: extend apply_probes() for hugetlb DAMON virtual address space operation set(vaddr) does not support hugetlb pages in apply_probes. Extend it for hugetlb pages. Link: https://lore.kernel.org/20260909140408.104699-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Gutierrez Asier --- mm/damon/vaddr.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 7c548ec0cf6b..45239f05e113 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -598,6 +598,34 @@ out: return 0; } +#ifdef CONFIG_HUGETLB_PAGE +static int damon_va_probe_hugetlb_entry(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, struct mm_walk *walk) +{ + struct damon_va_probe_walk_private *priv = walk->private; + struct hstate *h = hstate_vma(walk->vma); + struct folio *folio; + spinlock_t *ptl; + pte_t entry; + + ptl = huge_pte_lock(h, walk->mm, pte); + entry = huge_ptep_get(walk->mm, addr, pte); + if (!pte_present(entry)) + goto out; + + folio = pfn_folio(pte_pfn(entry)); + folio_get(folio); + damon_va_probe_folio(priv->ctx, priv->r, folio); + folio_put(folio); + +out: + spin_unlock(ptl); + return 0; +} +#else +#define damon_va_probe_hugetlb_entry NULL +#endif /* CONFIG_HUGETLB_PAGE */ + static void __damon_va_apply_probes(struct damon_ctx *ctx, struct mm_struct *mm, struct damon_region *r) { @@ -607,7 +635,7 @@ static void __damon_va_apply_probes(struct damon_ctx *ctx, }; struct mm_walk_ops damon_probe_walk_ops = { .pmd_entry = damon_va_probe_pmd_entry, - .hugetlb_entry = NULL, + .hugetlb_entry = damon_va_probe_hugetlb_entry, }; unsigned long addr = r->sampling_addr; From ca4082423a0cb4d9c69970aab08520671292fd4e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 9 Sep 2026 07:04:07 -0700 Subject: [PATCH 310/539] mm/damon/vaddr: support pgidle_unset probe filter type DAMON virtual address space operation set (vaddr) does not support pgidle_unset probe filter type. Add the support. Link: https://lore.kernel.org/20260909140408.104699-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Gutierrez Asier --- mm/damon/vaddr.c | 55 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 45239f05e113..9a38dc89a156 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -522,13 +522,49 @@ static void damon_va_prep_probes(struct damon_ctx *ctx, bool set_samples) } } -static bool damon_va_filter_pass(struct folio *folio, struct damon_probe *p) +static bool damon_va_young_addr(struct folio *folio, pte_t *pte, pmd_t *pmd, + struct mm_struct *mm, unsigned long addr) +{ + bool young = false; + + if (pte) + young = pte_young(*pte); + else if (pmd) + young = pmd_young(*pmd); + young = young || !folio_test_idle(folio) || + mmu_notifier_test_young(mm, addr); + return young; +} + +static bool damon_va_filter_match(struct damon_filter *filter, + struct folio *folio, pte_t *pte, pmd_t *pmd, + struct mm_struct *mm, unsigned long addr) +{ + bool matched = false; + + switch (filter->type) { + case DAMON_FILTER_TYPE_PGIDLE_UNSET: + if (!folio) + matched = false; + else + matched = damon_va_young_addr(folio, pte, pmd, mm, + addr); + break; + default: + return damon_ops_filter_match(filter, folio); + } + return matched == filter->matching; +} + +static bool damon_va_filter_pass(struct folio *folio, struct damon_probe *p, + pte_t *pte, pmd_t *pmd, struct mm_struct *mm, + unsigned long addr) { struct damon_filter *f; bool pass = true; damon_for_each_filter(f, p) { - if (damon_ops_filter_match(f, folio)) { + if (damon_va_filter_match(f, folio, pte, pmd, mm, addr)) { pass = f->allow; break; } @@ -543,13 +579,15 @@ struct damon_va_probe_walk_private { }; static void damon_va_probe_folio(struct damon_ctx *ctx, - struct damon_region *r, struct folio *folio) + struct damon_region *r, struct folio *folio, + pte_t *pte, pmd_t *pmd, struct mm_struct *mm) { struct damon_probe *probe; int i = 0; damon_for_each_probe(probe, ctx) { - if (damon_va_filter_pass(folio, probe)) + if (damon_va_filter_pass(folio, probe, pte, pmd, mm, + r->sampling_addr)) r->probe_hits[i]++; i++; } @@ -574,7 +612,8 @@ static int damon_va_probe_pmd_entry(pmd_t *pmd, unsigned long addr, folio = vm_normal_folio_pmd(walk->vma, addr, pmde); if (!folio) goto huge_out; - damon_va_probe_folio(priv->ctx, priv->r, folio); + damon_va_probe_folio(priv->ctx, priv->r, folio, NULL, &pmde, + walk->vma->vm_mm); huge_out: spin_unlock(ptl); @@ -591,7 +630,8 @@ huge_out: folio = vm_normal_folio(walk->vma, addr, ptent); if (!folio) goto out; - damon_va_probe_folio(priv->ctx, priv->r, folio); + damon_va_probe_folio(priv->ctx, priv->r, folio, &ptent, NULL, + walk->vma->vm_mm); out: pte_unmap_unlock(pte, ptl); @@ -615,7 +655,8 @@ static int damon_va_probe_hugetlb_entry(pte_t *pte, unsigned long hmask, folio = pfn_folio(pte_pfn(entry)); folio_get(folio); - damon_va_probe_folio(priv->ctx, priv->r, folio); + damon_va_probe_folio(priv->ctx, priv->r, folio, &entry, NULL, + walk->vma->vm_mm); folio_put(folio); out: From 97d44457f3138aaae583354d1fa6b7b6c31b1a0d Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 7 Sep 2026 09:19:38 -0700 Subject: [PATCH 311/539] mm: zswap: don't fail a large-folio swapin whose range is not in zswap thp_swapin_suitable_orders() and shmem_swap_alloc_folio() sample zswap_never_enabled() to decide whether a swapin may use a large folio. zswap_load() samples the same one-way static key again once the read reaches it. Nothing serialises the two reads, and in between the task allocates and pins a high-order folio, which can sleep. If zswap is enabled for the first time in that window, a large folio that was correctly permitted reaches zswap_load(), which rejects every large folio with -EINVAL. swap_read_folio() treats anything other than -ENOENT as "zswap handled it" and skips the backing-device read, so the folio comes back unlocked and not uptodate: SIGBUS for an anonymous fault, -EIO for shmem. The data is intact on the swap device - it was written there before zswap was ever enabled - and the not-uptodate folio stays in the swap cache, so every retry of the fault fails the same way. With panic_on_warn the WARN takes the machine down rather than the task. Scan the range instead of rejecting the folio. The caller has pinned every slot before issuing the read, so zswap cannot start a store or a writeback into the range and the scan is stable. If nothing in the range is in zswap it is all on the backing device: return -ENOENT and let swap_read_folio() read it. A range that does have a slot in zswap is still refused, because zswap stores large folios as order-0 entries and cannot reconstruct one. That stays reachable - a slot shared with another task can be stored inside the same window - and refusing is correct, since the alternative is returning the stale device copy. Report it as -EIO rather than -EINVAL: the request is valid, zswap just cannot serve it. The only caller distinguishes -ENOENT from everything else, so that part is a documentation fix. Link: https://lore.kernel.org/20260907161938.1932355-1-usama.arif@linux.dev Fixes: 242d12c98174 ("mm: support large folios swap-in for sync io devices") Co-developed-by: Alexandre Ghiti Signed-off-by: Alexandre Ghiti Signed-off-by: Usama Arif Signed-off-by: Andrew Morton Acked-by: Yosry Ahmed Acked-by: Nhat Pham Cc: Chengming Zhou Cc: Johannes Weiner --- mm/zswap.c | 58 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index 3a6f89017646..b5411ab16480 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1555,6 +1555,32 @@ check_old: return ret; } +/** + * zswap_is_present() - is any slot in [entry, entry + nr) in zswap? + * @entry: base swap entry of the range + * @nr: number of contiguous slots to check + * + * Context: The caller must keep the range pinned, otherwise the answer can + * change under it. + * Return: true if at least one slot in the range is in zswap. + */ +static bool zswap_is_present(swp_entry_t entry, unsigned int nr) +{ + pgoff_t offset = swp_offset(entry); + struct xarray *tree = swap_zswap_tree(entry); + unsigned long index = offset; + + /* + * A pinned range is at most SWAPFILE_CLUSTER slots and is aligned to + * its own size, so one tree covers all of it and a single lookup is + * enough. Scanning only part of the range would report a false + * "absent" and let the caller read a stale copy from the device. + */ + BUILD_BUG_ON(SWAPFILE_CLUSTER > ZSWAP_ADDRESS_SPACE_PAGES); + + return xa_find(tree, &index, offset + nr - 1, XA_PRESENT); +} + /** * zswap_load() - load a folio from zswap * @folio: folio to load @@ -1562,15 +1588,12 @@ check_old: * Return: 0 on success, with the folio unlocked and marked up-to-date, or one * of the following error codes: * - * -EIO: if the swapped out content was in zswap, but could not be loaded - * into the page due to a decompression failure. The folio is unlocked, but - * NOT marked up-to-date, so that an IO error is emitted (e.g. do_swap_page() - * will SIGBUS). - * - * -EINVAL: if the swapped out content was in zswap, but the page belongs - * to a large folio, which is not supported by zswap. The folio is unlocked, - * but NOT marked up-to-date, so that an IO error is emitted (e.g. - * do_swap_page() will SIGBUS). + * -EIO: if the swapped out content was in zswap but could not be handed + * back, either because decompression failed or because a slot in a + * large-folio range is still in zswap and zswap cannot reconstruct a large + * folio from per-page entries. The folio is unlocked, but NOT marked + * up-to-date, so that an IO error is emitted (e.g. do_swap_page() will + * SIGBUS). * * -ENOENT: if the swapped out content was not in zswap. The folio remains * locked on return. @@ -1589,13 +1612,18 @@ int zswap_load(struct folio *folio) return -ENOENT; /* - * Large folios should not be swapped in while zswap is being used, as - * they are not properly handled. Zswap does not properly load large - * folios, and a large folio may only be partially in zswap. + * A large folio can legitimately reach zswap_load() with its whole + * range on the backing device, so scan the range rather than rejecting + * it outright. The caller has pinned every slot, so zswap cannot start + * a store or a writeback into the range while we look. */ - if (WARN_ON_ONCE(folio_test_large(folio))) { - folio_unlock(folio); - return -EINVAL; + if (folio_test_large(folio)) { + if (WARN_ON_ONCE(zswap_is_present(swp, + folio_nr_pages(folio)))) { + folio_unlock(folio); + return -EIO; + } + return -ENOENT; } entry = xa_load(tree, offset); From e4949a80f3b07e52dca76c34ce353633927a28a5 Mon Sep 17 00:00:00 2001 From: Jinmeng Zhou Date: Mon, 7 Sep 2026 21:20:55 +0800 Subject: [PATCH 312/539] mm/hugetlb: fix subpool minimum reservation rollback When a reservation request is partially covered by a subpool minimum and the remaining global reservation fails, the error path first calls hugepage_subpool_put_pages() for the subpool-backed portion. It removes the failed global portion from used_hpages only afterwards. hugepage_subpool_put_pages() uses used_hpages to decide whether rsv_hpages should be restored. Since used_hpages still includes the global portion, it can remain at or above min_hpages and prevent that restoration. It then reports the subpool reservation as releasable, causing hugetlb_acct_memory() to incorrectly decrement h->resv_huge_pages. This was reproduced with four 2 MB huge pages and a hugetlbfs mount with size=10M,min_size=8M. After a successful three-page reservation, a two-page reservation which needed one subpool page and one global page failed with -ENOMEM. HugePages_Rsvd incorrectly dropped from four to three even though the subpool minimum was still four pages. Roll back the failed global portion from used_hpages first, so that hugepage_subpool_put_pages() evaluates the minimum reservation against the current usage and returns the correct global adjustment. Link: https://lore.kernel.org/20260907132055.26696-1-zhoujinmeng@bytedance.com Fixes: a833a693a490 ("mm: hugetlb: fix incorrect fallback for subpool") Signed-off-by: Jinmeng Zhou Signed-off-by: Andrew Morton Acked-by: Muchun Song Tested-by: Ackerley Tng Cc: David Hildenbrand Cc: Ma Wupeng Cc: Oscar Salvador Cc: --- mm/hugetlb.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index c59ebbcf6684..c1f76bde9ac0 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6854,15 +6854,6 @@ long hugetlb_reserve_pages(struct inode *inode, out_put_pages: spool_resv = chg - gbl_reserve; - if (spool_resv) { - /* put sub pool's reservation back, chg - gbl_reserve */ - gbl_resv = hugepage_subpool_put_pages(spool, spool_resv); - /* - * subpool's reserved pages can not be put back due to race, - * return to hstate. - */ - hugetlb_acct_memory(h, -gbl_resv); - } /* Restore used_hpages for pages that failed global reservation */ if (gbl_reserve && spool) { unsigned long flags; @@ -6872,6 +6863,15 @@ out_put_pages: spool->used_hpages -= gbl_reserve; unlock_or_release_subpool(spool, flags); } + if (spool_resv) { + /* put sub pool's reservation back, chg - gbl_reserve */ + gbl_resv = hugepage_subpool_put_pages(spool, spool_resv); + /* + * subpool's reserved pages can not be put back due to race, + * return to hstate. + */ + hugetlb_acct_memory(h, -gbl_resv); + } out_uncharge_cgroup: hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), chg * pages_per_huge_page(h), h_cg); From c1dd5e24a82d19bff8eef50ba8205d4776584aff Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Tue, 8 Sep 2026 09:28:01 +0800 Subject: [PATCH 313/539] mm/zswap: publish the initial pool with list_add_rcu() zswap_setup() publishes the pool on the zswap_pools list with a plain list_add(), but the list is walked by concurrent RCU readers holding nothing but rcu_read_lock() through zswap_total_pages(), e.g. /proc/meminfo and the shrinker count path. CPU 0 (writer) CPU 1 (reader) -------------- -------------- zswap_pool_create(): pool->zs_pool = zs_create_pool(); (1) list_add() -> __list_add(): WRITE_ONCE(zswap_pools.next, &pool->list); (2) zswap_total_pages(): pool = READ_ONCE( (a) zswap_pools.next); zs_get_total_pages( (b) pool->zs_pool); If (2) becomes visible to CPU 1 before (1), CPU 1 finds the pool at (a) but dereferences a wild pointer at (b). Publish the node with list_add_rcu(). Link: https://lore.kernel.org/20260908012801.1864430-1-xialonglong2025@163.com Fixes: 91cdcd8d624b ("mm: zswap: optimize zswap pool size tracking") Signed-off-by: Longlong Xia Signed-off-by: Andrew Morton Reported-by: Sashiko Closes: https://sashiko.dev/#/patchset/20260906133601.3563324-1-xialonglong2025%40163.com Acked-by: Yosry Ahmed Acked-by: Nhat Pham Assisted-by: Zcode:GLM-5.3 Cc: Chengming Zhou Cc: Johannes Weiner Cc: --- mm/zswap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/zswap.c b/mm/zswap.c index b5411ab16480..2a95aedc08fc 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1815,7 +1815,7 @@ static int zswap_setup(void) pool = __zswap_pool_create_fallback(); if (pool) { pr_info("loaded using pool %s\n", pool->tfm_name); - list_add(&pool->list, &zswap_pools); + list_add_rcu(&pool->list, &zswap_pools); zswap_has_pool = true; static_branch_enable(&zswap_ever_enabled); } else { From 0b1b457ecc4f636d85e1a0fade05f3dee264f59a Mon Sep 17 00:00:00 2001 From: Bingfang Guo Date: Thu, 10 Sep 2026 11:46:58 +0800 Subject: [PATCH 314/539] mm/memcg: clear folio memcg after changing per memcg stats I notice extremely high swapcached count in the per memcg level memory.stat when running tests with cgroupv1 setup by swapping pages in and out. It seems that the counter never gets decreased so the value is rather useless and confusing to users reading it. So I think fixing it so that the value can reflect the actual swapcache usage correctly could be helpful. __memcg1_swapout() transfers the memsw charge of a folio to its swap entry and clears folio->memcg_data as part of that. In the vmscan swapout path it runs before __swap_cache_del_folio(), which then decrements the swapcache stats through lruvec_stat_mod_folio(). Since folio->memcg_data has already been cleared, folio_memcg() returns NULL and the NR_SWAPCACHE decrement only updates the node-level counter instead of the memcg's lruvec, leaking the per-memcg swapcache count. Users using swaps will read totally meaningless swapcache value from per memcg memory.stat, like 200GB of swapcache on a 64GB setup, which is quite confusing and may trigger monitoring alerts, if any. Move the __memcg1_swapout() call into __swap_cache_del_folio(), after the NR_FILE_PAGES and NR_SWAPCACHE updates but before __swap_cache_do_del_folio() removes the folio from the swap cache. This keeps the stats attributed to the folio's memcg while still recording the swap cgroup with a valid folio->swap. Add a swapout parameter so the plain swap_cache_del_folio() path is left unchanged. Link: https://lore.kernel.org/20260910-memcg-swapcache-stats-fix-v5-1-033f510ba748@tencent.com Fixes: 2732acda82c9 ("mm, swap: use swap cache as the swap in synchronize layer") Signed-off-by: Bingfang Guo Signed-off-by: Andrew Morton Acked-by: Kairui Song Cc: Axel Rasmussen Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kemeng Shi Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Nhat Pham Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Cc: --- mm/swap.h | 6 ++++-- mm/swap_state.c | 13 ++++++++++--- mm/vmscan.c | 3 +-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/mm/swap.h b/mm/swap.h index 0b5d507739bc..b3b54c28929a 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -319,7 +319,8 @@ struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, void __swap_cache_add_folio(struct swap_cluster_info *ci, struct folio *folio, swp_entry_t entry); void __swap_cache_del_folio(struct swap_cluster_info *ci, - struct folio *folio, swp_entry_t entry, void *shadow); + struct folio *folio, swp_entry_t entry, void *shadow, + bool swapout); void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new); @@ -452,7 +453,8 @@ static inline void swap_cache_del_folio(struct folio *folio) } static inline void __swap_cache_del_folio(struct swap_cluster_info *ci, - struct folio *folio, swp_entry_t entry, void *shadow) + struct folio *folio, swp_entry_t entry, void *shadow, + bool swapout) { } diff --git a/mm/swap_state.c b/mm/swap_state.c index 305877e1f4d7..625c185a1ca4 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -306,21 +306,28 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci, * @folio: The folio. * @entry: The first swap entry that the folio corresponds to. * @shadow: shadow value to be filled in the swap cache. + * @swapout: whether this folio is being reclaimed after swapout. * * Removes a folio from the swap cache and fills a shadow in place. * This won't put the folio's refcount. The caller has to do that. * * Context: Caller must ensure the folio is locked and in the swap cache * using the index of @entry, and lock the cluster that holds the entries. + * If @swapout is set, the folio should be in reclaim path and IRQs + * should be disabled. */ void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio, - swp_entry_t entry, void *shadow) + swp_entry_t entry, void *shadow, bool swapout) { unsigned long nr_pages = folio_nr_pages(folio); - __swap_cache_do_del_folio(ci, folio, entry, shadow); node_stat_mod_folio(folio, NR_FILE_PAGES, -nr_pages); lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr_pages); + + if (swapout) + __memcg1_swapout(folio, ci); + + __swap_cache_do_del_folio(ci, folio, entry, shadow); } /** @@ -339,7 +346,7 @@ void swap_cache_del_folio(struct folio *folio) swp_entry_t entry = folio->swap; ci = swap_cluster_lock(__swap_entry_to_info(entry), swp_offset(entry)); - __swap_cache_del_folio(ci, folio, entry, NULL); + __swap_cache_del_folio(ci, folio, entry, NULL, false); swap_cluster_unlock(ci); folio_ref_sub(folio, folio_nr_pages(folio)); diff --git a/mm/vmscan.c b/mm/vmscan.c index a49da82ed479..8e9c73dcd19b 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -807,8 +807,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, if (reclaimed && !mapping_exiting(mapping)) shadow = workingset_eviction(folio, target_memcg); - __memcg1_swapout(folio, ci); - __swap_cache_del_folio(ci, folio, swap, shadow); + __swap_cache_del_folio(ci, folio, swap, shadow, true); swap_cluster_unlock_irq(ci); } else { void (*free_folio)(struct folio *); From 4844f5b84a8efdb030e5810ab7c0c838a4907c16 Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Thu, 10 Sep 2026 20:56:44 +0800 Subject: [PATCH 315/539] selftests/mm: reject invalid test selections before running tests Patch series "selftests/mm: Validate selections and scope memfd_secret setup", v3. run_vmtests.sh can reach test setup after invalid options or category selections. Its memfd_secret preparation can also change ptrace_scope when that category was not selected. Patch 1 rejects invalid selections before setup. Patch 2 gates memfd_secret preparation on category selection and executable presence. This patch (of 2): getopts reports unknown options and missing arguments, but run_vmtests.sh ignores its error result and continues with test setup. An empty -t argument also falls back to the default selection, while unknown category names can silently select no tests and still reach setup code. Exit on getopts errors and validate category names against the existing list in usage() before any test setup. Reject empty and whitespace-only selections, and normalize category separators so validation and execution agree. Initialize the default selection before parsing options so only -t changes the selection. Link: https://lore.kernel.org/20260910125645.285866-1-diannaaav@gmail.com Link: https://lore.kernel.org/20260910125645.285866-2-diannaaav@gmail.com Fixes: 85463321e726 ("selftests/vm: enable running select groups of tests") Signed-off-by: Tianyi Chen Signed-off-by: Andrew Morton Assisted-by: LLM Cc: David Hildenbrand (Arm) Cc: Joel Savitz Cc: Shuah Khan --- tools/testing/selftests/mm/run_vmtests.sh | 26 +++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index d09f9f6a384e..9e62ab4c6775 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -96,26 +96,44 @@ separated by spaces: example: ./run_vmtests.sh -t "hmm mmap ksm" EOF - exit 0 } RUN_ALL=false RUN_DESTRUCTIVE=false TAP_PREFIX="# " +VM_SELFTEST_ITEMS="default" + while getopts "aht:nd" OPT; do case ${OPT} in "a") RUN_ALL=true ;; - "h") usage ;; + "h") usage; exit 0 ;; "t") VM_SELFTEST_ITEMS=${OPTARG} ;; "n") TAP_PREFIX= ;; "d") RUN_DESTRUCTIVE=true ;; + "?") exit 1 ;; esac done shift $((OPTIND -1)) -# default behavior: run all tests -VM_SELFTEST_ITEMS=${VM_SELFTEST_ITEMS:-default} +# Normalize whitespace so validation and test_selected() use the same names. +read -r -a selected_categories <<< "${VM_SELFTEST_ITEMS//$'\n'/ }" +VM_SELFTEST_ITEMS="${selected_categories[*]}" +if [ -z "$VM_SELFTEST_ITEMS" ]; then + echo "No test categories specified" >&2 + exit 1 +fi + +if [ "$VM_SELFTEST_ITEMS" != "default" ]; then + # Keep the documented category list as the source of valid names. + valid_categories=$(usage | sed -n 's/^- //p') + for category in "${selected_categories[@]}"; do + if ! grep -Fxq -- "$category" <<< "$valid_categories"; then + echo "Unknown test category: $category" >&2 + exit 1 + fi + done +fi test_selected() { if [ "$VM_SELFTEST_ITEMS" == "default" ]; then From 0bbe9b917d965d2deafdcda8569268fb5beb0d28 Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Thu, 10 Sep 2026 20:56:45 +0800 Subject: [PATCH 316/539] selftests/mm: only prepare ptrace_scope when memfd_secret is selected The memfd_secret setup clears ptrace_scope whenever its test binary is executable, even when a different category was selected. run_test() filters the test invocation, but it does not protect the preceding setup. Check the category selection before entering the memfd_secret block so running unrelated categories does not change ptrace_scope. Keep the existing executable check and the behavior when memfd_secret is selected. Link: https://lore.kernel.org/20260910125645.285866-3-diannaaav@gmail.com Signed-off-by: Tianyi Chen Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Joel Savitz Cc: Shuah Khan --- tools/testing/selftests/mm/run_vmtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 9e62ab4c6775..9bbef9410ccc 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -370,7 +370,7 @@ CATEGORY="process_madv" run_test ./process_madv CATEGORY="vma_merge" run_test ./merge -if [ -x ./memfd_secret ] +if test_selected "memfd_secret" && [ -x ./memfd_secret ] then if [ -f /proc/sys/kernel/yama/ptrace_scope ]; then (echo 0 > /proc/sys/kernel/yama/ptrace_scope 2>&1) | tap_prefix From 329808c9cf418a7d174016e88e51fa7a668b7cb1 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 10 Sep 2026 20:35:42 +0800 Subject: [PATCH 317/539] mm: zswap: convert zswap_invalidate() to take a range Patch series "mm: zswap: optimize zswap invalidate and store", v3. This series range-ifies zswap_invalidate() to skip xarray lookups when zswap is unused, and reuses it in zswap_store() to eliminate redundant per-slot lookups and open-coded logic. This patch (of 3): zswap_invalidate() takes a swp_entry_t only to unpack it right back into type and offset, and both callers already have those values in hand. Pass type, offset, and nr_entries directly so swap_range_free() can invalidate an entire range in one call instead of looping in the caller. Link: https://lore.kernel.org/20260910123544.818146-1-wangkefeng.wang@huawei.com Link: https://lore.kernel.org/20260910123544.818146-2-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Signed-off-by: Andrew Morton Suggested-by: Johannes Weiner Acked-by: Yosry Ahmed Reviewed-by: Johannes Weiner Cc: Chengming Zhou Cc: Kairui Song Cc: Nhat Pham Cc: Yosry Ahmed --- include/linux/zswap.h | 8 ++++++-- mm/swapfile.c | 4 +--- mm/zswap.c | 29 +++++++++++++++++++---------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/include/linux/zswap.h b/include/linux/zswap.h index 30c193a1207e..df6cafbe95dc 100644 --- a/include/linux/zswap.h +++ b/include/linux/zswap.h @@ -27,7 +27,7 @@ struct zswap_lruvec_state { unsigned long zswap_total_pages(void); bool zswap_store(struct folio *folio); int zswap_load(struct folio *folio); -void zswap_invalidate(swp_entry_t swp); +void zswap_invalidate(int type, pgoff_t offset, unsigned long nr_entries); int zswap_swapon(int type, unsigned long nr_pages); void zswap_swapoff(int type); void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg); @@ -49,7 +49,11 @@ static inline int zswap_load(struct folio *folio) return -ENOENT; } -static inline void zswap_invalidate(swp_entry_t swp) {} +static inline void zswap_invalidate(int type, pgoff_t offset, + unsigned long nr_entries) +{ +} + static inline int zswap_swapon(int type, unsigned long nr_pages) { return 0; diff --git a/mm/swapfile.c b/mm/swapfile.c index a8118f095f4d..2c263563b70e 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1316,10 +1316,8 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset, { unsigned long end = offset + nr_entries - 1; void (*swap_slot_free_notify)(struct block_device *, unsigned long); - unsigned int i; - for (i = 0; i < nr_entries; i++) - zswap_invalidate(swp_entry(si->type, offset + i)); + zswap_invalidate(si->type, offset, nr_entries); if (si->flags & SWP_BLKDEV) swap_slot_free_notify = diff --git a/mm/zswap.c b/mm/zswap.c index 2a95aedc08fc..9153bdfdd5df 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -228,10 +228,15 @@ static bool zswap_has_pool; /* One swap address space for each 64M swap space */ #define ZSWAP_ADDRESS_SPACE_SHIFT 14 #define ZSWAP_ADDRESS_SPACE_PAGES (1 << ZSWAP_ADDRESS_SPACE_SHIFT) + +static inline struct xarray *zswap_tree(int type, pgoff_t offset) +{ + return &zswap_trees[type][offset >> ZSWAP_ADDRESS_SPACE_SHIFT]; +} + static inline struct xarray *swap_zswap_tree(swp_entry_t swp) { - return &zswap_trees[swp_type(swp)][swp_offset(swp) - >> ZSWAP_ADDRESS_SPACE_SHIFT]; + return zswap_tree(swp_type(swp), swp_offset(swp)); } #define zswap_pool_debug(msg, p) \ @@ -1656,18 +1661,22 @@ int zswap_load(struct folio *folio) return 0; } -void zswap_invalidate(swp_entry_t swp) +void zswap_invalidate(int type, pgoff_t offset, unsigned long nr_entries) { - pgoff_t offset = swp_offset(swp); - struct xarray *tree = swap_zswap_tree(swp); struct zswap_entry *entry; + struct xarray *tree; + unsigned long i; - if (xa_empty(tree)) - return; + for (i = 0; i < nr_entries; i++) { + tree = zswap_tree(type, offset + i); - entry = xa_erase(tree, offset); - if (entry) - zswap_entry_free(entry); + if (xa_empty(tree)) + continue; + + entry = xa_erase(tree, offset + i); + if (entry) + zswap_entry_free(entry); + } } int zswap_swapon(int type, unsigned long nr_pages) From 7dd5f47046dd9547aabbd53158da6b95b7011023 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 10 Sep 2026 20:35:43 +0800 Subject: [PATCH 318/539] mm: zswap: skip xarray walk in zswap_invalidate() when zswap is unused zswap_invalidate() still walks the per-area xarray even when zswap has never been enabled. Add a zswap_never_enabled() to skip it. Link: https://lore.kernel.org/20260910123544.818146-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Signed-off-by: Andrew Morton Acked-by: Yosry Ahmed Reviewed-by: Johannes Weiner Cc: Chengming Zhou Cc: Kairui Song Cc: Nhat Pham Cc: Yosry Ahmed --- mm/zswap.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/zswap.c b/mm/zswap.c index 9153bdfdd5df..bf86651d7464 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1667,6 +1667,9 @@ void zswap_invalidate(int type, pgoff_t offset, unsigned long nr_entries) struct xarray *tree; unsigned long i; + if (zswap_never_enabled()) + return; + for (i = 0; i < nr_entries; i++) { tree = zswap_tree(type, offset + i); From c7248110e5641fc47e42c15c3c64e801d588a02e Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 10 Sep 2026 20:35:44 +0800 Subject: [PATCH 319/539] mm: zswap: reuse zswap_invalidate() in zswap_store() Reuse zswap_invalidate() in zswap_store() check_old path. Its zswap_never_enabled() and xa_empty() guards avoid redundant xarray lookups and deduplicate the per-slot free logic. Link: https://lore.kernel.org/20260910123544.818146-4-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Signed-off-by: Andrew Morton Acked-by: Yosry Ahmed Reviewed-by: Johannes Weiner Cc: Chengming Zhou Cc: Kairui Song Cc: Nhat Pham Cc: Yosry Ahmed --- mm/zswap.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index bf86651d7464..d0b6c229b616 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1543,19 +1543,8 @@ check_old: * offsets corresponding to each page of the folio. Otherwise, * writeback could overwrite the new data in the swapfile. */ - if (!ret) { - unsigned type = swp_type(swp); - pgoff_t offset = swp_offset(swp); - struct zswap_entry *entry; - struct xarray *tree; - - for (index = 0; index < nr_pages; ++index) { - tree = swap_zswap_tree(swp_entry(type, offset + index)); - entry = xa_erase(tree, offset + index); - if (entry) - zswap_entry_free(entry); - } - } + if (!ret) + zswap_invalidate(swp_type(swp), swp_offset(swp), nr_pages); return ret; } From 21f514ad520459fbfa58ccdf171bb68148c2f5aa Mon Sep 17 00:00:00 2001 From: Huaisheng Ye Date: Wed, 9 Sep 2026 15:46:42 +0800 Subject: [PATCH 320/539] mm/hugetlb: account for allowed nodes when gathering surplus pages Hugetlb reservations are accounted globally, but hugetlb_acct_memory() also verifies that the current cpuset and MPOL_BIND policy contain enough free huge pages to add a new reservation. gather_surplus_pages() calculates its allocation shortfall from the global free and reserved counters. If the global pool has enough free pages, but those pages reside outside the nodes allowed by the task, it allocates no surplus pages. The subsequent allowed_mems_nr() check then rejects the reservation and mmap() fails with ENOMEM, even when nr_overcommit_hugepages permits allocating surplus pages on the allowed nodes. Calculate both the global shortfall and the shortfall within the allowed nodes, and allocate the larger of the two. Include surplus pages allocated outside hugetlb_lock in both calculations when rechecking after reacquiring the lock. These pages are constrained by alloc_nodemask, so they satisfy both shortages. Easy way to reproduce this issue with 2+ NUMA nodes system: # echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepage # echo 3 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages # echo 1 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_overcommit_hugepages # cd tools/testing/selftests/mm # numactl --membind=1 ./hugetlb-mmap 2 21 TAP version 13 # [INFO] detected hugetlb page size: 2048 KiB # [INFO] detected hugetlb page size: 1048576 KiB # 2048 kB hugepages 1..2 # Mapping 2 Mbytes Bail out! mmap: Cannot allocate memory (12) # Planned tests != run tests (2 != 0) # Totals: pass:0 fail:0 xfail:0 xpass:0 skip:0 error:0 This fixes hugetlb mappings when, for example, a task runs with MPOL_BIND on Node 1 while the existing free huge pages are on Node 0. Similar issue also could be found in ltp if the free pages of global pool reside outside the nodes allowed by the application. # cd ltp/testcases/kernel/mem/hugetlb/hugemmap/ # numactl --cpunodebind=0 --membind=1 ./hugemmap10 Link: https://lore.kernel.org/20260909074642.7308-1-yehuaisheng@open-hieco.net Fixes: e4e574b767ba ("hugetlb: Try to grow hugetlb pool for MAP_SHARED mappings") Signed-off-by: Huaisheng Ye Signed-off-by: Andrew Morton Acked-by: Muchun Song Cc: David Hildenbrand Cc: Oscar Salvador --- mm/hugetlb.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index c1f76bde9ac0..afd669640cc3 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -125,6 +125,7 @@ struct mutex *hugetlb_fault_mutex_table __ro_after_init; /* Forward declaration */ static int hugetlb_acct_memory(struct hstate *h, long delta); +static unsigned int allowed_mems_nr(struct hstate *h); static void hugetlb_vma_lock_free(struct vm_area_struct *vma); static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma); static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma); @@ -2252,6 +2253,19 @@ static nodemask_t *policy_mbind_nodemask(gfp_t gfp) return NULL; } +/* + * Reservations are globally accounted, but they must also be backed by free + * pages on nodes allowed by the current cpuset and MPOL_BIND policy. + */ +static long surplus_pages_needed(struct hstate *h, long delta, long allocated) +{ + long global_free = (long)h->free_huge_pages + allocated; + long allowed_free = (long)allowed_mems_nr(h) + allocated; + + return max((long)h->resv_huge_pages + delta - global_free, + delta - allowed_free); +} + /* * Increase the hugetlb pool such that it can accommodate a reservation * of size 'delta'. @@ -2274,7 +2288,7 @@ static int gather_surplus_pages(struct hstate *h, long delta) alloc_nodemask = cpuset_current_mems_allowed; lockdep_assert_held(&hugetlb_lock); - needed = (h->resv_huge_pages + delta) - h->free_huge_pages; + needed = surplus_pages_needed(h, delta, 0); if (needed <= 0) { h->resv_huge_pages += delta; return 0; @@ -2305,11 +2319,10 @@ retry: /* * After retaking hugetlb_lock, we need to recalculate 'needed' - * because either resv_huge_pages or free_huge_pages may have changed. + * because either resv_huge_pages or the free page counts may have changed. */ spin_lock_irq(&hugetlb_lock); - needed = (h->resv_huge_pages + delta) - - (h->free_huge_pages + allocated); + needed = surplus_pages_needed(h, delta, allocated); if (needed > 0) { if (alloc_ok) goto retry; From cc598b44a6bca0b33f10d8f9f041880a80c03e53 Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Thu, 10 Sep 2026 14:44:15 +0800 Subject: [PATCH 321/539] selftests/mm: fix ptrace PEEKDATA check in memfd_secret test try_ptrace() treats PTRACE_PEEKDATA return value as a boolean check. A successful read returns non-zero data (memory filled with 0x55), causing the test to incorrectly report PASS when secret memory protection is broken. Check the return value against -1 instead. The test should only pass when PTRACE_PEEKDATA fails, which means secret memory protection works. Link: https://lore.kernel.org/20260910064415.71623-1-hongfu.li@linux.dev Fixes: 76fe17ef588a ("secretmem: test: add basic selftest for memfd_secret(2)") Signed-off-by: Hongfu Li Signed-off-by: Andrew Morton Acked-by: Lorenzo Stoakes (ARM) Acked-by: Mike Rapoport (Microsoft) Cc: David Hildenbrand Cc: James Bottomley Cc: Liam R. Howlett Cc: Michal Hocko Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/memfd_secret.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/memfd_secret.c b/tools/testing/selftests/mm/memfd_secret.c index c55d84c5e613..aef774be87f1 100644 --- a/tools/testing/selftests/mm/memfd_secret.c +++ b/tools/testing/selftests/mm/memfd_secret.c @@ -145,7 +145,8 @@ static void try_ptrace(int fd, int pipefd[2]) exit(KSFT_FAIL); } - if (ptrace(PTRACE_PEEKDATA, ppid, mem, 0)) + /* PEEKDATA on secret memory must fail, else protection is broken. */ + if (ptrace(PTRACE_PEEKDATA, ppid, mem, 0) == -1) exit(KSFT_PASS); exit(KSFT_FAIL); From cc500013c86cb87845b7f2b2164296ae197495ce Mon Sep 17 00:00:00 2001 From: Qiqi Liu Date: Tue, 8 Sep 2026 18:23:56 +0800 Subject: [PATCH 322/539] mm: page_alloc: add missing hooks to bulk allocation path The bulk allocation path in alloc_pages_bulk_noprof() currently misses trace_mm_page_alloc() and kmsan_alloc_page() calls, leaving bulk-allocated pages invisible to ftrace/BPF/perf and leaving KMSAN shadow memory stale. Add both calls in the bulk loop to match the standard allocation path, placing them before set_page_refcounted() for consistency. The gfp mask passed to kmsan_alloc_page() is stripped of __GFP_RECLAIM because the bulk loop runs under the PCP spinlock, and KMSAN's stack depot allocation must not sleep. Both are no-ops when their respective features are disabled, so there is no overhead in production kernels. Link: https://lore.kernel.org/20260908102356.344075-1-liuqiqi@kylinos.cn Signed-off-by: Qiqi Liu Signed-off-by: Andrew Morton Suggested-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Gregory Price (Meta) Reviewed-by: Zi Yan Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan --- mm/page_alloc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c4dc61ec663e..cfafc9be9e30 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5261,6 +5261,8 @@ retry_this_zone: nr_account++; prep_new_page(page, 0, gfp, ALLOC_DEFAULT); + trace_mm_page_alloc(page, 0, gfp, ac.migratetype); + kmsan_alloc_page(page, 0, gfp & ~__GFP_RECLAIM); set_page_refcounted(page); page_array[nr_populated++] = page; } From 803f6a03f243ef544d8a9fe6c9314247b34cf02b Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Mon, 7 Sep 2026 19:57:28 +0900 Subject: [PATCH 323/539] zram: convert to SG-list zsmalloc object read API Patch series "zsmallc: remove old object read API". zram remains the only user of old zsmalloc object read API. This series removes the old API and converts zram to use the new SG-list based API. This patch (of 2): zram remains the last user of old zsmalloc object read API, that performed linearisation on the zsmalloc side. There is a new SG-list API, that has a bunch of benefits. Switch zram to SG-list zsmalloc object read API. Link: https://lore.kernel.org/20260907105739.1793316-1-senozhatsky@chromium.org Link: https://lore.kernel.org/20260907105739.1793316-2-senozhatsky@chromium.org Signed-off-by: Sergey Senozhatsky Signed-off-by: Andrew Morton Cc: Minchan Kim Cc: Nhat Pham --- drivers/block/zram/zcomp.c | 23 ++++++++++++-- drivers/block/zram/zcomp.h | 4 ++- drivers/block/zram/zram_drv.c | 56 ++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c index 974c4691887e..028e2f0f587e 100644 --- a/drivers/block/zram/zcomp.c +++ b/drivers/block/zram/zcomp.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -158,17 +160,32 @@ int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, } int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm, - const void *src, unsigned int src_len, void *dst) + struct scatterlist *sg, unsigned int src_len, void *dst) { struct zcomp_req req = { - .src = src, .dst = dst, .src_len = src_len, .dst_len = PAGE_SIZE, }; + void *src = NULL; + int ret; might_sleep(); - return comp->ops->decompress(comp->params, &zstrm->ctx, &req); + + if (sg_is_last(sg)) { + /* the object is contained within one page, read it in-place */ + src = kmap_local_page(sg_page(sg)); + req.src = src + sg->offset; + } else { + /* the object spans two pages, linearize it into local copy */ + sg_copy_to_buffer(sg, 2, zstrm->local_copy, src_len); + req.src = zstrm->local_copy; + } + + ret = comp->ops->decompress(comp->params, &zstrm->ctx, &req); + if (src) + kunmap_local(src); + return ret; } int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h index 81a0f3f6ff48..f38fd31f9e4e 100644 --- a/drivers/block/zram/zcomp.h +++ b/drivers/block/zram/zcomp.h @@ -5,6 +5,8 @@ #include +struct scatterlist; + #define ZCOMP_PARAM_NOT_SET INT_MIN struct deflate_params { @@ -91,6 +93,6 @@ void zcomp_stream_put(struct zcomp_strm *zstrm); int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, const void *src, unsigned int *dst_len); int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm, - const void *src, unsigned int src_len, void *dst); + struct scatterlist *sg, unsigned int src_len, void *dst); #endif /* _ZCOMP_H_ */ diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 2359eaa6f531..024402438bd1 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -1347,9 +1348,9 @@ static int decompress_bdev_page(struct zram *zram, struct page *page, unsigned long index) { struct zcomp_strm *zstrm; + struct scatterlist sg[1]; unsigned int size; int ret, prio; - void *src; slot_lock(zram, index); /* Since slot was unlocked we need to make sure it's still ZRAM_WB */ @@ -1368,13 +1369,18 @@ static int decompress_bdev_page(struct zram *zram, struct page *page, size = get_slot_size(zram, index); prio = get_slot_comp_priority(zram, index); + sg_init_table(sg, 1); + sg_set_page(sg, page, size, 0); + zstrm = zcomp_stream_get(zram->comps[prio]); - src = kmap_local_page(page); - ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, + ret = zcomp_decompress(zram->comps[prio], zstrm, sg, size, zstrm->local_copy); - if (!ret) - copy_page(src, zstrm->local_copy); - kunmap_local(src); + if (!ret) { + void *dst = kmap_local_page(page); + + copy_page(dst, zstrm->local_copy); + kunmap_local(dst); + } zcomp_stream_put(zstrm); slot_unlock(zram, index); @@ -2086,15 +2092,19 @@ static int read_same_filled_page(struct zram *zram, struct page *page, static int read_incompressible_page(struct zram *zram, struct page *page, unsigned long index) { + struct scatterlist sg[2]; unsigned long handle; void *src, *dst; handle = get_slot_handle(zram, index); - src = zs_obj_read_begin(zram->mem_pool, handle, PAGE_SIZE, NULL); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, PAGE_SIZE); + /* an incompressible object never spans two pages */ + src = kmap_local_page(sg_page(sg)); dst = kmap_local_page(page); copy_page(dst, src); kunmap_local(dst); - zs_obj_read_end(zram->mem_pool, handle, PAGE_SIZE, src); + kunmap_local(src); + zs_obj_read_sg_end(zram->mem_pool, handle); return 0; } @@ -2103,9 +2113,10 @@ static int read_compressed_page(struct zram *zram, struct page *page, unsigned long index) { struct zcomp_strm *zstrm; + struct scatterlist sg[2]; unsigned long handle; unsigned int size; - void *src, *dst; + void *dst; int ret, prio; handle = get_slot_handle(zram, index); @@ -2113,12 +2124,11 @@ static int read_compressed_page(struct zram *zram, struct page *page, prio = get_slot_comp_priority(zram, index); zstrm = zcomp_stream_get(zram->comps[prio]); - src = zs_obj_read_begin(zram->mem_pool, handle, size, - zstrm->local_copy); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size); dst = kmap_local_page(page); - ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, dst); + ret = zcomp_decompress(zram->comps[prio], zstrm, sg, size, dst); kunmap_local(dst); - zs_obj_read_end(zram->mem_pool, handle, size, src); + zs_obj_read_sg_end(zram->mem_pool, handle); zcomp_stream_put(zstrm); return ret; @@ -2128,25 +2138,23 @@ static int read_compressed_page(struct zram *zram, struct page *page, static int read_from_zspool_raw(struct zram *zram, struct page *page, unsigned long index) { - struct zcomp_strm *zstrm; + struct scatterlist sg[2]; unsigned long handle; unsigned int size; - void *src; + void *dst; handle = get_slot_handle(zram, index); size = get_slot_size(zram, index); /* - * We need to get stream just for ->local_copy buffer, in - * case if object spans two physical pages. No decompression - * takes place here, as we read raw compressed data. + * No decompression takes place here, we copy out raw compressed + * data directly into the destination page. */ - zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]); - src = zs_obj_read_begin(zram->mem_pool, handle, size, - zstrm->local_copy); - memcpy_to_page(page, 0, src, size); - zs_obj_read_end(zram->mem_pool, handle, size, src); - zcomp_stream_put(zstrm); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size); + dst = kmap_local_page(page); + sg_copy_to_buffer(sg, sg_nents(sg), dst, size); + kunmap_local(dst); + zs_obj_read_sg_end(zram->mem_pool, handle); memzero_page(page, size, PAGE_SIZE - size); From 7ade67b8d5b4792db0efa623b12b613255b4624a Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Mon, 7 Sep 2026 19:57:29 +0900 Subject: [PATCH 324/539] zsmalloc: remove old object read API There are no users left of the old API, all have switched to SG-list object read API. Remove it. Link: https://lore.kernel.org/20260907105739.1793316-3-senozhatsky@chromium.org Signed-off-by: Sergey Senozhatsky Signed-off-by: Andrew Morton Cc: Minchan Kim Cc: Nhat Pham --- include/linux/zsmalloc.h | 4 --- mm/zsmalloc.c | 77 ---------------------------------------- 2 files changed, 81 deletions(-) diff --git a/include/linux/zsmalloc.h b/include/linux/zsmalloc.h index 478410c880b1..5b7298a5026b 100644 --- a/include/linux/zsmalloc.h +++ b/include/linux/zsmalloc.h @@ -40,10 +40,6 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size); void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats); -void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *local_copy); -void zs_obj_read_end(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *handle_mem); void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle, struct scatterlist *sg, size_t mem_len); void zs_obj_read_sg_end(struct zs_pool *pool, unsigned long handle); diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 825022a7a328..11be37c43171 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -1134,83 +1134,6 @@ unsigned long zs_get_total_pages(struct zs_pool *pool) } EXPORT_SYMBOL_GPL(zs_get_total_pages); -void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *local_copy) -{ - struct zspage *zspage; - struct zpdesc *zpdesc; - unsigned long obj, off; - unsigned int obj_idx; - struct size_class *class; - void *addr; - - /* Guarantee we can get zspage from handle safely */ - read_lock(&pool->lock); - obj = handle_to_obj(handle); - obj_to_location(obj, &zpdesc, &obj_idx); - zspage = get_zspage(zpdesc); - - /* Make sure migration doesn't move any pages in this zspage */ - zspage_read_lock(zspage); - read_unlock(&pool->lock); - - class = zspage_class(pool, zspage); - off = offset_in_page(class->size * obj_idx); - - if (!ZsHugePage(zspage)) - off += ZS_HANDLE_SIZE; - - if (off + mem_len <= PAGE_SIZE) { - /* this object is contained entirely within a page */ - addr = kmap_local_zpdesc(zpdesc); - addr += off; - } else { - size_t sizes[2]; - - /* this object spans two pages */ - sizes[0] = PAGE_SIZE - off; - sizes[1] = mem_len - sizes[0]; - addr = local_copy; - - memcpy_from_page(addr, zpdesc_page(zpdesc), - off, sizes[0]); - zpdesc = get_next_zpdesc(zpdesc); - memcpy_from_page(addr + sizes[0], - zpdesc_page(zpdesc), - 0, sizes[1]); - } - - return addr; -} -EXPORT_SYMBOL_GPL(zs_obj_read_begin); - -void zs_obj_read_end(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *handle_mem) -{ - struct zspage *zspage; - struct zpdesc *zpdesc; - unsigned long obj, off; - unsigned int obj_idx; - struct size_class *class; - - obj = handle_to_obj(handle); - obj_to_location(obj, &zpdesc, &obj_idx); - zspage = get_zspage(zpdesc); - class = zspage_class(pool, zspage); - off = offset_in_page(class->size * obj_idx); - - if (!ZsHugePage(zspage)) - off += ZS_HANDLE_SIZE; - - if (off + mem_len <= PAGE_SIZE) { - handle_mem -= off; - kunmap_local(handle_mem); - } - - zspage_read_unlock(zspage); -} -EXPORT_SYMBOL_GPL(zs_obj_read_end); - void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle, struct scatterlist *sg, size_t mem_len) { From 103e9222968e9fefcdf8c4be0c60c647e0577e29 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Mon, 7 Sep 2026 17:13:53 +0800 Subject: [PATCH 325/539] mm, swap: fix potential NULL dereference when trying a sleep table allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm, swap: some random fixes and cleanups", v3. This series contains some random fixes and cleanups. More details can be found in respective patches. This patch (of 4): The root cause of this issue is because multi-tables are updated in non atomic context. To be more specific, the issue could be triggerred as following: swap_alloc_fast swap_cluster_populate() /* Try a sleep allocation */ spin_unlock(&ci->lock); swap_cluster_alloc_table() rcu_assign_pointer(ci->table, table); ci = swap_cluster_lock(si, offset) cluster_is_usable(ci, order) if (!cluster_table_is_alloced(ci)) // ok alloc_swap_scan_cluster() cluster_scan_range() __swap_table_get() /* free table when more table allocation fails */ ci->memcg_table = kzalloc_obj(*ci->memcg_table, gfp); if (!ci->memcg_table) swap_cluster_free_table() rcu_assign_pointer(ci->table, NULL); table = rcu_dereference_check(ci->table, lockdep_is_held(&ci->lock)); atomic_long_read(&table[off]); // NULL dereference Since memory order guarantee between ci->table, as well as between ci->table and ci->zero_bitmap, fix the issue by making tables visible at the end of swap_cluster_populate(). Current memory order guarantee is as following: On write side: rcu_assign_pointer(ci->table, table) will offer release to ensure zero_bitmap and memcg_table visible before ci->table. On read side: folio_alloc_swap swap_alloc_fast/swap_alloc_slow /* ci->table: protected by cluster lock */ swap_cluster_lock cluster_is_usable ... __swap_table_set ... swap_cluster_unlock mem_cgroup_try_charge_swap ... /* memcg_table: protected by cluster lock */ swap_cluster_get_and_lock __swap_cgroup_set swap_cluster_unlock swap_writeout swap_zeromap_folio_set /* zero_bitmap: protected by cluster lock */ swap_cluster_get_and_lock __swap_table_set_zero swap_cluster_unlock Link: https://lore.kernel.org/20260907091356.53026-1-shikemeng@huaweicloud.com Link: https://lore.kernel.org/20260907091356.53026-2-shikemeng@huaweicloud.com Fixes: b197d41462c2 ("mm/memcg, swap: store cgroup id in cluster table directly") Signed-off-by: Kemeng Shi Signed-off-by: Andrew Morton Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Nhat Pham Cc: Kairui Song Cc: Luiz Capitulino Cc: Youngjun Park --- mm/swapfile.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 2c263563b70e..8df8b2c2e5b4 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -416,6 +416,17 @@ static void swap_cluster_free_table_folio_rcu_cb(struct rcu_head *head) folio_put(folio); } +static void swap_cluster_free_count_table(struct swap_table *table) +{ + if (!SWP_TABLE_USE_PAGE) { + kmem_cache_free(swap_table_cachep, table); + return; + } + + call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head), + swap_cluster_free_table_folio_rcu_cb); +} + static void swap_cluster_free_table(struct swap_cluster_info *ci) { struct swap_table *table; @@ -435,13 +446,7 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci) return; rcu_assign_pointer(ci->table, NULL); - if (!SWP_TABLE_USE_PAGE) { - kmem_cache_free(swap_table_cachep, table); - return; - } - - call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head), - swap_cluster_free_table_folio_rcu_cb); + swap_cluster_free_count_table(table); } static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp) @@ -464,14 +469,12 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp) if (!table) return -ENOMEM; - rcu_assign_pointer(ci->table, table); - #ifdef CONFIG_MEMCG if (!mem_cgroup_disabled()) { VM_WARN_ON_ONCE(ci->memcg_table); ci->memcg_table = kzalloc_obj(*ci->memcg_table, gfp); if (!ci->memcg_table) { - swap_cluster_free_table(ci); + swap_cluster_free_count_table(table); return -ENOMEM; } } @@ -482,9 +485,16 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp) ci->zero_bitmap = bitmap_zalloc(SWAPFILE_CLUSTER, gfp); if (!ci->zero_bitmap) { swap_cluster_free_table(ci); + swap_cluster_free_count_table(table); return -ENOMEM; } #endif + + /* + * Make tables visible to cluster_is_usable() after everything is + * ready. + */ + rcu_assign_pointer(ci->table, table); return 0; } From be0291c3c43f8e43e96621e505583ee6b76f0718 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Mon, 7 Sep 2026 17:13:54 +0800 Subject: [PATCH 326/539] mm, swap: move setup_swap_clusters_info() after SWP_SOLIDSTATE initialization In setup_swap_clusters_info(), SWP_SOLIDSTATE is used to decide global_cluster allocation. Move setup_swap_clusters_info() after SWP_SOLIDSTATE initialization to avoid unneeded global_cluster allocation. Link: https://lore.kernel.org/20260907091356.53026-3-shikemeng@huaweicloud.com Fixes: 451c6326105b ("mm, swap: clean up swapon process and locking") Signed-off-by: Kemeng Shi Signed-off-by: Andrew Morton Reviewed-by: Luiz Capitulino Acked-by: Kairui Song Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Nhat Pham Cc: Youngjun Park --- mm/swapfile.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 8df8b2c2e5b4..40ceab21ed5b 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -3763,11 +3763,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) maxpages = si->max; - /* Set up the swap cluster info */ - error = setup_swap_clusters_info(si, swap_header, maxpages); - if (error) - goto bad_swap_unlock_inode; - if (si->bdev && bdev_stable_writes(si->bdev)) si->flags |= SWP_STABLE_WRITES; @@ -3781,6 +3776,14 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) inced_nr_rotate_swap = true; } + /* + * Set up the swap cluster info after SWP_ flags handling as + * setup_swap_clusters_info() checks SWP_SOLIDSTATE. + */ + error = setup_swap_clusters_info(si, swap_header, maxpages); + if (error) + goto bad_swap_unlock_inode; + if ((swap_flags & SWAP_FLAG_DISCARD) && si->bdev && bdev_max_discard_sectors(si->bdev)) { /* From 868346102067b12cbf2d639583079e0cb048ea55 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Mon, 7 Sep 2026 17:13:55 +0800 Subject: [PATCH 327/539] mm, swap: return early from swap_extend_table_try_free() on first non-zero entry Return immediately when the first non-zero swap count is found as any non-zero swap count prevents freeing extend_table and further iteration is pointless. Link: https://lore.kernel.org/20260907091356.53026-4-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Signed-off-by: Andrew Morton Reviewed-by: Youngjun Park Acked-by: Kairui Song Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Luiz Capitulino Cc: Nhat Pham --- mm/swapfile.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 40ceab21ed5b..7b35ca90776a 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1523,20 +1523,17 @@ int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp) static void swap_extend_table_try_free(struct swap_cluster_info *ci) { unsigned long i; - bool can_free = true; if (!ci->extend_table) return; for (i = 0; i < SWAPFILE_CLUSTER; i++) { if (ci->extend_table[i]) - can_free = false; + return; } - if (can_free) { - kfree(ci->extend_table); - ci->extend_table = NULL; - } + kfree(ci->extend_table); + ci->extend_table = NULL; } /* Decrease the swap count of one slot, without freeing it */ From 63a08a1124ae6658d795acaefcc651ad41cabb16 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Mon, 7 Sep 2026 17:13:56 +0800 Subject: [PATCH 328/539] mm, swap: remove unneeded swap_extend_table_try_free() in swap_dup_entries_cluster() Since commit 0475fde0f68de ("mm, swap: avoid leaving unused extend table after alloc race"), extend table is always allocated when any swap count reach MAX - 1 and is always freed when swap count decrease to MAX - 1 or MAX - 2 with cluster lock held. So the extend table will always be freed properly when decrease swap count in __swap_cluster_put_entry(). So swap_extend_table_try_free() outside of __swap_cluster_put_entry() is unneeded and can be removed. Link: https://lore.kernel.org/20260907091356.53026-5-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Signed-off-by: Andrew Morton Reviewed-by: Youngjun Park Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Kairui Song Cc: Luiz Capitulino Cc: Nhat Pham --- mm/swapfile.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 7b35ca90776a..2cd0d0ba966c 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1725,7 +1725,6 @@ restart: failed: while (ci_off-- > ci_start) __swap_cluster_put_entry(ci, ci_off); - swap_extend_table_try_free(ci); swap_cluster_unlock(ci); return err; } From 9ee6e52616a3231a34236220fb901675fda40291 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Sat, 26 Sep 2026 11:41:06 +0100 Subject: [PATCH 329/539] mm: move drivers/char/mem.c to mm/char-mem.c The memory character driver implements several mm-specific features and is always compiled into the kernel, so move it to mm/ where it belongs. Among other things the driver implements /dev/mem which provides raw access to physical memory, and /dev/zero which either allows mapping of a shmem region (if mapped with MAP_SHARED) or, uniquely, anonymous memory (if mapped MAP_PRIVATE). This change lays the foundations to allow MAP_PRIVATE-/dev/zero to be mapped precisely the same as anonymous memory is mapped as currently it is an edge case within mm. Also update a couple of comments that reference 'drivers/char/mem.c' to reference 'mm/char-mem.c'. Link: https://lore.kernel.org/20260926-map-private-dev-zero-v3-1-d4781e84ccfc@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Jann Horn Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Hugh Dickins Cc: Baolin Wang Cc: Matthew Wilcox (Oracle) Cc: Jan Kara --- MAINTAINERS | 4 ++-- drivers/char/Makefile | 2 +- mm/Makefile | 3 ++- drivers/char/mem.c => mm/char-mem.c | 2 +- mm/shmem.c | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) rename drivers/char/mem.c => mm/char-mem.c (99%) diff --git a/MAINTAINERS b/MAINTAINERS index 7235a92ff879..a7ff3349a524 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17269,7 +17269,7 @@ F: Documentation/ABI/testing/sysfs-kernel-mm-cma F: Documentation/ABI/testing/sysfs-kernel-mm-numa F: Documentation/admin-guide/mm/ F: Documentation/mm/ -F: drivers/char/mem.c +F: mm/char-mem.c F: include/linux/cma.h F: include/linux/dmapool.h F: include/linux/ioremap.h @@ -17493,7 +17493,7 @@ L: linux-mm@kvack.org S: Maintained W: http://www.linux-mm.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm -F: drivers/char/mem.c +F: mm/char-mem.c F: include/trace/events/mmap.h F: fs/proc/task_mmu.c F: fs/proc/task_nommu.c diff --git a/drivers/char/Makefile b/drivers/char/Makefile index a46d7bf7c4c8..bb3bf66937b3 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -3,7 +3,7 @@ # Makefile for the kernel character device drivers. # -obj-y += mem.o random.o +obj-y += random.o obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o obj-y += misc.o obj-$(CONFIG_TEST_MISC_MINOR) += misc_minor_kunit.o diff --git a/mm/Makefile b/mm/Makefile index e7245cb88c66..2a3ec53d62ee 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -55,7 +55,8 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ mm_init.o percpu.o slab_common.o \ compaction.o show_mem.o \ interval_tree.o list_lru.o workingset.o \ - debug.o gup.o mmap_lock.o vma_init.o $(mmu-y) + debug.o gup.o mmap_lock.o vma_init.o char-mem.o \ + $(mmu-y) # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o diff --git a/drivers/char/mem.c b/mm/char-mem.c similarity index 99% rename from drivers/char/mem.c rename to mm/char-mem.c index 5b93c92c2cf1..3cc48054a66e 100644 --- a/drivers/char/mem.c +++ b/mm/char-mem.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * linux/drivers/char/mem.c + * mm/char-mem.c * * Copyright (C) 1991, 1992 Linus Torvalds * diff --git a/mm/shmem.c b/mm/shmem.c index 84f0a2eb85fe..05bc7c3aa525 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2984,7 +2984,7 @@ unsigned long shmem_get_unmapped_area(struct file *file, sb = file_inode(file)->i_sb; } else { /* - * Called directly from mm/mmap.c, or drivers/char/mem.c + * Called directly from mm/mmap.c, or mm/char-mem.c * for "/dev/zero", to create a shared anonymous object. */ if (IS_ERR(shm_mnt)) From 4b019e056f6cbc6e8a5d0525a5e0d48d55b5e2dc Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Sat, 26 Sep 2026 11:41:07 +0100 Subject: [PATCH 330/539] mm: implement file_is_dev_zero() to uniquely identify /dev/zero To lay the foundation for a future change that converts MAP_PRIVATE-/dev/zero mappings to be truly anonymous, add the ability to uniquely identify these mappings. With the memory character device now part of mm/ this is trivially achievable through a file_is_dev_zero() predicate that simply tests that the file operation hooks are zero_fops. Also update userland VMA tests to expose file_is_dev_zero() and provide a stub zero_fops for testing. Link: https://lore.kernel.org/20260926-map-private-dev-zero-v3-2-d4781e84ccfc@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Jann Horn Cc: Pedro Falcato Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Hugh Dickins Cc: Baolin Wang Cc: Matthew Wilcox (Oracle) Cc: Jan Kara --- mm/char-mem.c | 13 +++++++++++++ mm/internal.h | 3 +++ tools/testing/vma/include/dup.h | 7 +++++++ tools/testing/vma/shared.c | 9 +++++++++ 4 files changed, 32 insertions(+) diff --git a/mm/char-mem.c b/mm/char-mem.c index 3cc48054a66e..87fb71a01191 100644 --- a/mm/char-mem.c +++ b/mm/char-mem.c @@ -31,6 +31,8 @@ #include #include +#include "internal.h" + #define DEVMEM_MINOR 1 #define DEVPORT_MINOR 4 @@ -707,6 +709,17 @@ static const struct memdev { #endif }; +/** + * file_is_dev_zero() - is the specified @file associated with the /dev/zero + * driver? + * @file: File to test. + * Returns: true if it is, false otherwise. + */ +bool file_is_dev_zero(const struct file *file) +{ + return file && file->f_op == &zero_fops; +} + static int memory_open(struct inode *inode, struct file *filp) { int minor; diff --git a/mm/internal.h b/mm/internal.h index e16f1250b25c..5d474e5f7709 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1638,4 +1638,7 @@ static inline bool can_spin_trylock(void) return true; } +/* char-mem.c */ +bool file_is_dev_zero(const struct file *file); + #endif /* __MM_INTERNAL_H */ diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 57046d8ac81d..0d1a2ac88922 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1641,3 +1641,10 @@ static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, return pgoff; } + +extern const struct file_operations zero_fops; + +static inline bool file_is_dev_zero(const struct file *file) +{ + return file && file->f_op == &zero_fops; +} diff --git a/tools/testing/vma/shared.c b/tools/testing/vma/shared.c index 4a39c9d50489..8c4826499f40 100644 --- a/tools/testing/vma/shared.c +++ b/tools/testing/vma/shared.c @@ -12,6 +12,15 @@ const struct vm_operations_struct vma_dummy_vm_ops; struct anon_vma dummy_anon_vma; struct task_struct __current; +static int mmap_zero_prepare(struct vm_area_desc *desc) +{ + return 0; +} + +const struct file_operations zero_fops = { + .mmap_prepare = mmap_zero_prepare, +}; + struct vm_area_struct *alloc_vma(struct mm_struct *mm, unsigned long start, unsigned long end, pgoff_t pgoff, vma_flags_t vma_flags) From fe73aa910ea53ed557998229fd1c124d88bd7625 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Sat, 26 Sep 2026 11:41:08 +0100 Subject: [PATCH 331/539] mm/vma: only permit MAP_PRIVATE /dev/zero to be mapped anonymous In order to use mmap_prepare() with MAP_PRIVATE mappings of /dev/zero without the success_hook hack we explicitly permitted mmap_prepare handlers to set NULL vm_ops. However this is dangerous and we really only want to allow this for MAP_PRIVATE-mapped /dev/zero. Therefore use the newly introduced file_is_dev_zero() to uniquely identify MAP_PRIVATE-/dev/zero mappings and only permit this behaviour for them. Then, remove all ability for mmap_prepare or mmap hooks to set a VMA anonymous and update mmap_zero_prepare() to leave it to the core mmap code to do so. Note that this disallows nested MAP_PRIVATE-mappings of /dev/zero regions. Doing this would be broken in any case. We therefore do not need to update the mmap_prepare() compatibility layer to reflect these changes, as the mmap hook check suffices to disallow this behaviour. Now we're setting vma->vm_ops to NULL for an mmap_prepare-initialised MAP_PRIVATE-/dev/zero mapping, we have to avoid a subtle issue when updating user-defined fields via set_vma_user_defined_fields(). The default for vma->vm_ops for all mmap_prepare-initialised mappings is vma_dummy_vm_ops, so map->vm_ops will be set to this and setting vma->vm_ops to this will render the VMA mistakenly non-anon. In general, we should never be setting user-defined fields for an anonymous VMA, so explicitly check for this to avoid doing so for the one case where a mapping can be both mmap_prepare and anonymous. In the case of legacy ->mmap hooks some drivers may set vma->vm_ops NULL believing this is the equivalent of setting no VMA operations. Therefore update mmap_file() to correct this by setting dummy VMA operations if this occurs. An example of this is drm_gem_shmem_mmap() which deliberately clears vma->vm_ops before handing the VMA to dma-buf. Cases such as this will be updated when they are converted to mmap_prepare. Also, in order to avoid a single commit bisection hazard, add a temporary workaround to set the VMA anonymous only after vma->vm_file is assigned in __mmap_new_file_vma(). This is because vma_set_range() calls vma_set_pgoff() and assert_sane_pgoff() in turn, prior to the vma->vm_file being assigned. If we set the VMA anonymous early then this assert will fail. This is removed in the subsequent commit. Link: https://lore.kernel.org/20260926-map-private-dev-zero-v3-3-d4781e84ccfc@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Jann Horn Cc: Pedro Falcato Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Hugh Dickins Cc: Baolin Wang Cc: Matthew Wilcox (Oracle) Cc: Jan Kara --- mm/char-mem.c | 6 +----- mm/internal.h | 17 ++++++++++------- mm/vma.c | 33 +++++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/mm/char-mem.c b/mm/char-mem.c index 87fb71a01191..e9f94d2b5133 100644 --- a/mm/char-mem.c +++ b/mm/char-mem.c @@ -508,11 +508,7 @@ static int mmap_zero_prepare(struct vm_area_desc *desc) if (vma_desc_test(desc, VMA_MAYSHARE_BIT)) return shmem_zero_setup_desc(desc); - /* - * This is a highly unique situation where we mark a MAP_PRIVATE mapping - * of /dev/zero anonymous, despite it not being. - */ - vma_desc_set_anonymous(desc); + /* MAP_PRIVATE semantics are taken care of for us by core mm. */ return 0; } diff --git a/mm/internal.h b/mm/internal.h index 5d474e5f7709..da14c56fb24e 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -226,15 +226,18 @@ static inline int mmap_file(struct file *file, struct vm_area_struct *vma) { int err = vfs_mmap(file, vma); - if (likely(!err)) - return 0; - /* - * OK, we tried to call the file hook for mmap(), but an error - * arose. The mapping is in an inconsistent state and we must not invoke - * any further hooks on it. + * Either we tried to call the file hook for mmap() and an error arose + * or a driver set vma->vm_ops = NULL intending there to be no VMA + * operations. + * + * In the former case the VMA is in an inconsistent state and we mustn't + * invoke any further hooks on it, in the latter case the hook actually + * wanted no further hooks to be invoked, so fix both by setting dummy + * VMA ops. */ - vma->vm_ops = &vma_dummy_vm_ops; + if (unlikely(err || !vma->vm_ops)) + vma->vm_ops = &vma_dummy_vm_ops; return err; } diff --git a/mm/vma.c b/mm/vma.c index 6c35f0d775ab..61c04c063c5f 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2644,6 +2644,19 @@ static int __mmap_new_file_vma(struct mmap_state *map, return 0; } +static bool map_is_private(const struct mmap_state *map) +{ + return !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); +} + +static bool map_is_anon(const struct mmap_state *map) +{ + if (!map_is_private(map)) + return false; + + return !map->file || file_is_dev_zero(map->file); +} + /* * __mmap_new_vma() - Allocate a new VMA for the region, as merging was not * possible. @@ -2657,8 +2670,7 @@ static int __mmap_new_file_vma(struct mmap_state *map, static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, struct mmap_action *action) { - const bool is_anon = !map->file && - !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); + const bool is_anon = map_is_anon(map); struct vma_iterator *vmi = map->vmi; int error = 0; struct vm_area_struct *vma; @@ -2674,7 +2686,7 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, vma_iter_config(vmi, map->addr, map->end); - if (is_anon) + if (is_anon && !map->file) vma_set_anonymous(vma); vma_set_range(vma, map->addr, map->end, map->pgoff, map->anon_pgoff); @@ -2692,6 +2704,10 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, else if (!is_anon) error = shmem_zero_setup(vma); + /* Temporary MAP_PRIVATE-/dev/zero workaround. */ + if (is_anon && map->file) + vma_set_anonymous(vma); + if (error) goto free_iter_vma; @@ -2800,6 +2816,10 @@ static int call_mmap_prepare(struct mmap_state *map, if (err) return err; + /* It's invalid for mmap_prepare hooks to clear vm_ops. */ + if (!desc->vm_ops) + return -EINVAL; + err = call_action_prepare(map, desc); if (err) return err; @@ -2822,10 +2842,7 @@ static int call_mmap_prepare(struct mmap_state *map, static void set_vma_user_defined_fields(struct vm_area_struct *vma, struct mmap_state *map) { - if (map->vm_ops) - vma->vm_ops = map->vm_ops; - else /* Only /dev/zero should do this. */ - vma_set_anonymous(vma); + vma->vm_ops = map->vm_ops; vma->vm_private_data = map->vm_private_data; } @@ -2907,7 +2924,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, allocated_new = true; } - if (have_mmap_prepare && allocated_new) + if (have_mmap_prepare && !map_is_anon(&map) && allocated_new) set_vma_user_defined_fields(vma, &map); __mmap_complete(&map, vma); From e59c9a971a9f4924b54e8abad3f5395c389f3f0f Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Sat, 26 Sep 2026 11:41:09 +0100 Subject: [PATCH 332/539] mm/vma: make MAP_PRIVATE-mapped /dev/zero mappings truly anonymous When mapping /dev/zero with MAP_PRIVATE, one ends up with strange VMAs originating from Linux's distant past. These have vma->vm_file set but NULL vma->vm_ops, meaning they satisfy vma_is_anonymous() but otherwise resemble a file-backed VMA. The introduction of anonymous page offsets and their subsequent use as indexes for MAP_PRIVATE-file-backed mappings mean the rmap does the right thing with these but we are left with inconsistencies. The vma_start_pgoff(vma) == vma_start_anon_pgoff(vma) invariant is true for all other anonymous VMAs, but not these. These VMAs are also observable as files in /proc//[maps, smaps, map_files] but otherwise behave like anonymous mappings. Therefore let's make these VMAs actually anonymous at mapping time which will activate the anonymous code path for mappings. This means we no longer have to account for this discrepancy anywhere and no longer have to think about these at all. This is user-observable, as MAP_PRIVATE-/dev/zero will no longer appear in procfs as a file-backed mapping, but the impact of this change should be low as likely nobody is relying upon this. However in any case, in using MAP_PRIVATE-/dev/zero they are explicitly asking anonymous memory, so no longer seeing these as file mappings is in fact correct. A previous commit gave us file_is_dev_zero() to positively identify these mappings, so we expressly only do so for these alone. Update assert_sane_pgoff(), the comment for vma_start_pgoff() and linear_anon_page_index() to reflect the change. We make this change in call_mmap_prepare() alone as /dev/zero has been converted to an mmap_prepare hook and we do not permit nested MAP_PRIVATE mapping of /dev/zero. We also remove the now defunct vma_desc_set_anonymous() and eliminate the temporary bisection hazard fix from the previous commit. Also update the VMA userland tests to reflect the change. Finally, update the procfs self tests proc-self-map-files-001 and proc-self-map-files-002 which both intend to map an arbitrary file MAP_PRIVATE then assert procfs state, but happen to choose /dev/zero. Fix them by updating these to /proc/self/exe which is guaranteed to be present if procfs is mounted. Link: https://lore.kernel.org/20260926-map-private-dev-zero-v3-4-d4781e84ccfc@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Jann Horn Cc: Pedro Falcato Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Hugh Dickins Cc: Baolin Wang Cc: Matthew Wilcox (Oracle) Cc: Jan Kara --- include/linux/mm.h | 10 ++----- include/linux/pagemap.h | 3 +-- mm/vma.c | 26 ++++++++++++------- mm/vma.h | 3 --- .../selftests/proc/proc-self-map-files-001.c | 2 +- .../selftests/proc/proc-self-map-files-002.c | 2 +- tools/testing/vma/include/dup.h | 3 +-- 7 files changed, 23 insertions(+), 26 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index c105a3758915..c49ef99b4413 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1529,11 +1529,6 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) vma->vm_ops = NULL; } -static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) -{ - desc->vm_ops = NULL; -} - static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; @@ -4389,9 +4384,8 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the * page offset within the file. * - * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies - * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap - * regions have their page offset set to the first PFN in the range. + * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions + * have their page offset set to the first PFN in the range. * * Returns: The page offset of the start of @vma. */ diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 0adfa6605653..939f3a5e973f 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1128,8 +1128,7 @@ static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, const pgoff_t pgoff = __linear_anon_page_index(vma, address); VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); - /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ - if (vma_is_anonymous(vma) && !vma->vm_file) + if (vma_is_anonymous(vma)) VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); return pgoff; diff --git a/mm/vma.c b/mm/vma.c index 61c04c063c5f..f56317ee8247 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2644,6 +2644,13 @@ static int __mmap_new_file_vma(struct mmap_state *map, return 0; } +static void map_set_anon(struct mmap_state *map) +{ + map->file = NULL; + map->vm_ops = NULL; + map->pgoff = map->addr >> PAGE_SHIFT; +} + static bool map_is_private(const struct mmap_state *map) { return !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); @@ -2651,10 +2658,7 @@ static bool map_is_private(const struct mmap_state *map) static bool map_is_anon(const struct mmap_state *map) { - if (!map_is_private(map)) - return false; - - return !map->file || file_is_dev_zero(map->file); + return map_is_private(map) && !map->file; } /* @@ -2686,7 +2690,7 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, vma_iter_config(vmi, map->addr, map->end); - if (is_anon && !map->file) + if (is_anon) vma_set_anonymous(vma); vma_set_range(vma, map->addr, map->end, map->pgoff, map->anon_pgoff); @@ -2704,10 +2708,6 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, else if (!is_anon) error = shmem_zero_setup(vma); - /* Temporary MAP_PRIVATE-/dev/zero workaround. */ - if (is_anon && map->file) - vma_set_anonymous(vma); - if (error) goto free_iter_vma; @@ -2836,6 +2836,14 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; + /* + * MAP_PRIVATE-/dev/zero mappings are an ancient way of getting + * anonymous mappings. Rather than allowing these mappings to be odd + * outliers, simply make them truly anonymous. + */ + if (map_is_private(map) && file_is_dev_zero(map->file)) + map_set_anon(map); + return 0; } diff --git a/mm/vma.h b/mm/vma.h index 36973abaa015..f856d9ace3a6 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -267,9 +267,6 @@ static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) */ if (!vma_is_anonymous(vma)) return; - /* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */ - if (vma->vm_file) - return; /* If faulted in, could have been remapped. */ if (vma->anon_vma) return; diff --git a/tools/testing/selftests/proc/proc-self-map-files-001.c b/tools/testing/selftests/proc/proc-self-map-files-001.c index 4209c64283d6..bbca9f9e2743 100644 --- a/tools/testing/selftests/proc/proc-self-map-files-001.c +++ b/tools/testing/selftests/proc/proc-self-map-files-001.c @@ -51,7 +51,7 @@ int main(void) int fd; unsigned long a, b; - fd = open("/dev/zero", O_RDONLY); + fd = open("/proc/self/exe", O_RDONLY); if (fd == -1) return 1; diff --git a/tools/testing/selftests/proc/proc-self-map-files-002.c b/tools/testing/selftests/proc/proc-self-map-files-002.c index e6aa00a183bc..5786cdffbbf6 100644 --- a/tools/testing/selftests/proc/proc-self-map-files-002.c +++ b/tools/testing/selftests/proc/proc-self-map-files-002.c @@ -57,7 +57,7 @@ int main(void) int fd; unsigned long a, b; - fd = open("/dev/zero", O_RDONLY); + fd = open("/proc/self/exe", O_RDONLY); if (fd == -1) return 1; diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 0d1a2ac88922..16c09dac59d9 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1635,8 +1635,7 @@ static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, const pgoff_t pgoff = __linear_anon_page_index(vma, address); VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); - /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ - if (vma_is_anonymous(vma) && !vma->vm_file) + if (vma_is_anonymous(vma)) VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); return pgoff; From 9a1d31e5a6f3a2989a9afba7a0bc68e7a595604e Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Sat, 26 Sep 2026 11:41:10 +0100 Subject: [PATCH 333/539] tools/testing/vma: add test to assert MAP_PRIVATE-/dev/zero is anon Now we've made MAP_PRIVATE-mapped /dev/zero mappings truly anonymous, add a VMA userland test to assert that this is the case and everything is as we would expect for an anonymous mapping. Link: https://lore.kernel.org/20260926-map-private-dev-zero-v3-5-d4781e84ccfc@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Jann Horn Cc: Pedro Falcato Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Hugh Dickins Cc: Baolin Wang Cc: Matthew Wilcox (Oracle) Cc: Jan Kara --- tools/testing/vma/tests/mmap.c | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb..fa73faff2262 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -45,7 +45,44 @@ static bool test_mmap_region_basic(void) return true; } +static bool test_pure_anon_dev_zero(void) +{ + const vma_flags_t vma_flags = mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT); + struct file file = { + .f_op = &zero_fops, + }; + struct mm_struct mm = {}; + struct vm_area_struct *vma; + unsigned long addr; + VMA_ITERATOR(vmi, &mm, 0); + + current->mm = &mm; + + /* + * Map a MAP_PRIVATE-/dev/zero mapping at address 0x300000 with a page + * offset of 0x10, which we expect to be reset to the anonymous page + * offset. + */ + addr = __mmap_region(&file, 0x300000, 0x3000, vma_flags, 0x10, NULL); + ASSERT_EQ(addr, 0x300000); + + /* Assert that it truly is an anonymous mapping. */ + vma = vma_lookup(&mm, addr); + ASSERT_NE(vma, NULL); + ASSERT_TRUE(vma_is_anonymous(vma)); + ASSERT_EQ(vma->vm_file, NULL); + ASSERT_EQ(vma->vm_private_data, NULL); + /* Expect anonymous page offsets. */ + ASSERT_EQ(vma->vm_pgoff, 0x300); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0x300); + + cleanup_mm(&mm, &vmi); + return true; +} + static void run_mmap_tests(int *num_tests, int *num_fail) { TEST(mmap_region_basic); + TEST(pure_anon_dev_zero); } From 41938a7834c6f89dc9489c9e9d32df88cb9d0ab4 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Sat, 26 Sep 2026 11:41:11 +0100 Subject: [PATCH 334/539] tools/testing/selftests/mm: add MAP_PRIVATE-/dev/zero merge tests Assert that MAP_PRIVATE-mapped /dev/zero mappings behave like they are anonymous. Test both unfaulted and faulted/unfaulted merges with page offset 0 which would not merge if the mappings were treated as if they were file-backed. With the recent change that makes them behave as pure anonymous mappings, the merges should succeed as their page offsets are equal to their anonymous page offsets. Link: https://lore.kernel.org/20260926-map-private-dev-zero-v3-6-d4781e84ccfc@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Jann Horn Cc: Pedro Falcato Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Hugh Dickins Cc: Baolin Wang Cc: Matthew Wilcox (Oracle) Cc: Jan Kara --- tools/testing/selftests/mm/merge.c | 95 ++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tools/testing/selftests/mm/merge.c b/tools/testing/selftests/mm/merge.c index 52b8727b6628..7dd2933417c1 100644 --- a/tools/testing/selftests/mm/merge.c +++ b/tools/testing/selftests/mm/merge.c @@ -1362,6 +1362,101 @@ TEST_F(merge, anon_and_page_offset_mismatch_memfd) ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 5 * page_size); } +TEST_F(merge, merge_map_private_dev_zero_unfaulted) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd_zero; + + if (access("/dev/zero", F_OK)) + SKIP(return, "No /dev/zero."); + fd_zero = open("/dev/zero", O_RDWR); + ASSERT_NE(fd_zero, -1); + + /* + * Map two MAP_PRIVATE-/dev/zero VMAs next to one another with offset 0 + * each. + * + * With these being made truly anonymous upon mapping, they will + * merge. If they were file-backed VMAs the page offsets would prevent + * the merge: + * + * |-----||------| |-------------| + * | ptr || ptr2 | -> | ptr | + * |-----||------| |-------------| + */ + ptr = mmap(carveout, 5 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + ptr2 = mmap(&carveout[5 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + close(fd_zero); + ASSERT_NE(ptr, MAP_FAILED); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they merged. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size); +} + +TEST_F(merge, merge_map_private_dev_zero_faulted_unfaulted) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd_zero; + + if (access("/dev/zero", F_OK)) + SKIP(return, "No /dev/zero."); + fd_zero = open("/dev/zero", O_RDWR); + ASSERT_NE(fd_zero, -1); + + /* + * Map a MAP_PRIVATE mapping of /dev/zero with page offset 0, then fault + * it in: + * + * |-------------------------------| + * | faulted | + * |-------------------------------| + */ + ptr = mmap(carveout, 15 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + ASSERT_NE(ptr, MAP_FAILED); + memset(ptr, 'x', 15 * page_size); + + /* + * Unmap the middle: + * + * |---------| |---------| + * | faulted | | faulted | + * |---------| |---------| + */ + ASSERT_EQ(munmap(&ptr[5 * page_size], 5 * page_size), 0); + + /* + * Map in a new unfaulted mapping in the middle with page offset 0 - + * this should merge and would not if it were treated as a file rather + * than pure anon: + * + * |---------|-----------|---------| + * | faulted | unfaulted | faulted | + * |---------|-----------|---------| + */ + ptr2 = mmap(&carveout[5 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, + fd_zero, 0); + close(fd_zero); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they merged. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size); +} + TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev) { struct procmap_fd *procmap = &self->procmap; From 9ecb05623cd6af751c38ef7f31391795d75c97b9 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 2 Sep 2026 11:10:11 +0300 Subject: [PATCH 335/539] docs/core-api: memory-allocation: add k[mz]alloc_obj() and clarify kmalloc Patch series "docs/core-api: memory-allocation: add k[mz]alloc_obj() and clarify kmalloc", v2. Update the memory-allocation guide to describe k[mz]alloc_obj() and clarify description of kmalloc() size limitations. And when I realized that get_maintainers.pl does not list any of mm people for that patch I added the MAINTAINERS update as well :) This patch (of 2): Since v7.0 the most used memory allocation function is kzalloc_obj(). Update the memory-allocation guide to describe k[mz]alloc_obj() family and make kzalloc_obj() the first answer to "How should I allocate memory?" question. While on it, clarify description of kmalloc() size limitations. Link: https://lore.kernel.org/20260902-docs-memalloc-guide-v2-0-218c1a4dcb80@kernel.org Link: https://lore.kernel.org/20260902-docs-memalloc-guide-v2-1-218c1a4dcb80@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Reviewed-by: Suren Baghdasaryan Acked-by: SJ Park Acked-by: Vlastimil Babka (SUSE) Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Randy Dunlap --- Documentation/core-api/memory-allocation.rst | 30 +++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/Documentation/core-api/memory-allocation.rst b/Documentation/core-api/memory-allocation.rst index 0f19dd524323..823f7fa57429 100644 --- a/Documentation/core-api/memory-allocation.rst +++ b/Documentation/core-api/memory-allocation.rst @@ -19,6 +19,12 @@ Diversity of the allocation APIs combined with the numerous GFP flags makes the question "How should I allocate memory?" not that easy to answer, although very likely you should use +:: + + kzalloc_obj(); + +or + :: kzalloc(, GFP_KERNEL); @@ -139,10 +145,13 @@ allocate memory for an array, there are kmalloc_array() and kcalloc() helpers. The helpers struct_size(), array_size() and array3_size() can be used to safely calculate object sizes without overflowing. -The maximal size of a chunk that can be allocated with `kmalloc` is -limited. The actual limit depends on the hardware and the kernel -configuration, but it is a good practice to use `kmalloc` for objects -smaller than page size. +Since 7.0 there are type aware kmalloc-family helpers that let you safely and +conveniently allocate a single object or arrays of objects with kzalloc_obj() +and kmalloc_obj() and their array versions kzalloc_objs() and +kmalloc_objs(). These helpers only need the type of the object that should be +allocated and the count of elements in the array for the array versions. + +As of v7.2, vast majority of the memory allocations use kzalloc_obj(). The address of a chunk allocated with `kmalloc` is aligned to at least ARCH_KMALLOC_MINALIGN bytes. For sizes which are a power of two, the @@ -154,9 +163,16 @@ Chunks allocated with kmalloc() can be resized with krealloc(). Similarly to kmalloc_array(): a helper for resizing arrays is provided in the form of krealloc_array(). -For large allocations you can use vmalloc() and vzalloc(), or directly -request pages from the page allocator. The memory allocated by `vmalloc` -and related functions is not physically contiguous. +`kmalloc` always allocates physically contiguous memory and the maximal size of +a chunk that can be allocated with `kmalloc` is limited by `KMALLOC_MAX_SIZE`, +which matches the page allocator's MAX_PAGE_ORDER limit. + +Internally, the slab allocator differentiates allocations of different orders +and delegates larger allocations to the page allocator, but for the users of +`kmalloc` family it is entirely transparent. + +For large allocations that do not require physically contiguous memory you can +use vmalloc() and vzalloc() family. If you are not sure whether the allocation size is too large for `kmalloc`, it is possible to use kvmalloc() and its derivatives. It will From 9c16259ac970597ffbadce84e3a6a8f249562b41 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 2 Sep 2026 11:10:12 +0300 Subject: [PATCH 336/539] MAINTAINERS: add memory related docs in core-mm/ to MM - MISC section Previous efforts to make sure that mm files are properly listed in MAINTAINERS missed Documentation/core-api/mm-api.rst Documentation/core-api/memory-allocation.rst Add them to "MEMORY MANAGEMENT - MISC" Link: https://lore.kernel.org/20260902-docs-memalloc-guide-v2-2-218c1a4dcb80@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: SJ Park Acked-by: Vlastimil Babka (SUSE) Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Michal Hocko Cc: Randy Dunlap Cc: Suren Baghdasaryan --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index a7ff3349a524..1d1b48c1cc42 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17268,6 +17268,8 @@ F: Documentation/ABI/testing/sysfs-kernel-mm F: Documentation/ABI/testing/sysfs-kernel-mm-cma F: Documentation/ABI/testing/sysfs-kernel-mm-numa F: Documentation/admin-guide/mm/ +F: Documentation/core-api/memory-allocation.rst +F: Documentation/core-api/mm-api.rst F: Documentation/mm/ F: mm/char-mem.c F: include/linux/cma.h From 6fc433516140e77f6fd54665bf975b5bc8413c3e Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Mon, 7 Sep 2026 09:36:54 +0300 Subject: [PATCH 337/539] docs/core-api: memory-allocation: clarify when to use kzalloc_obj and kzalloc Make it abundantly clear that in the most cases objects should be allocated with kzalloc_obj() and buffers should be allocated with kzalloc(). Link: https://lore.kernel.org/20260907063654.2248617-1-rppt@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Acked-by: Vlastimil Babka (SUSE) Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: David Hildenbrand (Arm) Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Michal Hocko Cc: Randy Dunlap Cc: SeongJae Park Cc: Suren Baghdasaryan --- Documentation/core-api/memory-allocation.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/core-api/memory-allocation.rst b/Documentation/core-api/memory-allocation.rst index 823f7fa57429..648222b6c857 100644 --- a/Documentation/core-api/memory-allocation.rst +++ b/Documentation/core-api/memory-allocation.rst @@ -23,12 +23,14 @@ answer, although very likely you should use kzalloc_obj(); -or +if you need memory for an object and :: kzalloc(, GFP_KERNEL); +if you need memory for a buffer. + Of course there are cases when other allocation APIs and different GFP flags must be used. From 7809ce47b05671e1fa6d74bead892d6ad6c1403b Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Mon, 7 Sep 2026 10:54:44 +0800 Subject: [PATCH 338/539] mm/page_counter: avoid integer overflow in effective_protection() Patch series "mm/mglru: fix ineffective memory protection for non-kswapd reclaim", v4. For MGLRU, memory.min/low is not honored during non-kswapd global reclaim (global direct reclaim and root-level memory.reclaim), because these paths shrink memcgs using stale protection (emin/elow). Patch 2 is the actual fix. Patch 1 is a prerequisite: an integer overflow in effective_protection(), spotted by the sashiko review tool, which patch 2's new caller would also be exposed to. This patch (of 2): effective_protection() scales a parent's protection by a ratio of page counts, e.g. for recursive protection: (parent_effective - siblings_protected) * (usage - protected) / (parent_usage - siblings_protected) The multiply is done at unsigned long width before dividing. On systems with >= 16TB RAM the product can exceed 2^64 and wrap, giving a bogus protection value and silently breaking memory.min/low enforcement. Use mul_u64_u64_div_u64() to multiply in a 128-bit intermediate. Because usage and parent_usage are not read atomically (a child is charged before its parent), usage - protected can briefly exceed the divisor, making the quotient overflow 64 bits and trap (#DE on x86). Cap it so the ratio stays <= 1. Reported by the sashiko review tool [1]. Link: https://lore.kernel.org/20260907025445.1836238-1-ridong.chen@linux.dev Link: https://lore.kernel.org/20260907025445.1836238-2-ridong.chen@linux.dev Link: https://sashiko.dev/#/patchset/20260826133054.88529-1-ridong.chen@linux.dev?part=1 [1] Fixes: bc50bcc6e00b ("mm: memcontrol: clean up and document effective low/min calculations") Fixes: 8a931f801340 ("mm: memcontrol: recursive memory.low protection") Reviewed-by: Barry Song Reviewed-by: Johannes Weiner Signed-off-by: Ridong Chen Signed-off-by: Andrew Morton Assisted-by: Claude:claude-opus-4-8 Cc: Axel Rasmussen Cc: Chris Down Cc: David Hildenbrand Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Tejun Heo Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao Cc: --- mm/page_counter.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/mm/page_counter.c b/mm/page_counter.c index 450543f4b318..98322803941a 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -376,7 +377,8 @@ static unsigned long effective_protection(unsigned long usage, * otherwise get a smaller chunk than what they claimed. */ if (siblings_protected > parent_effective) - return protected * parent_effective / siblings_protected; + return mul_u64_u64_div_u64(protected, parent_effective, + siblings_protected); /* * Ok, utilized protection of all children is within what the @@ -417,13 +419,20 @@ static unsigned long effective_protection(unsigned long usage, if (parent_effective > siblings_protected && parent_usage > siblings_protected && usage > protected) { - unsigned long unclaimed; + unsigned long parent_unclaimed, parent_unprotected, unprotected; - unclaimed = parent_effective - siblings_protected; - unclaimed *= usage - protected; - unclaimed /= parent_usage - siblings_protected; + parent_unclaimed = parent_effective - siblings_protected; + parent_unprotected = parent_usage - siblings_protected; - ep += unclaimed; + /* + * The usages aren't read atomically, so a child can transiently + * appear to use more than its parent, making the ratio exceed 1 + * and the quotient overflow 64 bits (#DE on x86). Cap it. + */ + unprotected = min(usage - protected, parent_unprotected); + + ep += mul_u64_u64_div_u64(parent_unclaimed, unprotected, + parent_unprotected); } return ep; From 15da1e4978dd4ea8f5faa6e15ee967ba4ad9a30a Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Mon, 7 Sep 2026 10:54:45 +0800 Subject: [PATCH 339/539] mm/mglru: fix ineffective memory protection for non-kswapd reclaim For MGLRU, memory.min/low is not honored during global proactive reclaim (writing to the root memory.reclaim) and global direct reclaim, because these paths shrink memcgs using stale protection (emin/elow). It can be reproduced as follows: # echo 7 > /sys/kernel/mm/lru_gen/enabled # cd /sys/fs/cgroup # mkdir -p a/b # echo 100M > a/memory.min # echo +memory > a/cgroup.subtree_control # echo 100M > a/b/memory.min # echo $$ > a/b/cgroup.procs # dd if=/dev/zero of=/tmp/testfile bs=1M count=200 # cat a/b/memory.current 222650368 # echo 500M > memory.reclaim -bash: echo: write error: Resource temporarily unavailable # cat a/b/memory.current 6070272 memory.min is 100M, yet reclaim drops a/b down to 6M, breaking the protection. The traditional LRU path is not affected because shrink_node() calls mem_cgroup_calculate_protection() for each memcg it visits during a top-down tree walk. Commit 30d77b7eef01 ("mm/mglru: fix ineffective protection calculation") moved the protection computation into lru_gen_age_node(), which only runs for kswapd. Non-kswapd global reclaim reaches shrink_one() through lru_gen_shrink_node() -> shrink_many() without any protection computation, so emin/elow are whatever a previous kswapd run left behind - or zero if kswapd never ran on this node. Relying on a prior kswapd pass is not correct either: a memcg's emin/elow are derived from its ancestors' memory.min/low settings and from children_min_usage, both of which change over time, so emin/elow go stale even after kswapd has run and must be recomputed at the point of reclaim. Introduce mem_cgroup_calculate_protection_path() which computes emin/elow along the root-to-target path only, by iterating through the cgroup ancestors array top-down. This avoids the full tree traversal that would be needed with mem_cgroup_calculate_protection(), limiting the cost to O(depth) per memcg - typically 3-5 levels. Call it from shrink_one() for the non-kswapd path so that each memcg about to be shrunk has correct protection values. Link: https://lore.kernel.org/20260907025445.1836238-3-ridong.chen@linux.dev Fixes: e4dde56cd208 ("mm: multi-gen LRU: per-node lru_gen_folio lists") Signed-off-by: Ridong Chen Signed-off-by: Andrew Morton Reviewed-by: Barry Song Reviewed-by: Johannes Weiner Assisted-by: Claude:claude-opus-4-8 Cc: Axel Rasmussen Cc: Chris Down Cc: David Hildenbrand Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Tejun Heo Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao Cc: --- include/linux/memcontrol.h | 10 +++++++++ mm/memcontrol.c | 45 ++++++++++++++++++++++++++++++++++++++ mm/vmscan.c | 8 ++++++- 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 4f720791a31c..46bf724cae7a 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1921,6 +1921,16 @@ static inline bool memcg_is_dying(struct mem_cgroup *memcg) } #endif /* CONFIG_MEMCG */ +#if defined(CONFIG_MEMCG) && defined(CONFIG_LRU_GEN) +void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, + struct mem_cgroup *memcg); +#else +static inline void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, + struct mem_cgroup *memcg) +{ +} +#endif + #if defined(CONFIG_MEMCG) && defined(CONFIG_ZSWAP) bool obj_cgroup_may_zswap(struct obj_cgroup *objcg); void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index cf53d4ac7ecc..791e536efaeb 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5267,6 +5267,51 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root, page_counter_calculate_protection(&root->memory, &memcg->memory, recursive_protection); } +#ifdef CONFIG_LRU_GEN +/** + * mem_cgroup_calculate_protection_path - compute protection along a path + * @root: the top ancestor of the sub-tree being checked (NULL for root_mem_cgroup) + * @memcg: the target memory cgroup + * + * Walk the ancestor path from @root down to @memcg and compute the effective + * protection at each level. This is safe for isolated queries because it + * ensures parents are computed before children. + */ +void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, + struct mem_cgroup *memcg) +{ + bool recursive_protection = + cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT; + struct cgroup *cg; + int root_level, i; + + if (mem_cgroup_disabled()) + return; + + if (!root) + root = root_mem_cgroup; + + if (memcg == root) + return; + + root_level = root->css.cgroup->level; + cg = memcg->css.cgroup; + + rcu_read_lock(); + for (i = root_level + 1; i <= cg->level; i++) { + struct mem_cgroup *cur; + + cur = mem_cgroup_from_css(cgroup_css(cg->ancestors[i], + &memory_cgrp_subsys)); + if (cur) + page_counter_calculate_protection(&root->memory, + &cur->memory, + recursive_protection); + } + rcu_read_unlock(); +} +#endif /* CONFIG_LRU_GEN */ + static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, gfp_t gfp) { diff --git a/mm/vmscan.c b/mm/vmscan.c index 8e9c73dcd19b..80041e2b8049 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -5251,7 +5251,13 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); - /* lru_gen_age_node() called mem_cgroup_calculate_protection() */ + /* + * For kswapd, mem_cgroup_calculate_protection() has already + * been called during the top-down cgroup traversal. + */ + if (!current_is_kswapd()) + mem_cgroup_calculate_protection_path(NULL, memcg); + if (mem_cgroup_below_min(NULL, memcg)) return MEMCG_LRU_YOUNG; From c32fddc0a01da9627789cd5cb3add00b75081da6 Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Tue, 8 Sep 2026 17:55:15 +0800 Subject: [PATCH 340/539] tools/testing/vma: cover hole filling through __mmap_region() The mmap tests extend existing mappings one neighbor at a time, while merge tests construct merge state directly. Neither exercises filling a hole between compatible mappings through the mmap setup and completion path. Fill a gap through __mmap_region() and require both neighbors to merge into one VMA. Repeat with only the new mapping's execute permission set and require three separate VMAs. Check boundaries, permissions, page offsets, map_count and cleanup. Link: https://lore.kernel.org/178886112560.138404.17741948638665342936.vma-v2@tychen.cc Signed-off-by: Tianyi Chen Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Assisted-by: Codex:GPT-6 Cc: Jann Horn Cc: Liam R. Howlett Cc: Pedro Falcato Cc: Vlastimil Babka --- tools/testing/vma/tests/mmap.c | 68 ++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index fa73faff2262..53e4abe6a63e 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -45,6 +45,72 @@ static bool test_mmap_region_basic(void) return true; } +static bool mmap_region_fill_hole(bool merge) +{ + const vma_flags_t vma_flags = mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, VMA_MAYEXEC_BIT); + vma_flags_t middle_flags = vma_flags; + struct mm_struct mm = {}; + struct vm_area_struct *vma; + unsigned long addr; + int count = 0; + VMA_ITERATOR(vmi, &mm, 0); + + current->mm = &mm; + if (!merge) + vma_flags_set(&middle_flags, VMA_EXEC_BIT); + + /* Map at 0x300000, length 0x3000. */ + addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, 0x300, NULL); + ASSERT_EQ(addr, 0x300000); + + /* Map at 0x306000, length 0x3000, leaving a hole. */ + addr = __mmap_region(NULL, 0x306000, 0x3000, vma_flags, 0x306, NULL); + ASSERT_EQ(addr, 0x306000); + ASSERT_EQ(mm.map_count, 2); + + /* Map at 0x303000, length 0x3000, filling the hole. */ + addr = __mmap_region(NULL, 0x303000, 0x3000, middle_flags, 0x303, NULL); + ASSERT_EQ(addr, 0x303000); + ASSERT_EQ(mm.map_count, merge ? 1 : 3); + + vma_iter_set(&vmi, 0); + for_each_vma(vmi, vma) { + const unsigned long start = 0x300000 + count * 0x3000; + const unsigned long end = merge ? 0x309000 : start + 0x3000; + /* Only the middle VMA in the non-merge case has VMA_EXEC. */ + const bool is_middle_vma = count == 1; + const bool expect_exec_vma = is_middle_vma && !merge; + + ASSERT_EQ(vma->vm_start, start); + ASSERT_EQ(vma->vm_end, end); + ASSERT_EQ(vma_start_pgoff(vma), start >> PAGE_SHIFT); + ASSERT_EQ(vma_start_anon_pgoff(vma), start >> PAGE_SHIFT); + + ASSERT_TRUE(vma_test_all(vma, VMA_READ_BIT, VMA_WRITE_BIT, + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, + VMA_MAYEXEC_BIT)); + ASSERT_EQ(vma_test(vma, VMA_EXEC_BIT), expect_exec_vma); + + count++; + } + + ASSERT_EQ(count, mm.map_count); + + ASSERT_EQ(cleanup_mm(&mm, &vmi), count); + return true; +} + +static bool test_mmap_region_fill_hole_merge(void) +{ + return mmap_region_fill_hole(true); +} + +static bool test_mmap_region_fill_hole_flags_mismatch(void) +{ + return mmap_region_fill_hole(false); +} + static bool test_pure_anon_dev_zero(void) { const vma_flags_t vma_flags = mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, @@ -84,5 +150,7 @@ static bool test_pure_anon_dev_zero(void) static void run_mmap_tests(int *num_tests, int *num_fail) { TEST(mmap_region_basic); + TEST(mmap_region_fill_hole_merge); + TEST(mmap_region_fill_hole_flags_mismatch); TEST(pure_anon_dev_zero); } From 91b4edb993e1549afdc3c2495216734b9f13f00e Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Sun, 6 Sep 2026 21:59:38 +0800 Subject: [PATCH 341/539] mm/zswap: enable zswap_ever_enabled in zswap_pool_create() If zswap is enabled by default at boot and pool creation fails, then a pool is later created by updating the compressor, data written to zswap is corrupted on swapin. Enable the static key in zswap_pool_create(), covering boot-time and runtime pool creation with a single site. Verified with fault injection on a stock kernel (compressor builtin, CONFIG_ZSWAP_DEFAULT_ON=n): 1. Boot with zswap.enabled=1; pool creation fails, init completes pool-less (static key off). 2. Echo an available compressor name to zswap.compressor. 3. Enable zswap. 4. madvise(MADV_PAGEOUT) a pattern-verified 512 MiB region, then fault it back in and verify. Step 4 reads back 131072/131072 zeroed pages (zswpin=0, zswpout=131072) without this patch; all pages intact (zswpin=131072) with it. Link: https://lore.kernel.org/20260906135938.3568108-1-xialonglong2025@163.com Fixes: 2d4d2b1cfb85 ("mm: zswap: add zswap_never_enabled()") Signed-off-by: Longlong Xia Signed-off-by: Andrew Morton Suggested-by: Yosry Ahmed Acked-by: Yosry Ahmed Assisted-by: Zcode:GLM-5.3 Cc: Chengming Zhou Cc: Johannes Weiner Cc: Nhat Pham Cc: --- mm/zswap.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/zswap.c b/mm/zswap.c index d0b6c229b616..cfaedc85dfed 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -329,6 +329,8 @@ static struct zswap_pool *zswap_pool_create(char *compressor) zswap_pool_debug("created", pool); + static_branch_enable(&zswap_ever_enabled); + return pool; ref_fail: @@ -1818,7 +1820,6 @@ static int zswap_setup(void) pr_info("loaded using pool %s\n", pool->tfm_name); list_add_rcu(&pool->list, &zswap_pools); zswap_has_pool = true; - static_branch_enable(&zswap_ever_enabled); } else { pr_err("pool creation failed\n"); zswap_enabled = false; From b2fb109cb027425b76609a55501ec28f4a791ffc Mon Sep 17 00:00:00 2001 From: Jianyue Wu Date: Sun, 6 Sep 2026 15:47:17 +0800 Subject: [PATCH 342/539] mm/zswap: release retired pools via queue_rcu_work() instead of synchronize_rcu() Patch series "mm/zswap: shrink zswap_entry via a pool id", v6. Every stored page has a struct zswap_entry, so its size is pure per-page overhead. On 64-bit it is currently 56 bytes, of which 8 bytes are a pointer to the owning zswap_pool. Only a handful of pools are ever live: a new pool is created only when the compressor is (re)set, and pools are reused across compressor switches. A list cannot look a pool up by id. An allocating xarray can, which lets each zswap_entry store a u8 instead of a pointer. This series: 1. Releases retired pools with queue_rcu_work() instead of a worker calling synchronize_rcu(), so the release worker no longer blocks on an RCU grace period. 2. Replaces the zswap_pools list with an allocating xarray (XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH) and a separate RCU-protected current-pool pointer, giving each pool a stable small id. Ids start at 1. The reserved id 0 is never allocated, so looking it up resolves to NULL. The table grows as needed up to 255 live pools (u8 pool_idx), not a fixed slot array. 3. Stores that u8 pool id in each zswap_entry instead of the pool pointer. The u8 fits in padding after the bool referenced field. On 64-bit that shrinks the entry from 56 to 48 bytes, which fits 73 to 85 objects in a 4K slab (~2MiB of metadata saved per 1GiB of data held in zswap). The 255-id cap counts every pool still in the xarray. Switching compressor kills the old pool, but that pool stays in the table until its last entry drops the pool's ref, so a draining pool still occupies an id. The id is reused only after xa_erase. Switching back to a compressor whose pool is still in the table resurrects it instead of allocating a new id. If every id is occupied, creating a pool for another compressor fails and the switch is rejected. Pool table locking: xa_for_each() walks and the current-pool pointer use an explicit rcu_read_lock(), because xa_for_each()'s own RCU does not span the loop body. xa_load() takes RCU around the lookup itself, so zswap_entry_pool() needs no extra rcu_read_lock(). The returned pool stays valid because a live entry pins it via percpu_ref, so the id cannot be reused under it. xa_lock is taken only in xa_alloc_bh() and xa_erase_bh(). Benchmark (x86_64, compressor=lzo, MADV_PAGEOUT store + fault-in load): - e2e store+load median latency: no measurable regression vs baseline at matched stored_delta Each store, free, and decompress looks up the pool with xa_load() instead of following a pointer. With only a handful of live pools the xarray walk is short. This patch (of 3): When a pool's last reference is dropped, __zswap_pool_empty() removes it from the pool list and schedules __zswap_pool_release(), which calls synchronize_rcu() to wait for readers before tearing the pool down. synchronize_rcu() is a synchronous, potentially long wait. Replace it with queue_rcu_work(): __zswap_pool_empty() hands the pool to queue_rcu_work(), which waits for a grace period asynchronously and then runs __zswap_pool_release() from a worker for the sleepable teardown (__zswap_pool_empty() can run in atomic context and must not block). The grace-period guarantee is unchanged; the retirement path just no longer blocks on it. Link: https://lore.kernel.org/20260906-shrink_zswap_entry_v6-v6-0-ac4cf61565fb@gmail.com Link: https://lore.kernel.org/20260906-shrink_zswap_entry_v6-v6-1-ac4cf61565fb@gmail.com Signed-off-by: Jianyue Wu Signed-off-by: Andrew Morton Suggested-by: Yosry Ahmed Acked-by: Yosry Ahmed Cc: Chengming Zhou Cc: Chris Li Cc: Johannes Weiner Cc: Nhat Pham --- mm/zswap.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index cfaedc85dfed..526327266e76 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -155,7 +155,7 @@ struct zswap_pool { struct crypto_acomp_ctx __percpu *acomp_ctx; struct percpu_ref ref; struct list_head list; - struct work_struct release_work; + struct rcu_work release_rwork; struct hlist_node node; char tfm_name[CRYPTO_MAX_ALG_NAME]; }; @@ -386,10 +386,8 @@ static void zswap_pool_destroy(struct zswap_pool *pool) static void __zswap_pool_release(struct work_struct *work) { - struct zswap_pool *pool = container_of(work, typeof(*pool), - release_work); - - synchronize_rcu(); + struct zswap_pool *pool = container_of(to_rcu_work(work), + typeof(*pool), release_rwork); /* nobody should have been able to get a ref... */ WARN_ON(!percpu_ref_is_zero(&pool->ref)); @@ -413,8 +411,8 @@ static void __zswap_pool_empty(struct percpu_ref *ref) list_del_rcu(&pool->list); - INIT_WORK(&pool->release_work, __zswap_pool_release); - schedule_work(&pool->release_work); + INIT_RCU_WORK(&pool->release_rwork, __zswap_pool_release); + queue_rcu_work(system_percpu_wq, &pool->release_rwork); spin_unlock_bh(&zswap_pools_lock); } From 6ff57b96ab6b471011a6dee5878e4741b70a156f Mon Sep 17 00:00:00 2001 From: Jianyue Wu Date: Sun, 6 Sep 2026 15:47:18 +0800 Subject: [PATCH 343/539] mm/zswap: replace the zswap_pools list with an allocating xarray Originally zswap kept its pools on an RCU list whose head also served as the current pool. Convert the pool table to an allocating xarray keyed by a small integer id, and track the current pool with a separate RCU-protected pointer. The xarray gives each pool a stable id for a later zswap_entry shrink. XA_FLAGS_ALLOC1 starts ids at 1, so id 0 remains reserved. The id range is bounded by ZSWAP_MAX_POOL_ID because the later entry field is a u8. Keep compressor switching close to the previous flow: look up an existing pool with xa_for_each(), resurrect it if reused, or create a new one. zswap_pool_create() allocates the pool's id and publishes it into the xarray as its final step, so the create call either fully publishes or fully unwinds on failure. Publishing makes the pool live, so a caller that later fails (e.g. param_set_charp()) must still kill the pool to erase it from the xarray. Compressor switches update zswap_current_pool with rcu_assign_pointer(), serialized by the module parameter lock, so no xa_lock is needed for that update. The pool walk above is lockless under RCU. xa_lock is taken only to allocate (xa_alloc_bh()) and erase (xa_erase_bh()) xarray entries. Link: https://lore.kernel.org/20260906-shrink_zswap_entry_v6-v6-2-ac4cf61565fb@gmail.com Signed-off-by: Jianyue Wu Signed-off-by: Andrew Morton Suggested-by: Nhat Pham Suggested-by: Yosry Ahmed Suggested-by: Johannes Weiner Acked-by: Yosry Ahmed Cc: Chengming Zhou Cc: Chris Li --- mm/zswap.c | 124 +++++++++++++++++++++++++++++------------------------ 1 file changed, 67 insertions(+), 57 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index 526327266e76..8b6b1dce7948 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -154,12 +155,23 @@ struct zswap_pool { struct zs_pool *zs_pool; struct crypto_acomp_ctx __percpu *acomp_ctx; struct percpu_ref ref; - struct list_head list; struct rcu_work release_rwork; struct hlist_node node; + u8 idx; char tfm_name[CRYPTO_MAX_ALG_NAME]; }; +/* + * Live pools keyed by id (1..ZSWAP_MAX_POOL_ID). XA_FLAGS_ALLOC1 keeps id 0 + * reserved so it is never handed to a live pool. XA_FLAGS_LOCK_BH makes the + * xa_lock softirq-safe: it is taken from __zswap_pool_empty(), which runs from + * a percpu_ref release callback in softirq context. + */ +#define ZSWAP_FIRST_POOL_ID 1 +#define ZSWAP_MAX_POOL_ID U8_MAX +static DEFINE_XARRAY_FLAGS(zswap_pools, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH); +static struct zswap_pool __rcu *zswap_current_pool; + /* Global LRU lists shared by all zswap pools. */ static struct list_lru zswap_list_lru; @@ -200,10 +212,6 @@ struct zswap_entry { static struct xarray *zswap_trees[MAX_SWAPFILES]; static unsigned int nr_zswap_trees[MAX_SWAPFILES]; -/* RCU-protected iteration */ -static LIST_HEAD(zswap_pools); -/* protects zswap_pools list modification */ -static DEFINE_SPINLOCK(zswap_pools_lock); /* pool counter to provide unique names to zsmalloc */ static atomic_t zswap_pools_count = ATOMIC_INIT(0); @@ -280,6 +288,7 @@ static struct zswap_pool *zswap_pool_create(char *compressor) struct zswap_pool *pool; char name[38]; /* 'zswap' + 32 char (max) num + \0 */ int ret, cpu; + u32 id; if (!zswap_has_pool && !strcmp(compressor, ZSWAP_PARAM_UNSET)) return NULL; @@ -325,7 +334,22 @@ static struct zswap_pool *zswap_pool_create(char *compressor) PERCPU_REF_ALLOW_REINIT, GFP_KERNEL); if (ret) goto ref_fail; - INIT_LIST_HEAD(&pool->list); + + /* + * Publish only after the pool is fully built, so lockless walkers + * never see a half-initialized pool. The _bh variant pairs with the + * softirq-context xa_lock taken in __zswap_pool_empty(). + */ + ret = xa_alloc_bh(&zswap_pools, &id, pool, + XA_LIMIT(ZSWAP_FIRST_POOL_ID, ZSWAP_MAX_POOL_ID), + GFP_KERNEL); + if (ret) { + if (ret == -EBUSY) + pr_err("cannot allocate pool id (max %d live pools)\n", + ZSWAP_MAX_POOL_ID - ZSWAP_FIRST_POOL_ID + 1); + goto xa_fail; + } + pool->idx = id; zswap_pool_debug("created", pool); @@ -333,6 +357,8 @@ static struct zswap_pool *zswap_pool_create(char *compressor) return pool; +xa_fail: + percpu_ref_exit(&pool->ref); ref_fail: cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); @@ -393,28 +419,22 @@ static void __zswap_pool_release(struct work_struct *work) WARN_ON(!percpu_ref_is_zero(&pool->ref)); percpu_ref_exit(&pool->ref); - /* pool is now off zswap_pools list and has no references. */ + /* The pool is no longer in zswap_pools and has no references. */ zswap_pool_destroy(pool); } -static struct zswap_pool *zswap_pool_current(void); - static void __zswap_pool_empty(struct percpu_ref *ref) { struct zswap_pool *pool; pool = container_of(ref, typeof(*pool), ref); - spin_lock_bh(&zswap_pools_lock); + WARN_ON(pool == rcu_access_pointer(zswap_current_pool)); - WARN_ON(pool == zswap_pool_current()); - - list_del_rcu(&pool->list); + xa_erase_bh(&zswap_pools, pool->idx); INIT_RCU_WORK(&pool->release_rwork, __zswap_pool_release); queue_rcu_work(system_percpu_wq, &pool->release_rwork); - - spin_unlock_bh(&zswap_pools_lock); } static int __must_check zswap_pool_tryget(struct zswap_pool *pool) @@ -440,20 +460,13 @@ static struct zswap_pool *__zswap_pool_current(void) { struct zswap_pool *pool; - pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); + pool = rcu_dereference(zswap_current_pool); WARN_ONCE(!pool && zswap_has_pool, "%s: no page storage pool!\n", __func__); return pool; } -static struct zswap_pool *zswap_pool_current(void) -{ - assert_spin_locked(&zswap_pools_lock); - - return __zswap_pool_current(); -} - static struct zswap_pool *zswap_pool_current_get(void) { struct zswap_pool *pool; @@ -469,23 +482,28 @@ static struct zswap_pool *zswap_pool_current_get(void) return pool; } -/* type and compressor must be null-terminated */ +/* compressor must be null-terminated */ static struct zswap_pool *zswap_pool_find_get(char *compressor) { struct zswap_pool *pool; + unsigned long id; - assert_spin_locked(&zswap_pools_lock); - - list_for_each_entry_rcu(pool, &zswap_pools, list) { + /* + * __zswap_pool_empty() can erase from zswap_pools in softirq while we + * walk. rcu_read_lock() keeps the walk consistent and each pool alive + * across tryget(). xa_for_each()'s own RCU does not span the loop body. + */ + rcu_read_lock(); + xa_for_each(&zswap_pools, id, pool) { if (strcmp(pool->tfm_name, compressor)) continue; /* if we can't get it, it's about to be destroyed */ - if (!zswap_pool_tryget(pool)) - continue; - return pool; + if (zswap_pool_tryget(pool)) + break; } + rcu_read_unlock(); - return NULL; + return pool; } static unsigned long zswap_max_pages(void) @@ -502,9 +520,14 @@ unsigned long zswap_total_pages(void) { struct zswap_pool *pool; unsigned long total = 0; + unsigned long id; + /* + * rcu_read_lock() keeps each pool alive across zs_get_total_pages(). + * xa_for_each()'s own RCU does not span the loop body. + */ rcu_read_lock(); - list_for_each_entry_rcu(pool, &zswap_pools, list) + xa_for_each(&zswap_pools, id, pool) total += zs_get_total_pages(pool->zs_pool); rcu_read_unlock(); @@ -561,20 +584,13 @@ static int zswap_compressor_param_set(const char *val, const struct kernel_param return -ENOENT; } - spin_lock_bh(&zswap_pools_lock); - pool = zswap_pool_find_get(s); - if (pool) { - zswap_pool_debug("using existing", pool); - WARN_ON(pool == zswap_pool_current()); - list_del_rcu(&pool->list); - } - - spin_unlock_bh(&zswap_pools_lock); - - if (!pool) + if (!pool) { pool = zswap_pool_create(s); - else { + } else { + zswap_pool_debug("using existing", pool); + WARN_ON(pool == rcu_access_pointer(zswap_current_pool)); + /* * Restore the initial ref dropped by percpu_ref_kill() * when the pool was decommissioned and switch it again @@ -591,24 +607,18 @@ static int zswap_compressor_param_set(const char *val, const struct kernel_param else ret = -EINVAL; - spin_lock_bh(&zswap_pools_lock); - + /* + * Compressor switches are serialized by the kernel param lock, so this + * is the only writer of zswap_current_pool: no xa_lock needed. + */ if (!ret) { - put_pool = zswap_pool_current(); - list_add_rcu(&pool->list, &zswap_pools); + put_pool = rcu_access_pointer(zswap_current_pool); + rcu_assign_pointer(zswap_current_pool, pool); zswap_has_pool = true; } else if (pool) { - /* - * Add the possibly pre-existing pool to the end of the pools - * list; if it's new (and empty) then it'll be removed and - * destroyed by the put after we drop the lock - */ - list_add_tail_rcu(&pool->list, &zswap_pools); put_pool = pool; } - spin_unlock_bh(&zswap_pools_lock); - /* * Drop the ref from either the old current pool, * or the new pool we failed to add @@ -1816,7 +1826,7 @@ static int zswap_setup(void) pool = __zswap_pool_create_fallback(); if (pool) { pr_info("loaded using pool %s\n", pool->tfm_name); - list_add_rcu(&pool->list, &zswap_pools); + rcu_assign_pointer(zswap_current_pool, pool); zswap_has_pool = true; } else { pr_err("pool creation failed\n"); From 82e64b8a1bf035a9ce57217ac5c7b64af8c1794f Mon Sep 17 00:00:00 2001 From: Jianyue Wu Date: Sun, 6 Sep 2026 15:47:19 +0800 Subject: [PATCH 344/539] mm/zswap: reference the pool by id to shrink struct zswap_entry struct zswap_entry is one allocation per stored page, so its size is pure overhead. It currently embeds an 8-byte pool pointer, even though the live pools now sit in an allocating xarray keyed by a small integer id that fits in a u8. Replace the per-entry pool pointer with that u8 id and resolve it through the xarray with xa_load(). xa_load() does its own RCU-protected lookup, so the caller needs no rcu_read_lock() section of its own. The resolved pool stays valid because a live entry pins it via percpu_ref (taken in zswap_store_page()), so its id cannot be reused. A live entry never uses the reserved id 0, so a zeroed id resolves to NULL and trips a WARN rather than aliasing a live pool. The u8 fits in the padding after the bool referenced field, shrinking the entry from 56 to 48 bytes on 64-bit. This raises objs_per_slab from 73 to 85 and saves about 2MiB of metadata per 1GiB of data held in zswap. Link: https://lore.kernel.org/20260906-shrink_zswap_entry_v6-v6-3-ac4cf61565fb@gmail.com Signed-off-by: Jianyue Wu Signed-off-by: Andrew Morton Suggested-by: Chris Li Acked-by: Yosry Ahmed Cc: Chengming Zhou Cc: Johannes Weiner Cc: Nhat Pham --- mm/zswap.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index 8b6b1dce7948..ff7c6742af50 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -194,7 +194,7 @@ static struct shrinker *zswap_shrinker; * writeback logic. The entry is only reclaimed by the writeback * logic if referenced is unset. See comments in the shrinker * section for context. - * pool - the zswap_pool the entry's data is in + * pool_idx - id of the zswap_pool that the entry's data is in. * handle - zsmalloc allocation handle that stores the compressed page data * objcg - the obj_cgroup that the compressed memory is charged to * lru - handle to the pool's lru used to evict pages. @@ -203,12 +203,22 @@ struct zswap_entry { swp_entry_t swpentry; unsigned int length; bool referenced; - struct zswap_pool *pool; + u8 pool_idx; unsigned long handle; struct obj_cgroup *objcg; struct list_head lru; }; +/* + * No RCU section is needed around the returned pointer: a stored entry pins + * its pool via percpu_ref (taken in zswap_store_page()), so the id cannot be + * reused under us. Callers WARN and handle a NULL from a corrupt pool_idx. + */ +static struct zswap_pool *zswap_entry_pool(struct zswap_entry *entry) +{ + return xa_load(&zswap_pools, entry->pool_idx); +} + static struct xarray *zswap_trees[MAX_SWAPFILES]; static unsigned int nr_zswap_trees[MAX_SWAPFILES]; @@ -766,9 +776,13 @@ static void zswap_entry_cache_free(struct zswap_entry *entry) */ static void zswap_entry_free(struct zswap_entry *entry) { + struct zswap_pool *pool = zswap_entry_pool(entry); + zswap_lru_del(entry); - zs_free(entry->pool->zs_pool, entry->handle); - zswap_pool_put(entry->pool); + if (!WARN_ON_ONCE(!pool)) { + zs_free(pool->zs_pool, entry->handle); + zswap_pool_put(pool); + } if (entry->objcg) { obj_cgroup_uncharge_zswap(entry->objcg, entry->length); obj_cgroup_put(entry->objcg); @@ -924,12 +938,15 @@ unlock: static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio) { - struct zswap_pool *pool = entry->pool; + struct zswap_pool *pool = zswap_entry_pool(entry); struct scatterlist input[2]; /* zsmalloc returns an SG list 1-2 entries */ struct scatterlist output; struct crypto_acomp_ctx *acomp_ctx; int ret = 0, dlen; + if (WARN_ON_ONCE(!pool)) + return false; + acomp_ctx = raw_cpu_ptr(pool->acomp_ctx); mutex_lock(&acomp_ctx->mutex); zs_obj_read_sg_begin(pool->zs_pool, entry->handle, input, entry->length); @@ -965,7 +982,7 @@ static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio) pr_alert_ratelimited("Decompression error from zswap (%d:%lu %s %u->%d)\n", swp_type(entry->swpentry), swp_offset(entry->swpentry), - entry->pool->tfm_name, + pool->tfm_name, entry->length, dlen); return false; } @@ -1423,6 +1440,13 @@ static bool zswap_store_page(struct folio *folio, long index, if (!zswap_compress(folio, index, entry, pool)) goto compress_failed; + /* + * Set pool_idx before the xa_store() below publishes the entry, or a + * concurrent reader could resolve a stale pool_idx left by slab reuse + * to an unrelated live pool. + */ + entry->pool_idx = pool->idx; + old = xa_store(swap_zswap_tree(page_swpentry), swp_offset(page_swpentry), entry, GFP_KERNEL); @@ -1468,7 +1492,6 @@ static bool zswap_store_page(struct folio *folio, long index, * The publishing order matters to prevent writeback from seeing * an incoherent entry. */ - entry->pool = pool; entry->swpentry = page_swpentry; entry->objcg = objcg; entry->referenced = true; From f8facff65cb3f17e1c5fd01b81d578bca25d8a01 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 10 Sep 2026 07:22:28 -0700 Subject: [PATCH 345/539] mm/damon/api: introduce DAMON_FILTER_TYPE_PGIDLE_SET Patch series "mm/damon: introduce pgidle_set probe filter type". Users can do effective access monitoring with DAMON, using set_pgidle probe prep operation and pgidle_unset allowing probe filter. In the setup, high probe_hits means the region was accessed frequently. Zero probe_hits means the region was not accessed. The filter, however, matches only the memory that can show the idleness. Regions of zero probe_hits may contain memory that was just unable to show the idleness. On virtual address space based monitoring, for example, non-present pages are reported as cold. It is truly cold, but if the purpose is to make some operations like reclaiming or compressing the data, this is not really useful. Introduce a new probe filter type, pgidle_set, that confirms the page is marked as idle. Using it instead of pgidle_unset, users can find cold pages that can effectively be processed. Patch 1 introduces the pgidle_set probe filter type to DAMON kernel API. Patches 2 and 3 add support for it on physical and virtual address space operation sets, respectively. Patch 4 adds support for it on the DAMON sysfs interface. Finally, patch 5 updates the documentation for the new filter type. This patch (of 5): DAMON_FILTER_TYPE_PGIDLE_UNSET matches only memory that was able to confirm if it is marked as idle. If the memory cannot be marked as idle, it simply doesn't match. Hence, finding memory that was marked as idle with only DAMON_FILTER_TYPE_PGIDLE_UNSET is impossible. Introduce a new filter for the purpose, DAMON_FILTER_TYPE_PGIDLE_SET. Link: https://lore.kernel.org/20260910142234.171562-1-sj@kernel.org Link: https://lore.kernel.org/20260910142234.171562-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 871d26adf6ae..16800b3dc379 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -774,11 +774,13 @@ struct damon_prep { * @DAMON_FILTER_TYPE_ANON: Anonymous pages. * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. * @DAMON_FILTER_TYPE_PGIDLE_UNSET: Pgidle is unset. + * @DAMON_FILTER_TYPE_PGIDLE_SET: Pgidle is set. */ enum damon_filter_type { DAMON_FILTER_TYPE_ANON, DAMON_FILTER_TYPE_MEMCG, DAMON_FILTER_TYPE_PGIDLE_UNSET, + DAMON_FILTER_TYPE_PGIDLE_SET, }; /** From 4ba6da6cbb7aa34eeddb9f6daa4783c01e329116 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 10 Sep 2026 07:22:29 -0700 Subject: [PATCH 346/539] mm/damon/paddr: support DAMON_FILTER_TYPE_PGIDLE_SET Implement DAMON_FILTER_TYPE_PGIDLE_SET support on the physical address space DAMON operation set (paddr). Link: https://lore.kernel.org/20260910142234.171562-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/paddr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index d7c81829445b..79195026b903 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -151,6 +151,12 @@ static bool damon_pa_filter_match(struct damon_filter *filter, else matched = damon_folio_young(folio); break; + case DAMON_FILTER_TYPE_PGIDLE_SET: + if (!folio) + matched = false; + else + matched = damon_folio_young(folio) == false; + break; default: return damon_ops_filter_match(filter, folio); } From d9acadc24722fa1ec3701547fadfd59f827d4044 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 10 Sep 2026 07:22:30 -0700 Subject: [PATCH 347/539] mm/damon/vaddr: support DAMON_FILTER_TYPE_PGIDLE_SET Implement DAMON_FILTER_TYPE_PGIDLE_SET support on the virtual address space DAMON operation set (vaddr). Link: https://lore.kernel.org/20260910142234.171562-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/vaddr.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 9a38dc89a156..7063356370c3 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -550,6 +550,13 @@ static bool damon_va_filter_match(struct damon_filter *filter, matched = damon_va_young_addr(folio, pte, pmd, mm, addr); break; + case DAMON_FILTER_TYPE_PGIDLE_SET: + if (!folio) + matched = false; + else + matched = !damon_va_young_addr(folio, pte, pmd, mm, + addr); + break; default: return damon_ops_filter_match(filter, folio); } From b4f6c7746fd2e3cc171afa184e75005ec00ece35 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 10 Sep 2026 07:22:31 -0700 Subject: [PATCH 348/539] mm/damon/sysfs: support DAMON_FILTER_TYPE_PGIDLE_SET Extend DAMON sysfs interface to let users setup DAMON_FILTER_TYPE_PGIDLE_SET probe filter. Link: https://lore.kernel.org/20260910142234.171562-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index b576e97cbfdb..51fa506c879b 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1001,6 +1001,10 @@ damon_sysfs_filter_type_names[] = { .type = DAMON_FILTER_TYPE_PGIDLE_UNSET, .name = "pgidle_unset", }, + { + .type = DAMON_FILTER_TYPE_PGIDLE_SET, + .name = "pgidle_set", + }, }; static ssize_t type_show(struct kobject *kobj, From 4ae7aac0b29d7876bfe5590f185b5d81c6056ca1 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 10 Sep 2026 07:22:32 -0700 Subject: [PATCH 349/539] Docs/mm/damon/design: update for pgidle_set probe filter Update DAMON design document for the newly added pgidle_set probe filter type. Link: https://lore.kernel.org/20260910142234.171562-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index aac84de261aa..22b785cd11df 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -299,6 +299,7 @@ filter types. Currently below filter types are supported. - ``memcg``: Same to that for DAMOS filters. - ``pgidle_unset``: Matches if the page for the memory is marked as not access-idle. +- ``pgidle_set``: Matches if the page for the memory is marked as access-idle. If such probes are registered, DAMON executes the probes for each region's sampling memory when it does the access :ref:`sampling @@ -314,7 +315,7 @@ actions are registered, DAMON applies the actions to each region's sampling memory before starting the next sampling interval. Currently only one action, ``set_pgidle`` is supported. The action marks the page for the probing target memory as access-idle. This can be useful to be used together with -``pgidle_unset`` probe filter. +``pgidle_unset`` or ``pgidle_set`` probe filter. This is a sampling based mechanism. Hence, it is lightweight but the output may include some measurement errors. The output should be used with good From 59cbd435300c613626b7291da533e3cde0f4b30a Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:16 +0800 Subject: [PATCH 350/539] mm/sparse-vmemmap: factor out shared vmemmap tail page allocation Patch series "mm: Switch device DAX to section-based vmemmap optimization", v6. This series is split out from the earlier, larger series "mm: Generalize HVO for HugeTLB and device DAX" [1]. While the parent series generalizes vmemmap optimization across HugeTLB and device DAX, this subset addresses a single, self-contained step: switching device DAX to the section-based sparse-vmemmap optimization infrastructure introduced for HugeTLB. After the HugeTLB conversion, optimized vmemmap state is described by the memory section and the sparse-vmemmap population path can allocate or reuse shared tail vmemmap pages based on that metadata. Device DAX still uses the older DAX-specific population model, including a separate tail vmemmap page reservation and architecture-specific logic to locate or populate reusable tail pages. This series makes device DAX use the same section-based model. Device DAX records the compound page order from pgmap->vmemmap_shift in section metadata before vmemmap population, uses the common per-zone shared tail vmemmap page, and drops the extra reserved tail page. The powerpc radix path is updated to use the same shared tail-page helper, so the generic and powerpc DAX paths follow the same reservation model. The first patches prepare the shared infrastructure by factoring out shared tail-page allocation, allocating the per-zone shared tail-page array dynamically, and introducing a generic CONFIG_VMEMMAP_OPTIMIZATION symbol. The middle patches move device DAX onto that infrastructure by recording the device DAX compound page order in memory-section metadata, using that metadata to back generic device DAX mappings with the common per-zone shared tail page, exposing the shared helpers so the powerpc radix path can use the same model, and dropping the extra DAX-only tail page reservation and the now-unused section accounting arguments. The final patch updates the documentation for the new DAX layout. This is intended to be the third smaller step toward the broader HVO generalization. The wider HVO consolidation between HugeTLB and device DAX is left for follow-up series. This patch (of 12): HugeTLB and sparse-vmemmap each have their own helper to allocate the shared vmemmap tail page used by vmemmap optimization. Factor that logic into a common vmemmap_shared_tail_page() helper. It allocates the page through vmemmap_alloc_block(), initializes the tail struct pages, and uses cmpxchg() to install the per-zone shared page. This removes duplicate allocation logic while handling both early boot and runtime allocation through the same helper. Link: https://lore.kernel.org/20260930140627.57431-1-songmuchun@bytedance.com Link: https://lore.kernel.org/20260930140627.57431-2-songmuchun@bytedance.com Link: https://lore.kernel.org/20260513130542.35604-1-songmuchun@bytedance.com/ [1] Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- mm/hugetlb_vmemmap.c | 29 +----------------- mm/sparse-vmemmap.c | 70 ++++++++++++++++++++------------------------ mm/sparse.h | 3 ++ 3 files changed, 36 insertions(+), 66 deletions(-) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index f977d0a7e002..76765c97ff68 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -19,7 +19,6 @@ #include #include "hugetlb_vmemmap.h" #include "sparse.h" -#include "internal.h" /** * struct vmemmap_remap_walk - walk vmemmap page table @@ -493,32 +492,6 @@ static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio * return true; } -static struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) -{ - const unsigned int idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; - struct page *tail, *p; - int node = zone_to_nid(zone); - - tail = READ_ONCE(zone->vmemmap_tails[idx]); - if (likely(tail)) - return tail; - - tail = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); - if (!tail) - return NULL; - - p = page_to_virt(tail); - for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) - init_compound_tail(p + i, NULL, order, zone); - - if (cmpxchg(&zone->vmemmap_tails[idx], NULL, tail)) { - __free_page(tail); - tail = READ_ONCE(zone->vmemmap_tails[idx]); - } - - return tail; -} - static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio, struct list_head *vmemmap_pages, @@ -535,7 +508,7 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, return ret; nid = folio_nid(folio); - vmemmap_tail = vmemmap_get_tail(h->order, folio_zone(folio)); + vmemmap_tail = vmemmap_shared_tail_page(h->order, folio_zone(folio)); if (!vmemmap_tail) return -ENOMEM; diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index f22d815d7af0..9af349581fbd 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -42,27 +42,13 @@ #include "mm_init.h" #include "sparse.h" -/* - * Allocate a block of memory to be used to back the virtual memory map - * or to back the page tables that are used to create the mapping. - * Uses the main allocators if they are available, else bootmem. - */ - -static void * __ref __earlyonly_bootmem_alloc(int node, - unsigned long size, - unsigned long align, - unsigned long goal) -{ - return memmap_alloc(size, align, goal, node, false); -} - -void * __meminit vmemmap_alloc_block(unsigned long size, int node) +void __ref *vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN; int order = get_order(size); - static bool warned __meminitdata; + static bool warned; struct page *page; page = alloc_pages_node(node, gfp_mask, order); @@ -76,8 +62,7 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node) } return NULL; } else - return __earlyonly_bootmem_alloc(node, size, size, - __pa(MAX_DMA_ADDRESS)); + return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), node, false); } static void * __meminit altmap_alloc_block_buf(unsigned long size, @@ -185,34 +170,43 @@ static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) } #ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP -static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) +struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zone) { - struct page *p, *tail; - unsigned int idx; - int node = zone_to_nid(zone); + const unsigned int idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; + struct page *page; + void *addr; - if (WARN_ON_ONCE(order < VMEMMAP_OPTIMIZATION_MIN_ORDER)) - return NULL; - if (WARN_ON_ONCE(order > MAX_FOLIO_ORDER)) + if (WARN_ON_ONCE(idx >= VMEMMAP_OPTIMIZATION_NR_ORDERS)) return NULL; - idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; - tail = zone->vmemmap_tails[idx]; - if (tail) - return tail; - p = vmemmap_alloc_block_zero(PAGE_SIZE, node); - if (!p) + page = READ_ONCE(zone->vmemmap_tails[idx]); + if (page) + return page; + + addr = vmemmap_alloc_block(PAGE_SIZE, zone_to_nid(zone)); + if (!addr) return NULL; - for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) - init_compound_tail(p + i, NULL, order, zone); - tail = virt_to_page(p); - zone->vmemmap_tails[idx] = tail; + for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) { + page = (struct page *)addr + i; + mm_zero_struct_page(page); + init_compound_tail(page, NULL, order, zone); + } - return tail; + page = virt_to_page(addr); + if (cmpxchg(&zone->vmemmap_tails[idx], NULL, page) != NULL) { + if (slab_is_available()) + __free_page(page); + else + memblock_free(addr, PAGE_SIZE); + page = READ_ONCE(zone->vmemmap_tails[idx]); + } + + return page; } #else -static inline struct page *vmemmap_get_tail(unsigned int order, struct zone *zone) +static inline struct page *vmemmap_shared_tail_page(unsigned int order, + struct zone *zone) { return NULL; } @@ -229,7 +223,7 @@ static __meminit void *vmemmap_alloc_pte(unsigned long pfn, int node, return vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); zone = pfn_to_zone(pfn, node); - page = vmemmap_get_tail(order, zone); + page = vmemmap_shared_tail_page(order, zone); if (!page) return NULL; diff --git a/mm/sparse.h b/mm/sparse.h index d3a71ef4fad0..6e7aaeaa5594 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -142,6 +142,9 @@ static inline void sparse_sections_init(void) {} * mm/sparse-vmemmap.c */ #ifdef CONFIG_SPARSEMEM_VMEMMAP +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); +#endif void sparse_init_subsection_map(void); int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); From 74c32c8ccdead37487db3dcfad8dbe320732c17e Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:17 +0800 Subject: [PATCH 351/539] mm/sparse-vmemmap: allocate shared tail page array dynamically Commit 622026e87c40 ("mm/hugetlb: remove fake head pages") added the per-zone vmemmap_tails array. Its size depends on MAX_FOLIO_ORDER, which had been moved to mmzone.h in preparation for the array. PUD_ORDER is defined by linux/pgtable.h, which cannot be included from mmzone.h without creating an include cycle. It was therefore open-coded as PUD_SHIFT - PAGE_SHIFT. This removed the dependency on PUD_ORDER, but not the underlying dependency on architecture page-table definitions. PUD_SHIFT is generally provided by architecture page-table headers, which are not guaranteed to have been included when mmzone.h is parsed. The dependency remained hidden because vmemmap_tails was originally guarded by CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP. Under that condition, MAX_FOLIO_ORDER resolves to either MAX_PAGE_ORDER or the fixed HugeTLB limit, rather than the PUD_SHIFT-based definition. Device DAX, however, does not require CONFIG_HUGETLB_PAGE. When it is converted to use section-based vmemmap optimization, MAX_FOLIO_ORDER can resolve to PUD_SHIFT - PAGE_SHIFT while it is being used to size vmemmap_tails. This would make struct zone depend on architecture page-table definitions being available when mmzone.h is parsed. Replace the embedded array with a pointer and allocate it on first use. This moves the order-count evaluation into sparse-vmemmap.c, after the architecture page-table definitions are available, and removes the dependency from mmzone.h. Removing the compile-time array also removes the original reason for keeping MAX_FOLIO_ORDER and the vmemmap optimization sizing definitions in mmzone.h. Follow-up cleanups can place each definition in the header owned by its respective subsystem. Link: https://lore.kernel.org/20260930140627.57431-3-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Mike Rapoport Cc: Qi Zheng Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- include/linux/mmzone.h | 7 +------ mm/sparse-vmemmap.c | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index acd94cecc0d3..68807ff7f946 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -113,11 +113,6 @@ (VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page)) #define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) -#define __VMEMMAP_OPTIMIZATION_NR_ORDERS \ - (MAX_FOLIO_ORDER - VMEMMAP_OPTIMIZATION_MIN_ORDER + 1) -#define VMEMMAP_OPTIMIZATION_NR_ORDERS \ - (__VMEMMAP_OPTIMIZATION_NR_ORDERS > 0 ? __VMEMMAP_OPTIMIZATION_NR_ORDERS : 0) - enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, @@ -1156,7 +1151,7 @@ struct zone { atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; #ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP - struct page *vmemmap_tails[VMEMMAP_OPTIMIZATION_NR_ORDERS]; + struct page **vmemmap_tails; #endif } ____cacheline_internodealigned_in_smp; diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 9af349581fbd..0ace48268095 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -170,16 +170,47 @@ static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) } #ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#define VMEMMAP_OPTIMIZATION_NR_ORDERS (MAX_FOLIO_ORDER - VMEMMAP_OPTIMIZATION_MIN_ORDER + 1) + +static __ref struct page **vmemmap_tails(struct zone *zone) +{ + const size_t size = array_size(VMEMMAP_OPTIMIZATION_NR_ORDERS, + sizeof(*zone->vmemmap_tails)); + struct page **pages = READ_ONCE(zone->vmemmap_tails); + + if (pages) + return pages; + + pages = slab_is_available() ? kzalloc_objs(*pages, VMEMMAP_OPTIMIZATION_NR_ORDERS) : + memblock_alloc(size, __alignof__(*pages)); + if (!pages) + return NULL; + + if (cmpxchg(&zone->vmemmap_tails, NULL, pages) != NULL) { + if (slab_is_available()) + kfree(pages); + else + memblock_free(pages, size); + pages = READ_ONCE(zone->vmemmap_tails); + } + + return pages; +} + struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zone) { const unsigned int idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER; - struct page *page; + struct page *page, **pages; void *addr; if (WARN_ON_ONCE(idx >= VMEMMAP_OPTIMIZATION_NR_ORDERS)) return NULL; - page = READ_ONCE(zone->vmemmap_tails[idx]); + pages = vmemmap_tails(zone); + if (!pages) + return NULL; + + page = READ_ONCE(pages[idx]); if (page) return page; @@ -194,12 +225,12 @@ struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zon } page = virt_to_page(addr); - if (cmpxchg(&zone->vmemmap_tails[idx], NULL, page) != NULL) { + if (cmpxchg(&pages[idx], NULL, page) != NULL) { if (slab_is_available()) __free_page(page); else memblock_free(addr, PAGE_SIZE); - page = READ_ONCE(zone->vmemmap_tails[idx]); + page = READ_ONCE(pages[idx]); } return page; From d8777177dcb9d3cd60e9361b1b5be3be372349fe Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:18 +0800 Subject: [PATCH 352/539] mm/sparse-vmemmap: introduce CONFIG_VMEMMAP_OPTIMIZATION The section-based vmemmap optimization infrastructure is guarded by CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP, but it can also be used by ZONE_DEVICE users that set dev_pagemap::vmemmap_shift. Introduce CONFIG_VMEMMAP_OPTIMIZATION as a common config for the shared infrastructure. Select the new option from HUGETLB_PAGE_OPTIMIZE_VMEMMAP and from ZONE_DEVICE when the architecture opts in to DAX vmemmap optimization, and use it to guard the generic sparse-vmemmap state and helpers. Link: https://lore.kernel.org/20260930140627.57431-4-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- arch/x86/entry/vdso/vdso32/fake_32bit_build.h | 2 +- fs/Kconfig | 1 + include/linux/mm.h | 3 +++ include/linux/mmzone.h | 10 +++++----- include/linux/page-flags.h | 5 ++--- mm/Kconfig | 5 +++++ mm/sparse-vmemmap.c | 2 +- mm/sparse.h | 6 +++--- 8 files changed, 21 insertions(+), 13 deletions(-) diff --git a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h index bc3e549795c3..72a92cb9b53d 100644 --- a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h +++ b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h @@ -11,7 +11,7 @@ #undef CONFIG_PGTABLE_LEVELS #undef CONFIG_ILLEGAL_POINTER_VALUE #undef CONFIG_SPARSEMEM_VMEMMAP -#undef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#undef CONFIG_VMEMMAP_OPTIMIZATION #undef CONFIG_NR_CPUS #undef CONFIG_PARAVIRT_XXL diff --git a/fs/Kconfig b/fs/Kconfig index d1c210c6508f..1454b7fe9641 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -278,6 +278,7 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP def_bool HUGETLB_PAGE depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP depends on SPARSEMEM_VMEMMAP + select VMEMMAP_OPTIMIZATION config HUGETLB_PMD_PAGE_TABLE_SHARING def_bool HUGETLB_PAGE diff --git a/include/linux/mm.h b/include/linux/mm.h index c49ef99b4413..070ce27e9cd3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5175,6 +5175,9 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, unsigned long nr_pages; unsigned long nr_vmemmap_pages; + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) + return false; + if (!pgmap || !is_power_of_2(sizeof(struct page))) return false; diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 68807ff7f946..ee9cbaaa63f4 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -102,9 +102,9 @@ * * HVO which is only active if the size of struct page is a power of 2. */ -#define MAX_FOLIO_VMEMMAP_ALIGN \ - (IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && \ - is_power_of_2(sizeof(struct page)) ? \ +#define MAX_FOLIO_VMEMMAP_ALIGN \ + (IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \ + is_power_of_2(sizeof(struct page)) ? \ MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) /* The number of retained vmemmap pages with HVO enabled. */ @@ -1150,7 +1150,7 @@ struct zone { /* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#ifdef CONFIG_VMEMMAP_OPTIMIZATION struct page **vmemmap_tails; #endif } ____cacheline_internodealigned_in_smp; @@ -2014,7 +2014,7 @@ struct mem_section { unsigned long section_mem_map; struct mem_section_usage *usage; -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#ifdef CONFIG_VMEMMAP_OPTIMIZATION /* * Normally, sections hold regular (order-0) pages. However, for * sections with HVO enabled, this tracks the compound page order diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 86dd0470da11..7080a6a1a79e 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -208,14 +208,13 @@ enum pageflags { static __always_inline bool compound_info_has_mask(void) { /* - * Limit mask usage to HugeTLB vmemmap optimization (HVO) where it - * makes a difference. + * Limit mask usage to HVO where it makes a difference. * * The approach with mask would work in the wider set of conditions, * but it requires validating that struct pages are naturally aligned * for all orders up to the MAX_FOLIO_ORDER, which can be tricky. */ - if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP)) + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) return false; return is_power_of_2(sizeof(struct page)); diff --git a/mm/Kconfig b/mm/Kconfig index bc7befafb47b..30170a936f1f 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -461,6 +461,10 @@ config SPARSEMEM_VMEMMAP pfn_to_page and page_to_pfn operations. This is the most efficient option when sufficient kernel resources are available. +config VMEMMAP_OPTIMIZATION + bool + depends on SPARSEMEM_VMEMMAP + # # Select this config option from the architecture Kconfig, if it is preferred # to enable the feature of HugeTLB/dev_dax vmemmap optimization. @@ -1220,6 +1224,7 @@ config ZONE_DMA32 config ZONE_DEVICE bool "Device memory (pmem, HMM, etc...) hotplug support" depends on MEMORY_HOTREMOVE + select VMEMMAP_OPTIMIZATION if ARCH_WANT_OPTIMIZE_DAX_VMEMMAP select XARRAY_MULTI help diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 0ace48268095..6916a3690778 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -169,7 +169,7 @@ static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) return p; } -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#ifdef CONFIG_VMEMMAP_OPTIMIZATION #define VMEMMAP_OPTIMIZATION_NR_ORDERS (MAX_FOLIO_ORDER - VMEMMAP_OPTIMIZATION_MIN_ORDER + 1) static __ref struct page **vmemmap_tails(struct zone *zone) diff --git a/mm/sparse.h b/mm/sparse.h index 6e7aaeaa5594..326ad43bb5c3 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -10,7 +10,7 @@ #include -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#ifdef CONFIG_VMEMMAP_OPTIMIZATION static inline unsigned int section_compound_order(const struct mem_section *section) { return section->compound_page_order; @@ -75,7 +75,7 @@ static inline bool vmemmap_optimizable_pfn(unsigned long pfn) static inline bool vmemmap_optimizable_order(unsigned int order) { - if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP)) + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) return false; if (!is_power_of_2(sizeof(struct page))) @@ -142,7 +142,7 @@ static inline void sparse_sections_init(void) {} * mm/sparse-vmemmap.c */ #ifdef CONFIG_SPARSEMEM_VMEMMAP -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#ifdef CONFIG_VMEMMAP_OPTIMIZATION struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); #endif void sparse_init_subsection_map(void); From 69392d8e9e1d471b95238a5836bb7c2d98374dda Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:19 +0800 Subject: [PATCH 353/539] mm/sparse-vmemmap: open-code init_compound_tail() init_compound_tail() is only used by vmemmap_shared_tail_page(), where the shared tail page setup intentionally passes NULL as the compound head. Keeping this helper in mm/internal.h exposes that special case to the rest of the MM code and can make the NULL head argument look generally valid. Open-code the initialization at the only call site so the special-case use stays local to sparse vmemmap optimization. No functional change intended. Link: https://lore.kernel.org/20260930140627.57431-5-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Acked-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- mm/internal.h | 9 --------- mm/sparse-vmemmap.c | 5 ++++- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index da14c56fb24e..0dca33db068f 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -786,15 +786,6 @@ static inline void prep_compound_tail(struct page *tail, VM_WARN_ON_ONCE(tail->private); } -static inline void init_compound_tail(struct page *tail, - const struct page *head, unsigned int order, struct zone *zone) -{ - atomic_set(&tail->_mapcount, -1); - set_page_node(tail, zone_to_nid(zone)); - set_page_zone(tail, zone_idx(zone)); - prep_compound_tail(tail, head, order); -} - #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 6916a3690778..f77e6e1e5a8b 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -221,7 +221,10 @@ struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zon for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) { page = (struct page *)addr + i; mm_zero_struct_page(page); - init_compound_tail(page, NULL, order, zone); + atomic_set(&page->_mapcount, -1); + set_page_node(page, zone_to_nid(zone)); + set_page_zone(page, zone_idx(zone)); + prep_compound_tail(page, NULL, order); } page = virt_to_page(addr); From 0e13353ec9369f9918c0160129ea9bfff7d7907c Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:20 +0800 Subject: [PATCH 354/539] mm/sparse-vmemmap: prepare DAX vmemmap population for compound page orders Device DAX still uses vmemmap_populate_compound_pages() to populate its compound-page vmemmap mappings. That helper allocates the head and first tail vmemmap pages explicitly, then reuses the first tail page for the remaining tail page mappings. Device DAX is being moved to the section-based vmemmap optimization infrastructure, but it cannot switch to the generic section-based population path yet. Once a later patch records the DAX compound page order in section metadata, DAX head and first-tail PFNs can look optimizable to the generic helpers as well. Rename the existing VMEMMAP_POPULATE_PAGEREF flag to VMEMMAP_POPULATE_DAX and pass it through all paths in vmemmap_populate_compound_pages() during this transition. This keeps DAX head/first-tail allocations on the normal vmemmap allocation path, while preserving the existing page reference for reused DAX tail mappings. Link: https://lore.kernel.org/20260930140627.57431-6-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Cc: David Hildenbrand Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Mike Rapoport Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- mm/sparse-vmemmap.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index f77e6e1e5a8b..b4b72f233bf6 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -35,8 +35,8 @@ /* * Flags for vmemmap_populate_range and friends. */ -/* Get a ref on the head page struct page, for ZONE_DEVICE compound pages */ -#define VMEMMAP_POPULATE_PAGEREF 0x0001 +/* Vmemmap population for ZONE_DEVICE compound pages */ +#define VMEMMAP_POPULATE_DAX 0x0001 #include "internal.h" #include "mm_init.h" @@ -247,13 +247,17 @@ static inline struct page *vmemmap_shared_tail_page(unsigned int order, #endif static __meminit void *vmemmap_alloc_pte(unsigned long pfn, int node, - struct vmem_altmap *altmap) + struct vmem_altmap *altmap, unsigned long flags) { struct zone *zone; struct page *page; const unsigned int order = pfn_to_section_compound_order(pfn); - if (!vmemmap_optimizable_pfn(pfn)) + /* + * Device DAX still relies on vmemmap_populate_compound_pages() for + * head/first-tail allocation and tail-page reuse. + */ + if (!vmemmap_optimizable_pfn(pfn) || flags & VMEMMAP_POPULATE_DAX) return vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); zone = pfn_to_zone(pfn, node); @@ -275,7 +279,7 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in pte_t entry; if (ptpfn == (unsigned long)-1) { - void *p = vmemmap_alloc_pte(pfn, node, altmap); + void *p = vmemmap_alloc_pte(pfn, node, altmap, flags); if (!p) return NULL; @@ -290,7 +294,7 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in * and through vmemmap_populate_compound_pages() when * slab is available. */ - if (flags & VMEMMAP_POPULATE_PAGEREF) + if (flags & VMEMMAP_POPULATE_DAX) get_page(pfn_to_page(ptpfn)); } entry = pfn_pte(ptpfn, PAGE_KERNEL); @@ -547,6 +551,7 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, unsigned long end, int node, struct dev_pagemap *pgmap) { + const unsigned long flags = VMEMMAP_POPULATE_DAX; unsigned long size, addr; pte_t *pte; int rc; @@ -561,8 +566,7 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * with just tail struct pages. */ return vmemmap_populate_range(start, end, node, NULL, - pte_pfn(ptep_get(pte)), - VMEMMAP_POPULATE_PAGEREF); + pte_pfn(ptep_get(pte)), flags); } size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page)); @@ -570,13 +574,13 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, unsigned long next, last = addr + size; /* Populate the head page vmemmap page */ - pte = vmemmap_populate_address(addr, node, NULL, -1, 0); + pte = vmemmap_populate_address(addr, node, NULL, -1, flags); if (!pte) return -ENOMEM; /* Populate the tail pages vmemmap page */ next = addr + PAGE_SIZE; - pte = vmemmap_populate_address(next, node, NULL, -1, 0); + pte = vmemmap_populate_address(next, node, NULL, -1, flags); if (!pte) return -ENOMEM; @@ -586,8 +590,7 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, */ next += PAGE_SIZE; rc = vmemmap_populate_range(next, last, node, NULL, - pte_pfn(ptep_get(pte)), - VMEMMAP_POPULATE_PAGEREF); + pte_pfn(ptep_get(pte)), flags); if (rc) return -ENOMEM; } From 2ef77f4a7d5a9c3c40c34bd85ceaa617ad520d83 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:21 +0800 Subject: [PATCH 355/539] mm/sparse-vmemmap: set compound page order for device DAX Device DAX can use vmemmap optimization only when a full section is populated with a compound-page geometry. Record that geometry as the compound page order in section metadata before populating the section, so later vmemmap accounting and population decisions can use the section state directly. Clear the compound page order when the section becomes empty again. Also reject partial additions to a section that already has optimized vmemmap mappings. compound_nr_pages() determines how many struct pages to initialize with a section as the smallest granularity. A section therefore cannot safely mix optimized and ordinary vmemmap layouts. Partial additions continue to use ordinary vmemmap population, so they do not save vmemmap memory. Such additions are uncommon, and the lost saving is negligible. Link: https://lore.kernel.org/20260930140627.57431-7-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Mike Rapoport Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- mm/mm_init.c | 15 +++++---------- mm/sparse-vmemmap.c | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 97e0158d2aca..efffa8609b85 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1049,16 +1049,11 @@ static void zone_device_page_init_from_template(struct page *page, * of an altmap. See vmemmap_populate_compound_pages(). */ static inline unsigned long compound_nr_pages(unsigned long pfn, - struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - /* - * If DAX memory is hot-plugged into an unoccupied subsection - * of an early section, the unoptimized boot memmap is reused. - * See section_activate(). - */ - if (early_section(__pfn_to_section(pfn)) || - !vmemmap_can_optimize(altmap, pgmap)) + const struct mem_section *ms = __pfn_to_section(pfn); + + if (!section_vmemmap_optimizable(ms)) return pgmap_vmemmap_nr(pgmap); return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); @@ -1144,7 +1139,7 @@ void __ref memmap_init_zone_device(struct zone *zone, memcpy(&template, page, sizeof(*page)); if (pfns_per_compound != 1) memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(pfn, altmap, pgmap)); + compound_nr_pages(pfn, pgmap)); pfn += pfns_per_compound; /* Initialize the remaining head pages from template. */ @@ -1160,7 +1155,7 @@ void __ref memmap_init_zone_device(struct zone *zone, continue; memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(pfn, altmap, pgmap)); + compound_nr_pages(pfn, pgmap)); } pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index b4b72f233bf6..26be355aaa37 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -135,14 +135,14 @@ int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { const struct mem_section *ms = __pfn_to_section(pfn); - const int order = pgmap ? pgmap->vmemmap_shift : section_compound_order(ms); + const int order = section_compound_order(ms); const int vmemmap_pages = pgmap ? VMEMMAP_RESERVE_NR : VMEMMAP_OPTIMIZATION_PAGES; const unsigned long pages_per_compound = 1UL << order; VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION); - if (!vmemmap_can_optimize(altmap, pgmap) && !section_vmemmap_optimizable(ms)) + if (!section_vmemmap_optimizable(ms)) return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); if (order < PFN_SECTION_SHIFT) { @@ -612,7 +612,7 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn, !IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION))) return NULL; - if (vmemmap_can_optimize(altmap, pgmap)) + if (pgmap && section_vmemmap_optimizable(__pfn_to_section(pfn))) r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap); else r = vmemmap_populate(start, end, nid, altmap); @@ -831,8 +831,10 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, else if (memmap) free_map_bootmem(memmap); - if (empty) + if (empty) { ms->section_mem_map = (unsigned long)NULL; + section_set_compound_order(ms, 0); + } } static struct page * __meminit section_activate(int nid, unsigned long pfn, @@ -842,8 +844,13 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn, struct mem_section *ms = __pfn_to_section(pfn); struct mem_section_usage *usage = NULL; struct page *memmap; + unsigned int order; int rc; + order = vmemmap_can_optimize(altmap, pgmap) ? pgmap->vmemmap_shift : 0; + if (nr_pages < PAGES_PER_SECTION && section_compound_order(ms)) + return ERR_PTR(-EOPNOTSUPP); + if (!ms->usage) { usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); if (!usage) @@ -869,6 +876,7 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn, if (nr_pages < PAGES_PER_SECTION && early_section(ms)) return pfn_to_page(pfn); + section_set_compound_order_range(pfn, nr_pages, order); memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); if (!memmap) { section_deactivate(pfn, nr_pages, altmap, pgmap); From ba9164f880bfe5e58e1706cd38a2b7a975849cdd Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:22 +0800 Subject: [PATCH 356/539] mm/sparse-vmemmap: switch device DAX to shared tail vmemmap pages HugeTLB vmemmap optimization now uses per-zone shared tail vmemmap pages. Device DAX has not been switched to that mechanism yet. Switch device DAX to vmemmap_shared_tail_page() as well. This aligns DAX with HugeTLB by using the common per-zone shared tail vmemmap page. The optimization is enabled only for DEV-DAX through pgmap->vmemmap_shift, which supplies the compound page order recorded in section metadata before vmemmap population. Unlike FS-DAX, DEV-DAX does not modify tail struct pages, so sharing them is safe. Since the shared tail page can now back ZONE_DEVICE vmemmap mappings, initialize its entries with PG_reserved for device zones. Also skip poisoning vmemmap-optimizable sections while their struct pages may be shared. Each PTE mapping the shared device DAX tail page takes a page reference. A sufficiently large range could therefore cycle the reference count back to zero if population were allowed to continue after it became non-positive. Use try_get_page() so further mappings fail at that point. The section population error path tears down mappings created for the failed section, while the warning makes this currently impractical limit visible. [muchun.song@linux.dev: set PG_reserved before preparing the compound tail] Link: https://lore.kernel.org/20260930150748.1134516-1-songmuchun@bytedance.com Link: https://lore.kernel.org/20260930140627.57431-8-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Cc: David Hildenbrand Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Mike Rapoport Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- include/linux/mmzone.h | 10 +++++++ mm/memory_hotplug.c | 6 ++-- mm/sparse-vmemmap.c | 67 ++++++++++++++++++------------------------ 3 files changed, 42 insertions(+), 41 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ee9cbaaa63f4..cd68c1904c91 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2143,11 +2143,21 @@ static inline int online_device_section(const struct mem_section *section) return section && ((section->section_mem_map & flags) == flags); } + +static inline struct zone *device_zone(int nid) +{ + return &NODE_DATA(nid)->node_zones[ZONE_DEVICE]; +} #else static inline int online_device_section(const struct mem_section *section) { return 0; } + +static inline struct zone *device_zone(int nid) +{ + return NULL; +} #endif static inline int online_section_nr(unsigned long nr) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index b428da66d279..d7a59167bec4 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -43,6 +43,7 @@ #include "mm_init.h" #include "page_alloc.h" #include "shuffle.h" +#include "sparse.h" enum { MEMMAP_ON_MEMORY_DISABLE = 0, @@ -554,8 +555,9 @@ void remove_pfn_range_from_zone(struct zone *zone, /* Select all remaining pages up to the next section boundary */ cur_nr_pages = min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn); - page_init_poison(pfn_to_page(pfn), - sizeof(struct page) * cur_nr_pages); + if (!section_vmemmap_optimizable(__pfn_to_section(pfn))) + page_init_poison(pfn_to_page(pfn), + sizeof(struct page) * cur_nr_pages); } /* diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 26be355aaa37..060342ac4882 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -224,6 +224,8 @@ struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zon atomic_set(&page->_mapcount, -1); set_page_node(page, zone_to_nid(zone)); set_page_zone(page, zone_idx(zone)); + if (zone_is_zone_device(zone)) + __SetPageReserved(page); prep_compound_tail(page, NULL, order); } @@ -288,14 +290,18 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in /* * When a PTE/PMD entry is freed from the init_mm * there's a free_pages() call to this page allocated - * above. Thus this get_page() is paired with the + * above. Thus this try_get_page() is paired with the * put_page_testzero() on the freeing path. * This can only called by certain ZONE_DEVICE path, * and through vmemmap_populate_compound_pages() when * slab is available. + * + * Use try_get_page() to prevent the shared page refcount + * from overflowing. */ - if (flags & VMEMMAP_POPULATE_DAX) - get_page(pfn_to_page(ptpfn)); + if ((flags & VMEMMAP_POPULATE_DAX) && + !try_get_page(pfn_to_page(ptpfn))) + return NULL; } entry = pfn_pte(ptpfn, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); @@ -529,47 +535,27 @@ static bool __meminit reuse_compound_section(unsigned long start_pfn, return !IS_ALIGNED(offset, nr_pages) && nr_pages > PAGES_PER_SUBSECTION; } -static pte_t * __meminit compound_section_tail_page(unsigned long addr) -{ - pte_t *pte; - - addr -= PAGE_SIZE; - - /* - * Assuming sections are populated sequentially, the previous section's - * page data can be reused. - */ - pte = pte_offset_kernel(pmd_off_k(addr), addr); - if (!pte) - return NULL; - - return pte; -} - static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, unsigned long start, unsigned long end, int node, struct dev_pagemap *pgmap) { const unsigned long flags = VMEMMAP_POPULATE_DAX; + const unsigned int order = pfn_to_section_compound_order(start_pfn); unsigned long size, addr; pte_t *pte; + struct page *page; int rc; - if (reuse_compound_section(start_pfn, pgmap)) { - pte = compound_section_tail_page(start); - if (!pte) - return -ENOMEM; + page = vmemmap_shared_tail_page(order, device_zone(node)); + if (!page) + return -ENOMEM; - /* - * Reuse the page that was populated in the prior iteration - * with just tail struct pages. - */ + if (reuse_compound_section(start_pfn, pgmap)) return vmemmap_populate_range(start, end, node, NULL, - pte_pfn(ptep_get(pte)), flags); - } + page_to_pfn(page), flags); - size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page)); + size = min(end - start, (1UL << order) * sizeof(struct page)); for (addr = start; addr < end; addr += size) { unsigned long next, last = addr + size; @@ -585,12 +571,12 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; /* - * Reuse the previous page for the rest of tail pages + * Reuse the shared page for the rest of tail pages * See layout diagram in Documentation/mm/vmemmap_dedup.rst */ next += PAGE_SIZE; rc = vmemmap_populate_range(next, last, node, NULL, - pte_pfn(ptep_get(pte)), flags); + page_to_pfn(page), flags); if (rc) return -ENOMEM; } @@ -922,13 +908,16 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn, if (IS_ERR(memmap)) return PTR_ERR(memmap); - /* - * Poison uninitialized struct pages in order to catch invalid flags - * combinations. - */ - page_init_poison(memmap, sizeof(struct page) * nr_pages); - ms = __nr_to_section(section_nr); + /* + * Poison uninitialized struct pages to catch invalid flag combinations. + * + * Tail struct pages in a vmemmap-optimized section are initialized and + * shared during vmemmap population, so they must not be overwritten here. + */ + if (!section_vmemmap_optimizable(ms)) + page_init_poison(memmap, sizeof(struct page) * nr_pages); + __section_mark_present(ms, section_nr); /* Align memmap to section boundary in the subsection case */ From d3a699634c2a69d5ae0773535d4135eba14da557 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:23 +0800 Subject: [PATCH 357/539] mm/sparse-vmemmap: move vmemmap optimization helpers to a public header The vmemmap optimization helpers currently live in mm/sparse.h, which is an internal MM header. That works for MM code, but prevents powerpc from using the same interfaces without including a private header. Move the declarations and inline helpers to vmemmap-optimization.h. This is a preparatory change for powerpc, which has its own vmemmap optimization implementation and needs to use the common vmemmap optimization interfaces from architecture code. Link: https://lore.kernel.org/20260930140627.57431-9-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- MAINTAINERS | 1 + arch/loongarch/include/asm/pgtable.h | 1 + arch/riscv/mm/init.c | 1 + include/linux/mmzone.h | 17 ----- include/linux/vmemmap-optimization.h | 109 +++++++++++++++++++++++++++ mm/hugetlb.c | 2 +- mm/hugetlb_vmemmap.c | 2 +- mm/sparse.h | 78 +------------------ 8 files changed, 115 insertions(+), 96 deletions(-) create mode 100644 include/linux/vmemmap-optimization.h diff --git a/MAINTAINERS b/MAINTAINERS index 1d1b48c1cc42..edad4bf21735 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12098,6 +12098,7 @@ F: Documentation/mm/hugetlbfs_reserv.rst F: Documentation/mm/vmemmap_dedup.rst F: fs/hugetlbfs/ F: include/linux/hugetlb.h +F: include/linux/vmemmap-optimization.h F: include/trace/events/hugetlbfs.h F: mm/hugetlb.c F: mm/hugetlb_cgroup.c diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index cf29a4c8ac59..f87603135131 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -72,6 +72,7 @@ #include #include +#include #include #include diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index fb37b0b67efe..857f9a55039c 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index cd68c1904c91..65de3bb13eb3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -96,23 +96,6 @@ #define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER) -/* - * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to - * be naturally aligned with regard to the folio size. - * - * HVO which is only active if the size of struct page is a power of 2. - */ -#define MAX_FOLIO_VMEMMAP_ALIGN \ - (IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \ - is_power_of_2(sizeof(struct page)) ? \ - MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) - -/* The number of retained vmemmap pages with HVO enabled. */ -#define VMEMMAP_OPTIMIZATION_PAGES 1 -#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \ - (VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page)) -#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) - enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, diff --git a/include/linux/vmemmap-optimization.h b/include/linux/vmemmap-optimization.h new file mode 100644 index 000000000000..bd0974b262a4 --- /dev/null +++ b/include/linux/vmemmap-optimization.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * vmemmap-optimization.h + * + * Generic vmemmap optimization declarations. + * + * Author: Muchun Song + */ +#ifndef _LINUX_VMEMMAP_OPTIMIZATION_H +#define _LINUX_VMEMMAP_OPTIMIZATION_H + +#include +#include +#include +#include + +/* + * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to + * be naturally aligned with regard to the folio size. + * + * HVO which is only active if the size of struct page is a power of 2. + */ +#define MAX_FOLIO_VMEMMAP_ALIGN \ + (IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \ + is_power_of_2(sizeof(struct page)) ? \ + MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) + +/* The number of retained vmemmap pages with HVO enabled. */ +#define VMEMMAP_OPTIMIZATION_PAGES 1 +#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \ + (VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page)) +#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) + +#ifdef CONFIG_VMEMMAP_OPTIMIZATION +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return section->compound_page_order; +} + +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ + VM_WARN_ON(section_compound_order(section) && order && + section_compound_order(section) != order); + section->compound_page_order = order; +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ + unsigned long section_nr = pfn_to_section_nr(pfn); + + if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)) + return; + + for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++) + section_set_compound_order(__nr_to_section(section_nr + i), order); +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return section_compound_order(__pfn_to_section(pfn)); +} + +struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); +#else +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return 0; +} + +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return 0; +} +#endif /* CONFIG_VMEMMAP_OPTIMIZATION */ + +static inline bool vmemmap_optimizable_pfn(unsigned long pfn) +{ + const unsigned int order = pfn_to_section_compound_order(pfn); + const unsigned long nr_pages = 1UL << order; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; +} + +static inline bool vmemmap_optimizable_order(unsigned int order) +{ + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) + return false; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER; +} +#endif /* _LINUX_VMEMMAP_OPTIMIZATION_H */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index afd669640cc3..3403ce546606 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -52,7 +53,6 @@ #include "hugetlb_cma.h" #include "hugetlb_internal.h" #include "mm_init.h" -#include "sparse.h" #include #define HUGE_BOOTMEM_ZONES_VALID BIT(0) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 76765c97ff68..0057fa2a16a8 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -15,10 +15,10 @@ #include #include #include +#include #include #include "hugetlb_vmemmap.h" -#include "sparse.h" /** * struct vmemmap_remap_walk - walk vmemmap page table diff --git a/mm/sparse.h b/mm/sparse.h index 326ad43bb5c3..a5111087ee3a 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -9,80 +9,7 @@ #define __MM_SPARSE_H #include - -#ifdef CONFIG_VMEMMAP_OPTIMIZATION -static inline unsigned int section_compound_order(const struct mem_section *section) -{ - return section->compound_page_order; -} - -static inline void section_set_compound_order(struct mem_section *section, - unsigned int order) -{ - VM_WARN_ON(section_compound_order(section) && order && - section_compound_order(section) != order); - section->compound_page_order = order; -} - -static inline void section_set_compound_order_range(unsigned long pfn, - unsigned long nr_pages, unsigned int order) -{ - unsigned long section_nr = pfn_to_section_nr(pfn); - - if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)) - return; - - for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++) - section_set_compound_order(__nr_to_section(section_nr + i), order); -} - -static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) -{ - return section_compound_order(__pfn_to_section(pfn)); -} -#else -static inline unsigned int section_compound_order(const struct mem_section *section) -{ - return 0; -} - -static inline void section_set_compound_order(struct mem_section *section, - unsigned int order) -{ -} - -static inline void section_set_compound_order_range(unsigned long pfn, - unsigned long nr_pages, unsigned int order) -{ -} - -static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) -{ - return 0; -} -#endif - -static inline bool vmemmap_optimizable_pfn(unsigned long pfn) -{ - const unsigned int order = pfn_to_section_compound_order(pfn); - const unsigned long nr_pages = 1UL << order; - - if (!is_power_of_2(sizeof(struct page))) - return false; - - return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; -} - -static inline bool vmemmap_optimizable_order(unsigned int order) -{ - if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) - return false; - - if (!is_power_of_2(sizeof(struct page))) - return false; - - return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER; -} +#include /* * mm/sparse.c @@ -142,9 +69,6 @@ static inline void sparse_sections_init(void) {} * mm/sparse-vmemmap.c */ #ifdef CONFIG_SPARSEMEM_VMEMMAP -#ifdef CONFIG_VMEMMAP_OPTIMIZATION -struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); -#endif void sparse_init_subsection_map(void); int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); From a4fabaa14c2dc93892e31b1ac132f2f680aedb55 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:24 +0800 Subject: [PATCH 358/539] powerpc/mm: switch device DAX to shared tail vmemmap pages The powerpc radix compound vmemmap population path still finds a reusable tail page by walking the vmemmap page tables. Switch it to the common vmemmap_shared_tail_page() helper instead, so it can use the shared vmemmap page directly to simplify the code. This removes the powerpc-specific tail-page lookup and its fallback path and aligns the device DAX vmemmap optimization path with HugeTLB. [muchun.song@linux.dev: use try_get_page()] Link: https://lore.kernel.org/20260930152826.76084-1-songmuchun@bytedance.com Link: https://lore.kernel.org/20260930140627.57431-10-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Mike Rapoport Cc: Qi Zheng Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- arch/powerpc/mm/book3s64/radix_pgtable.c | 88 ++++-------------------- include/linux/vmemmap-optimization.h | 6 ++ mm/sparse-vmemmap.c | 6 -- 3 files changed, 21 insertions(+), 79 deletions(-) diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index cf692b2b5f7b..0e778d17240c 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -1041,13 +1042,17 @@ static pte_t * __meminit radix__vmemmap_pte_populate(pmd_t *pmdp, unsigned long /* * When a PTE/PMD entry is freed from the init_mm * there's a free_pages() call to this page allocated - * above. Thus this get_page() is paired with the + * above. Thus this try_get_page() is paired with the * put_page_testzero() on the freeing path. * This can only called by certain ZONE_DEVICE path, * and through vmemmap_populate_compound_pages() when * slab is available. + * + * Use try_get_page() to prevent the shared page refcount + * from overflowing. */ - get_page(reuse); + if (!try_get_page(reuse)) + return NULL; p = page_to_virt(reuse); pr_debug("Tail page reuse vmemmap mapping\n"); } @@ -1250,59 +1255,6 @@ static pte_t * __meminit radix__vmemmap_populate_address(unsigned long addr, int return pte; } -static pte_t * __meminit vmemmap_compound_tail_page(unsigned long addr, - unsigned long pfn_offset, int node) -{ - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - unsigned long map_addr; - - /* the second vmemmap page which we use for duplication */ - map_addr = addr - pfn_offset * sizeof(struct page) + PAGE_SIZE; - pgd = pgd_offset_k(map_addr); - p4d = p4d_offset(pgd, map_addr); - pud = vmemmap_pud_alloc(p4d, node, map_addr); - if (!pud) - return NULL; - pmd = vmemmap_pmd_alloc(pud, node, map_addr); - if (!pmd) - return NULL; - if (pmd_leaf(*pmd)) - /* - * The second page is mapped as a hugepage due to a nearby request. - * Force our mapping to page size without deduplication - */ - return NULL; - pte = vmemmap_pte_alloc(pmd, node, map_addr); - if (!pte) - return NULL; - /* - * Check if there exist a mapping to the left - */ - if (pte_none(*pte)) { - /* - * Populate the head page vmemmap page. - * It can fall in different pmd, hence - * vmemmap_populate_address() - */ - pte = radix__vmemmap_populate_address(map_addr - PAGE_SIZE, node, NULL, NULL); - if (!pte) - return NULL; - /* - * Populate the tail pages vmemmap page - */ - pte = radix__vmemmap_pte_populate(pmd, map_addr, node, NULL, NULL); - if (!pte) - return NULL; - vmemmap_verify(pte, node, map_addr, map_addr + PAGE_SIZE); - return pte; - } - return pte; -} - int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, unsigned long start, unsigned long end, int node, @@ -1320,6 +1272,12 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, pud_t *pud; pmd_t *pmd; pte_t *pte; + struct page *tail_page; + unsigned int order = pfn_to_section_compound_order(start_pfn); + + tail_page = vmemmap_shared_tail_page(order, device_zone(node)); + if (!tail_page) + return -ENOMEM; for (addr = start; addr < end; addr = next) { @@ -1349,10 +1307,9 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, next = addr + PAGE_SIZE; continue; } else { - unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); + unsigned long nr_pages = 1UL << order; unsigned long addr_pfn = page_to_pfn((struct page *)addr); unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages); - pte_t *tail_page_pte; /* * if the address is aligned to huge page size it is the @@ -1377,23 +1334,8 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, next = addr + 2 * PAGE_SIZE; continue; } - /* - * get the 2nd mapping details - * Also create it if that doesn't exist - */ - tail_page_pte = vmemmap_compound_tail_page(addr, pfn_offset, node); - if (!tail_page_pte) { - pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, NULL); - if (!pte) - return -ENOMEM; - vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - - next = addr + PAGE_SIZE; - continue; - } - - pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, pte_page(*tail_page_pte)); + pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, tail_page); if (!pte) return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); diff --git a/include/linux/vmemmap-optimization.h b/include/linux/vmemmap-optimization.h index bd0974b262a4..fa9e9abd6656 100644 --- a/include/linux/vmemmap-optimization.h +++ b/include/linux/vmemmap-optimization.h @@ -83,6 +83,12 @@ static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) { return 0; } + +static inline struct page *vmemmap_shared_tail_page(unsigned int order, + struct zone *zone) +{ + return NULL; +} #endif /* CONFIG_VMEMMAP_OPTIMIZATION */ static inline bool vmemmap_optimizable_pfn(unsigned long pfn) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 060342ac4882..14f99c129b38 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -240,12 +240,6 @@ struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zon return page; } -#else -static inline struct page *vmemmap_shared_tail_page(unsigned int order, - struct zone *zone) -{ - return NULL; -} #endif static __meminit void *vmemmap_alloc_pte(unsigned long pfn, int node, From 2f84a6a92724292b0b914d31bfb04301dee0479e Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:25 +0800 Subject: [PATCH 359/539] mm/sparse-vmemmap: drop the extra tail page from device DAX reservation The device DAX vmemmap population still reserves one extra tail vmemmap page after the head page. Drop that extra reservation and let the shared tail page cover all tail vmemmap pages after the head page, so DAX follows the same reservation model as HugeTLB. This reduces the reserved vmemmap pages for optimized DAX mappings to one and removes the now-unneeded first-tail population from the generic and powerpc paths to simplify the code as well. Link: https://lore.kernel.org/20260930140627.57431-11-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Mike Rapoport Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- arch/powerpc/mm/book3s64/radix_pgtable.c | 46 ++---------------------- include/linux/mm.h | 4 +-- mm/mm_init.c | 2 +- mm/sparse-vmemmap.c | 13 ++----- 4 files changed, 8 insertions(+), 57 deletions(-) diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 0e778d17240c..a3332f32ffb2 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1222,39 +1222,6 @@ base_mapping: return 0; } -static pte_t * __meminit radix__vmemmap_populate_address(unsigned long addr, int node, - struct vmem_altmap *altmap, - struct page *reuse) -{ - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - - pgd = pgd_offset_k(addr); - p4d = p4d_offset(pgd, addr); - pud = vmemmap_pud_alloc(p4d, node, addr); - if (!pud) - return NULL; - pmd = vmemmap_pmd_alloc(pud, node, addr); - if (!pmd) - return NULL; - if (pmd_leaf(*pmd)) - /* - * The second page is mapped as a hugepage due to a nearby request. - * Force our mapping to page size without deduplication - */ - return NULL; - pte = vmemmap_pte_alloc(pmd, node, addr); - if (!pte) - return NULL; - radix__vmemmap_pte_populate(pmd, addr, node, NULL, NULL); - vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - - return pte; -} - int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, unsigned long start, unsigned long end, int node, @@ -1301,7 +1268,7 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (!pte_none(*pte)) { /* * This could be because we already have a compound - * page whose VMEMMAP_RESERVE_NR pages were mapped and + * page whose retained vmemmap page was mapped and * this request fall in those pages. */ next = addr + PAGE_SIZE; @@ -1322,16 +1289,7 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - /* - * Populate the tail pages vmemmap page - * It can fall in different pmd, hence - * vmemmap_populate_address() - */ - pte = radix__vmemmap_populate_address(addr + PAGE_SIZE, node, NULL, NULL); - if (!pte) - return -ENOMEM; - - next = addr + 2 * PAGE_SIZE; + next = addr + PAGE_SIZE; continue; } diff --git a/include/linux/mm.h b/include/linux/mm.h index 070ce27e9cd3..30a3365bca82 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -38,6 +38,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -5167,7 +5168,6 @@ static inline void vmem_altmap_free(struct vmem_altmap *altmap, } #endif -#define VMEMMAP_RESERVE_NR 2 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) @@ -5187,7 +5187,7 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, * For vmemmap optimization with DAX we need minimum 2 vmemmap * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst */ - return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR); + return !altmap && (nr_vmemmap_pages > VMEMMAP_OPTIMIZATION_PAGES); } /* * If we don't have an architecture override, use the generic rule diff --git a/mm/mm_init.c b/mm/mm_init.c index efffa8609b85..56bb4567a494 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1056,7 +1056,7 @@ static inline unsigned long compound_nr_pages(unsigned long pfn, if (!section_vmemmap_optimizable(ms)) return pgmap_vmemmap_nr(pgmap); - return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); + return VMEMMAP_OPTIMIZATION_PAGES * (PAGE_SIZE / sizeof(struct page)); } static void __ref memmap_init_compound(struct page *head, diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 14f99c129b38..0f4c2a893030 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -136,7 +136,6 @@ int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages { const struct mem_section *ms = __pfn_to_section(pfn); const int order = section_compound_order(ms); - const int vmemmap_pages = pgmap ? VMEMMAP_RESERVE_NR : VMEMMAP_OPTIMIZATION_PAGES; const unsigned long pages_per_compound = 1UL << order; VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); @@ -147,13 +146,13 @@ int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages if (order < PFN_SECTION_SHIFT) { VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound)); - return vmemmap_pages * nr_pages / pages_per_compound; + return VMEMMAP_OPTIMIZATION_PAGES * nr_pages / pages_per_compound; } VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)); if (IS_ALIGNED(pfn, pages_per_compound)) - return vmemmap_pages; + return VMEMMAP_OPTIMIZATION_PAGES; return 0; } @@ -558,17 +557,11 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (!pte) return -ENOMEM; - /* Populate the tail pages vmemmap page */ - next = addr + PAGE_SIZE; - pte = vmemmap_populate_address(next, node, NULL, -1, flags); - if (!pte) - return -ENOMEM; - /* * Reuse the shared page for the rest of tail pages * See layout diagram in Documentation/mm/vmemmap_dedup.rst */ - next += PAGE_SIZE; + next = addr + PAGE_SIZE; rc = vmemmap_populate_range(next, last, node, NULL, page_to_pfn(page), flags); if (rc) From 76df418d2957fc10de866e83a9d5c000e15b25c5 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:26 +0800 Subject: [PATCH 360/539] mm/sparse-vmemmap: drop unused section_nr_vmemmap_pages() arguments section_nr_vmemmap_pages() no longer uses the altmap or pgmap arguments, so drop them from the helper and its callers. Link: https://lore.kernel.org/20260930140627.57431-12-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Mike Rapoport Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- mm/sparse-vmemmap.c | 10 ++++------ mm/sparse.c | 3 +-- mm/sparse.h | 6 ++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 0f4c2a893030..23b31bc4bf94 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -131,8 +131,7 @@ void __meminit vmemmap_verify(pte_t *pte, int node, start, end - 1); } -int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, - struct vmem_altmap *altmap, struct dev_pagemap *pgmap) +int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages) { const struct mem_section *ms = __pfn_to_section(pfn); const int order = section_compound_order(ms); @@ -674,7 +673,7 @@ static struct page * __meminit populate_section_memmap(unsigned long pfn, struct page *page = __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); - memmap_pages_add(section_nr_vmemmap_pages(pfn, nr_pages, altmap, pgmap)); + memmap_pages_add(section_nr_vmemmap_pages(pfn, nr_pages)); return page; } @@ -685,7 +684,7 @@ static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, unsigned long start = (unsigned long) pfn_to_page(pfn); unsigned long end = start + nr_pages * sizeof(struct page); - memmap_pages_add(-section_nr_vmemmap_pages(pfn, nr_pages, altmap, pgmap)); + memmap_pages_add(-section_nr_vmemmap_pages(pfn, nr_pages)); vmemmap_free(start, end, altmap); } @@ -695,8 +694,7 @@ static void free_map_bootmem(struct page *memmap) unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); unsigned long pfn = page_to_pfn(memmap); - memmap_boot_pages_add(-section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION, - NULL, NULL)); + memmap_boot_pages_add(-section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION)); vmemmap_free(start, end, NULL); } diff --git a/mm/sparse.c b/mm/sparse.c index cc28bb41fdb1..b75921c622ed 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -250,8 +250,7 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, nid, NULL, NULL); if (!map) panic("Failed to allocate memmap for section %lu\n", pnum); - memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION, - NULL, NULL)); + memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION)); sparse_init_one_section(__nr_to_section(pnum), pnum, map, usage, SECTION_IS_EARLY); usage = (void *)usage + mem_section_usage_size(); diff --git a/mm/sparse.h b/mm/sparse.h index a5111087ee3a..530692cdd516 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -70,12 +70,10 @@ static inline void sparse_sections_init(void) {} */ #ifdef CONFIG_SPARSEMEM_VMEMMAP void sparse_init_subsection_map(void); -int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, - struct vmem_altmap *altmap, struct dev_pagemap *pgmap); +int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages); #else static inline void sparse_init_subsection_map(void) {} -static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, - struct vmem_altmap *altmap, struct dev_pagemap *pgmap) +static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages) { return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); } From 27ffeec7b0457d973b8b096763c97e12a17cf961 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 30 Sep 2026 22:06:27 +0800 Subject: [PATCH 361/539] Documentation/mm: update DAX vmemmap deduplication docs Device DAX now uses the common per-zone shared tail page for vmemmap deduplication. The old documentation still described a DAX-specific layout with a separately populated tail vmemmap page and half the HugeTLB savings. Update the generic and powerpc documentation to describe the shared layout. In the powerpc document, keep the radix and 64K-specific details, drop the duplicated 4K PUD arithmetic, and replace the repeated device-dax diagrams with a single parameterized PMD/PUD diagram. Link: https://lore.kernel.org/20260930140627.57431-13-songmuchun@bytedance.com Signed-off-by: Muchun Song Signed-off-by: Andrew Morton Acked-by: Qi Zheng Acked-by: David Hildenbrand (Arm) Cc: Oscar Salvador Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Jonathan Corbet Cc: Lorenzo Stoakes Cc: Mike Rapoport Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Ritesh Harjani (IBM) Cc: Shrikanth Hegde Cc: Randy Dunlap Cc: Lance Yang --- Documentation/arch/powerpc/vmemmap_dedup.rst | 88 ++++---------------- Documentation/mm/vmemmap_dedup.rst | 32 +------ 2 files changed, 20 insertions(+), 100 deletions(-) diff --git a/Documentation/arch/powerpc/vmemmap_dedup.rst b/Documentation/arch/powerpc/vmemmap_dedup.rst index dc4db59fdf87..8286acbca9bc 100644 --- a/Documentation/arch/powerpc/vmemmap_dedup.rst +++ b/Documentation/arch/powerpc/vmemmap_dedup.rst @@ -19,82 +19,28 @@ With 1G PUD level mapping, we require 16384 struct pages and a single 64K vmemmap page can contain 1024 struct pages (64K/sizeof(struct page)). Hence we require 16 64K pages in vmemmap to map the struct page for 1G PUD level mapping. -Here's how things look like on device-dax after the sections are populated:: - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PUD | +-----------+ | | | - | level | | . | ----------------------+ | | - | mapping | +-----------+ | | - | | | . | ------------------------+ | - | | +-----------+ | - | | | 15 | --------------------------+ - | | +-----------+ - | | - | | - | | - +-----------+ - - With 4K page size, 2M PMD level mapping requires 512 struct pages and a single 4K vmemmap page contains 64 struct pages(4K/sizeof(struct page)). Hence we require 8 4K pages in vmemmap to map the struct page for 2M pmd level mapping. -Here's how things look like on device-dax after the sections are populated:: +Here's how things look on device-dax after vmemmap-optimized sections are +populated. ``N`` is the number of vmemmap pages required by the DAX mapping +above:: - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PMD | +-----------+ | | | - | level | | 5 | ----------------------+ | | - | mapping | +-----------+ | | - | | | 6 | ------------------------+ | - | | +-----------+ | - | | | 7 | --------------------------+ - | | +-----------+ - | | - | | - | | - +-----------+ - -With 1G PUD level mapping, we require 262144 struct pages and a single 4K -vmemmap page can contain 64 struct pages (4K/sizeof(struct page)). Hence we -require 4096 4K pages in vmemmap to map the struct pages for 1G PUD level -mapping. - -Here's how things look like on device-dax after the sections are populated:: - - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PUD | +-----------+ | | | - | level | | . | ----------------------+ | | - | mapping | +-----------+ | | - | | | . | ------------------------+ | - | | +-----------+ | - | | | 4095 | --------------------------+ - | | +-----------+ + Device DAX vmemmap pages (N pages) backing page frames + +-----------+ ---virt_to_page---> +-----------+ mapping to +-------------+ + | | | 0 | -------------> | 0 | + | | +-----------+ +-------------+ + | | | 1 | ------+ + | | +-----------+ | + | | | 2 | ------+ + | | +-----------+ | + | | | . | ------+ +-------------+ + | PMD/PUD | +-----------+ | | A single, | + | level | | . | ------+------> | per-zone | + | mapping | +-----------+ | | shared tail | + | | | N - 1 | ------+ | page | + | | +-----------+ +-------------+ | | | | | | diff --git a/Documentation/mm/vmemmap_dedup.rst b/Documentation/mm/vmemmap_dedup.rst index 9fa8642ded48..8c287ae3f86c 100644 --- a/Documentation/mm/vmemmap_dedup.rst +++ b/Documentation/mm/vmemmap_dedup.rst @@ -1,4 +1,3 @@ - .. SPDX-License-Identifier: GPL-2.0 ========================================= @@ -192,32 +191,7 @@ to 4 on HugeTLB pages. There's no remapping of vmemmap given that device-dax memory is not part of System RAM ranges initialized at boot. Thus the tail page deduplication -happens at a later stage when we populate the sections. HugeTLB reuses the -the head vmemmap page representing, whereas device-dax reuses the tail -vmemmap page. This results in only half of the savings compared to HugeTLB. +happens at a later stage when we populate the sections. -Deduplicated tail pages are not mapped read-only. - -Here's how things look like on device-dax after the sections are populated:: - - +-----------+ ---virt_to_page---> +-----------+ mapping to +-----------+ - | | | 0 | -------------> | 0 | - | | +-----------+ +-----------+ - | | | 1 | -------------> | 1 | - | | +-----------+ +-----------+ - | | | 2 | ----------------^ ^ ^ ^ ^ ^ - | | +-----------+ | | | | | - | | | 3 | ------------------+ | | | | - | | +-----------+ | | | | - | | | 4 | --------------------+ | | | - | PMD | +-----------+ | | | - | level | | 5 | ----------------------+ | | - | mapping | +-----------+ | | - | | | 6 | ------------------------+ | - | | +-----------+ | - | | | 7 | --------------------------+ - | | +-----------+ - | | - | | - | | - +-----------+ +Deduplicated tail pages are not mapped read-only. The mapping layout is the same +as HugeTLB. From 718de76e2d3dc38a30026f23909842ad26010b6e Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 11 Sep 2026 11:52:43 -0400 Subject: [PATCH 362/539] lib: fix lock initialization in region allocation benchmark The Maple Tree benchmark uses MTREE_INIT() for a stack-allocated tree. Its static spinlock initializer leaves lockdep to use the lock address as the class key. Since the address is on the stack, the first allocation triggers "INFO: trying to register non-static key" and disables lockdep. The IDA benchmark has the same problem through IDA_INIT(), but runs after Maple Tree and therefore encounters an already disabled lockdep. Use mt_init_flags() and ida_init() to initialize the locks with persistent lock-class keys. Keep initialization outside the timed allocation paths. Link: https://lore.kernel.org/20260911155244.1406122-1-ynorov@nvidia.com Fixes: f4806cc63cc6 ("lib: test bitmap vs IDA vs Maple Tree performance for region allocations") Signed-off-by: Yury Norov Signed-off-by: Andrew Morton Closes: https://lore.kernel.org/oe-lkp/202609101106.771b567e-lkp@intel.com Acked-by: Liam R. Howlett (Oracle) Cc: Alice Ryhl Cc: Andrew Ballance Cc: Matthew Wilcox (Oracle) Cc: Rasmus Villemoes --- lib/region_alloc_benchmark.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/region_alloc_benchmark.c b/lib/region_alloc_benchmark.c index e88b4cf55c62..a644f3d5431a 100644 --- a/lib/region_alloc_benchmark.c +++ b/lib/region_alloc_benchmark.c @@ -78,11 +78,13 @@ static size_t __init ida_size(unsigned long nr_ids) static unsigned long __init benchmark_ida(unsigned long cap) { - struct ida ida = IDA_INIT(ida); + struct ida ida; unsigned long cnt, idx, off, nr_ids = 0; ktime_t alloc_time, free_time; int id = -ENOSPC; + ida_init(&ida); + alloc_time = ktime_get(); for (cnt = 0; cnt <= cap; cnt++) { for (off = 0; off < reg_sz[cnt]; off++) { @@ -125,12 +127,14 @@ static unsigned long __init benchmark_ida(unsigned long cap) static unsigned long __init benchmark_maple_tree(unsigned long cap) { - struct maple_tree mt = MTREE_INIT(mt, MT_FLAGS_ALLOC_RANGE); + struct maple_tree mt; unsigned long cnt, idx; ktime_t alloc_time, free_time; size_t sz; int ret; + mt_init_flags(&mt, MT_FLAGS_ALLOC_RANGE); + alloc_time = ktime_get(); for (cnt = 0; cnt <= cap; cnt++) { ret = mtree_alloc_range(&mt, &idx, xa_mk_value(cnt + 1), From a0dba61b040abbfe4ab991d725a59077d7411a4f Mon Sep 17 00:00:00 2001 From: Yeoreum Yun Date: Fri, 11 Sep 2026 15:29:03 +0100 Subject: [PATCH 363/539] kselftest: mm: fix potential failure for merged VMA in guard-regions check_vmflag_guard() uses /proc/self/smaps to retrieve the VMA flags, but this can fail if the mapping is merged with an adjacent VMA. To avoid this potential failure, first allocate a temporary region with extra pages at both ends, unmap it, and then map the test region within the temporary address range, leaving an unmapped page on each side to prevent VMA merging. Link: https://lore.kernel.org/20260911142904.1825452-1-yeoreum.yun@arm.com Signed-off-by: Yeoreum Yun Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/mm/guard-regions.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/mm/guard-regions.c b/tools/testing/selftests/mm/guard-regions.c index 5c8ec3ca75d7..b724d62d2b75 100644 --- a/tools/testing/selftests/mm/guard-regions.c +++ b/tools/testing/selftests/mm/guard-regions.c @@ -2257,8 +2257,18 @@ TEST_F(guard_regions, smaps) char *ptr, *ptr2; int i; - /* Map a region. */ - ptr = mmap_(self, variant, NULL, 10 * page_size, PROT_READ | PROT_WRITE, 0, 0); + /* Map then unmap placeholder to avoid adjacent merges */ + ptr = mmap_(self, variant, NULL, 12 * page_size, PROT_NONE, 0, 0); + ASSERT_NE(ptr, MAP_FAILED); + ASSERT_EQ(munmap(ptr, 12 * page_size), 0); + + /* + * Map a region for the test. Since the preceding temporary mapping + * succeeded, this mapping should also succeed without merging with + * adjacent VMAs. + */ + ptr = mmap_(self, variant, ptr + page_size, 10 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED, 0); ASSERT_NE(ptr, MAP_FAILED); /* We shouldn't yet see a guard flag. */ From 724468135d75874f14d1f66ab4d330de636170e6 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 11 Sep 2026 06:55:03 -0700 Subject: [PATCH 364/539] mm/damon/api: introduce DAMOS_FILTER_TYPE_PROBE_HITS_WSUM Patch series "mm/damon: introduce probe_hits_wsum DAMOS core filter", v2. DAMON can do flexible data attributes monitoring. The classical data access monitoring can also be done using the attributes monitoring. The access event is just one of the data attributes that DAMON supports. Users do monitoring to make some actions based on it. DAMOS is a feature for automating that. However, DAMOS cannot utilize the data attributes monitoring results. It is still Data "Access" Monitoring-based Operation Schemes. It requires users to set the target "access" pattern. DAMOS core filter is effectively the same as the target access pattern. It is just a more generalized and flexible way of describing the operation action target region. Introduce a new DAMOS core filter type, probe_hits_wsum. It specifies the filter target based on a range of the probe hits weighted sum. Using this, users can apply DAMOS actions to regions of specific data attributes pattern. Note that the classic target access pattern still works. Hence the target nr_accesses range should still be properly configured. The new filter would be used in only data attributes-only mode. In the mode, classic access monitoring is just turned off, and therefore nr_accesses of regions are always zero. Users could simply set the target nr_accesses range to include the zero nr_Accesses regions. Patches Sequence ================ Patch 1 updates the DAMON kernel API for the new filter type. Patch 2 extends damon_probe_hits_wsum() to do the calculation based on moving sum. Patch 3 implements the filter type in the core layer. Patch 4 refactors DAMON sysfs interface internal data structure for efficient reuse of data structure for the probe hits weighted sum range user inputs. Patch 5 updates DAMON sysfs interface to support the new filter type. Patches 6 and 7 update design and usage documents for the new filter type, respectively. This patch (of 7): Update DAMON kernel API to introduce new DAMOS core filter type, PROBE_HITS_WSUM. It will allow API callers to describe the DAMOS action target regions based on their probe_hits weighted sum. For describing the filtering target weighted sum range, add two type-dependent union fields to damos_filter. [sj@kernel.org: clarify DAMOS_FILTER_TYPE_PROBE_HITS_WSUM behavior] Link: https://lore.kernel.org/20260911142522.98013-1-sj@kernel.org Link: https://lore.kernel.org/20260911135510.96914-1-sj@kernel.org Link: https://lore.kernel.org/20260911135510.96914-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 16800b3dc379..e27ed156e765 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -399,14 +399,15 @@ struct damos_stat { * @DAMOS_FILTER_TYPE_UNMAPPED: Unmapped pages. * @DAMOS_FILTER_TYPE_ADDR: Address range. * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. + * @DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: probe_hits weighted sum range. * @NR_DAMOS_FILTER_TYPES: Number of filter types. * - * All types except &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET - * are handled by the underlying &struct damon_operations as a part of scheme - * action trying, and therefore accounted as 'tried'. In contrast, - * &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET filters are handled - * by the core layer before trying of the action, and therefore not accounted - * as 'tried'. + * All types except &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET and + * &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM are handled by the underlying &struct + * damon_operations as a part of scheme action trying, and therefore accounted + * as 'tried'. In contrast, &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET + * and &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM filters are handled by the core layer + * before trying of the action, and therefore not accounted as 'tried'. * * Support for the operations-handled filters depends on the running * &struct damon_operations. @@ -420,6 +421,7 @@ enum damos_filter_type { DAMOS_FILTER_TYPE_UNMAPPED, DAMOS_FILTER_TYPE_ADDR, DAMOS_FILTER_TYPE_TARGET, + DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, NR_DAMOS_FILTER_TYPES, }; @@ -434,6 +436,8 @@ enum damos_filter_type { * &damon_ctx->adaptive_targets if @type is * DAMOS_FILTER_TYPE_TARGET. * @sz_range: Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE. + * @range_min: Minimum value of range arguments. + * @range_max: Maximum value of range arguments. * * Before applying the &damos->action to a memory region, DAMOS checks if each * byte of the region matches to this given condition and avoid applying the @@ -450,6 +454,10 @@ struct damos_filter { struct damon_addr_range addr_range; int target_idx; struct damon_size_range sz_range; + struct { + unsigned long range_min; + unsigned long range_max; + }; }; /* private: */ /* List head for siblings. */ From 283b3d31ec7a62bf4517fc57d07d8faf2bdbc841 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 11 Sep 2026 06:55:04 -0700 Subject: [PATCH 365/539] mm/damon/core: extend probe_hits_wsum() for moving sum based calculation damon_probe_hits_wsum() is being called only in aggregation time. In future, it could also be used by DAMOS. In this case, since DAMOS uses its own apply_interval, it could be called in sampling time. Then using the not yet fully aggregated probe_hits could result in suboptimum outcomes. Extend damon_probe_hits_wsum() to get the weighted sum based on moving sum to prepare the DAMOS usage. Link: https://lore.kernel.org/20260911135510.96914-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 2 +- mm/damon/core.c | 8 ++++++-- mm/damon/paddr.c | 2 +- mm/damon/vaddr.c | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index e27ed156e765..4be7d1df8e71 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1075,7 +1075,7 @@ unsigned int damon_nr_accesses_mvsum(struct damon_region *r, struct damon_ctx *ctx); unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, struct damon_ctx *ctx); -unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, bool mv, struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, diff --git a/mm/damon/core.c b/mm/damon/core.c index ea6df4311ceb..2b361ab24078 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -469,11 +469,12 @@ static bool damon_is_last_region(struct damon_region *r, * damon_probe_hits_wsum() - Returns probe hits weighted sum of a region. * @r: region to get the weighted sum of. * @last: if the request is for last-window aggregated probe hits. + * @mv: use moving sum. * @ctx: context of &r. * * Return: the weighted sum of probe hits of the region. */ -unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, bool mv, struct damon_ctx *ctx) { struct damon_probe *probe; @@ -483,6 +484,9 @@ unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, damon_for_each_probe(probe, ctx) { if (last) sum += r->last_probe_hits[i++] * probe->weight; + else if (mv) + sum += damon_probe_hits_mvsum(i++, r, ctx) * + probe->weight; else sum += r->probe_hits[i++] * probe->weight; } @@ -3472,7 +3476,7 @@ static unsigned int damon_merge_score(struct damon_region *r, bool last, struct damon_ctx *ctx, bool use_probe_hits) { if (use_probe_hits) - return damon_probe_hits_wsum(r, last, ctx); + return damon_probe_hits_wsum(r, last, false, ctx); if (last) return r->last_nr_accesses; return r->nr_accesses; diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 79195026b903..5abfabaa339e 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -208,7 +208,7 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, folio_put(folio); if (return_max_wsum) max_wsum = max(damon_probe_hits_wsum(r, false, - ctx), max_wsum); + false, ctx), max_wsum); } } return max_wsum; diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 7063356370c3..d5dde97b3cd0 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -711,7 +711,7 @@ static unsigned int damon_va_apply_probes(struct damon_ctx *ctx, __damon_va_apply_probes(ctx, mm, r); if (return_max_wsum) max_wsum = max(damon_probe_hits_wsum(r, false, - ctx), max_wsum); + false, ctx), max_wsum); } if (mm) mmput(mm); From 17d1f22173f5071095b41a1ae5453902289f8ad2 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 11 Sep 2026 06:55:05 -0700 Subject: [PATCH 366/539] mm/damon/core: support probe_hits_wsum damos core filter Implement probe_hits_wsum DAMOS core filter support in the core layer. Make three small changes for the support. First, update damos_filter_for_ops() to treat probe_hits_wsum filter as core filter. Second, Update destination damos_filter->range_{min,max} for probe_hits_wsum type damos filter commits. Third, extend damos_filter_match() to handle probe_hits_wsum type filter. Calculate the weighted sum of the given region and compare it with the given filter's target weighted sum range. Link: https://lore.kernel.org/20260911135510.96914-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 2b361ab24078..38383ced3dd6 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -658,6 +658,7 @@ bool damos_filter_for_ops(enum damos_filter_type type) switch (type) { case DAMOS_FILTER_TYPE_ADDR: case DAMOS_FILTER_TYPE_TARGET: + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: return false; default: break; @@ -1332,6 +1333,10 @@ static void damos_commit_filter_arg( case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE: dst->sz_range = src->sz_range; break; + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: + dst->range_min = src->range_min; + dst->range_max = src->range_max; + break; default: break; } @@ -2510,7 +2515,7 @@ static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t, bool matched = false; struct damon_target *ti; int target_idx = 0; - unsigned long start, end; + unsigned long start, end, wsum; switch (filter->type) { case DAMOS_FILTER_TYPE_TARGET: @@ -2545,6 +2550,11 @@ static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t, damon_split_region_at(t, r, end - r->ar.start); matched = true; break; + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: + wsum = damon_probe_hits_wsum(r, false, true, ctx); + matched = filter->range_min <= wsum && + wsum <= filter->range_max; + break; default: return false; } From 7bcb11735cee6f936a66ac718fd582bb4a85b11e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 11 Sep 2026 06:55:06 -0700 Subject: [PATCH 367/539] mm/damon/sysfs-schemes: rename sysfs_filter->sz_range to range_{min,max} DAMON sysfs interface provides 'min' and 'max' files under the DAMOS filter directory. The purpose is setting the general range arguments for hugepage_size like filters that require range arguments. So far, hugepage_size was the only filter using it. Hence sz_range field of damon_sysfs_scheme_filter struct was connected to the files. In future, we could add a new filter that can reuse the 'min' and 'max' files. And the filter might use a range of a type that is not size. For example, probe_hits weighted sum. In this case, simply reusing the sz_range field would make it a little confusing. Rename sz_range to range_{min,max} to avoid such confusion. Link: https://lore.kernel.org/20260911135510.96914-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs-schemes.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index d9b81d7b5910..4d9147d2a269 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -534,7 +534,8 @@ struct damon_sysfs_scheme_filter { bool allow; char *memcg_path; struct damon_addr_range addr_range; - struct damon_size_range sz_range; + unsigned long range_min; + unsigned long range_max; int target_idx; }; @@ -588,6 +589,7 @@ damos_sysfs_filter_type_names[] = { .type = DAMOS_FILTER_TYPE_TARGET, .name = "target", }, + }; static ssize_t type_show(struct kobject *kobj, @@ -778,7 +780,7 @@ static ssize_t min_show(struct kobject *kobj, struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); - return sysfs_emit(buf, "%lu\n", filter->sz_range.min); + return sysfs_emit(buf, "%lu\n", filter->range_min); } static ssize_t min_store(struct kobject *kobj, @@ -786,7 +788,7 @@ static ssize_t min_store(struct kobject *kobj, { struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); - int err = kstrtoul(buf, 0, &filter->sz_range.min); + int err = kstrtoul(buf, 0, &filter->range_min); return err ? err : count; } @@ -797,7 +799,7 @@ static ssize_t max_show(struct kobject *kobj, struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); - return sysfs_emit(buf, "%lu\n", filter->sz_range.max); + return sysfs_emit(buf, "%lu\n", filter->range_max); } static ssize_t max_store(struct kobject *kobj, @@ -805,7 +807,7 @@ static ssize_t max_store(struct kobject *kobj, { struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); - int err = kstrtoul(buf, 0, &filter->sz_range.max); + int err = kstrtoul(buf, 0, &filter->range_max); return err ? err : count; } @@ -2835,12 +2837,13 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) { filter->target_idx = sysfs_filter->target_idx; } else if (filter->type == DAMOS_FILTER_TYPE_HUGEPAGE_SIZE) { - if (sysfs_filter->sz_range.min > - sysfs_filter->sz_range.max) { + if (sysfs_filter->range_min > + sysfs_filter->range_max) { damos_destroy_filter(filter); return -EINVAL; } - filter->sz_range = sysfs_filter->sz_range; + filter->sz_range.min = sysfs_filter->range_min; + filter->sz_range.max = sysfs_filter->range_max; } damos_add_filter(scheme, filter); From 0e8f03b730fbf4413683a6017192ecdda4bc35ac Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 11 Sep 2026 06:55:07 -0700 Subject: [PATCH 368/539] mm/damon/sysfs-schemes: support probe_hits_wsum damos core filter Extend DAMON sysfs interface to support probe_hits_wsum input. Also update sysfs input based scheme build logic to setup the min/max probe hits weighted sum range as user provided. Link: https://lore.kernel.org/20260911135510.96914-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs-schemes.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 4d9147d2a269..3de4d804e049 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -589,7 +589,10 @@ damos_sysfs_filter_type_names[] = { .type = DAMOS_FILTER_TYPE_TARGET, .name = "target", }, - + { + .type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, + .name = "probe_hits_wsum", + }, }; static ssize_t type_show(struct kobject *kobj, @@ -2844,6 +2847,13 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, } filter->sz_range.min = sysfs_filter->range_min; filter->sz_range.max = sysfs_filter->range_max; + } else if (filter->type == DAMOS_FILTER_TYPE_PROBE_HITS_WSUM) { + filter->range_min = sysfs_filter->range_min; + filter->range_max = sysfs_filter->range_max; + if (filter->range_min > filter->range_max) { + damos_destroy_filter(filter); + return -EINVAL; + } } damos_add_filter(scheme, filter); From ebb73ca521ebef52bced2c33c5662b9e10de0e04 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 11 Sep 2026 06:55:08 -0700 Subject: [PATCH 369/539] Docs/mm/damon/design: update for probe_hits_wsum DAMOS core filter Update DAMON design document for the newly added probe hits weighted sum based DAMOS core filter type. Link: https://lore.kernel.org/20260911135510.96914-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 22b785cd11df..cb116a82ff20 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -823,6 +823,9 @@ Below ``type`` of filters are currently supported. - Applied to pages that belonging to a given address range. - target - Applied to pages that belonging to a given DAMON monitoring target. + - probe_hits_wsum + - Matches to monitoring regions having a given range of :ref:`probe + hits weighted sum ` value. - Operations layer handled, supported by only ``paddr`` operations set. - anon - Applied to pages that containing data that not stored in files. From 8c32a1afe96c1ec3d9f21d0528d21e10d6737718 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 11 Sep 2026 06:55:09 -0700 Subject: [PATCH 370/539] Docs/admin-guide/mm/damon/usage: update for probe_hits_wsum DAMOS filter Update DAMON usage document for the newly added probe hits weighted sum based DAMOS core filter type. Link: https://lore.kernel.org/20260911135510.96914-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/admin-guide/mm/damon/usage.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 023c6334024f..d3e37400367b 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -551,6 +551,10 @@ and ``damon_target_idx``. To ``type`` file, you can write the type of the filter. Refer to :ref:`the design doc ` for available type names, their meaning and on what layer those are handled. +For ``probe_hits_wsum`` type, you can specify the minimum and maximum probe +hits weighted sum value for the filter to ``min`` and ``max`` files, +respectively. + For ``memcg`` type, you can specify the memory cgroup of the interest by writing the path of the memory cgroup from the cgroups mount point to ``memcg_path`` file. For ``addr`` type, you can specify the start and end From 6593a15af1081a1ea79fe3a70c4da183cc1ef165 Mon Sep 17 00:00:00 2001 From: Jaeyeon Lee Date: Sat, 12 Sep 2026 22:29:03 +0200 Subject: [PATCH 371/539] selftests/mm: skip khugepaged file tests if mkfs.xfs is unavailable The XFS setup for ./khugepaged all:file checks that the kernel supports XFS but not that mkfs.xfs is installed. When CONFIG_XFS_FS=y and xfsprogs is missing, mkfs.xfs and mount both fail, but SPLIT_HUGE_PAGE_TEST_XFS_PATH is assigned from mktemp -d and stays set. The test then runs against plain tmpfs and six subtests fail. The script already has a skip path for this test, but it never runs because the path variable is always set. Assign SPLIT_HUGE_PAGE_TEST_XFS_PATH only once mkfs.xfs and mount have both succeeded, and remove the image and directory otherwise. Link: https://lore.kernel.org/20260912202903.16157-1-jaeyeon.lee.dev@gmail.com Signed-off-by: Jaeyeon Lee Signed-off-by: Andrew Morton Suggested-by: Zi Yan Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Assisted-by: LLM Cc: Shuah Khan Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko --- tools/testing/selftests/mm/run_vmtests.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 9bbef9410ccc..2e5e7975ff4d 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -435,11 +435,15 @@ if [ -z "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then if test_selected "thp"; then if grep xfs /proc/filesystems &>/dev/null; then XFS_IMG=$(mktemp /tmp/xfs_img_XXXXXX) - SPLIT_HUGE_PAGE_TEST_XFS_PATH=$(mktemp -d /tmp/xfs_dir_XXXXXX) + XFS_DIR=$(mktemp -d /tmp/xfs_dir_XXXXXX) truncate -s 314572800 ${XFS_IMG} - mkfs.xfs -q ${XFS_IMG} - mount -o loop ${XFS_IMG} ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} - MOUNTED_XFS=1 + if mkfs.xfs -q ${XFS_IMG} && mount -t xfs -o loop ${XFS_IMG} ${XFS_DIR}; then + SPLIT_HUGE_PAGE_TEST_XFS_PATH=${XFS_DIR} + MOUNTED_XFS=1 + else + rmdir ${XFS_DIR} + rm -f ${XFS_IMG} + fi fi fi fi From 9057170a941f0c5f4058f6fb9ab5d5fc9fcc9e83 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sat, 26 Sep 2026 06:51:09 -0400 Subject: [PATCH 372/539] mm/mempolicy: use vm_normal_folio_pmd() in queue_folios_pmd() Patch series "mm: stop calling pmd_folio() on special PMDs", v3. Two page table walkers resolve the folio behind a PMD with pmd_folio(), which is only valid for a PMD mapping a refcounted struct page: madvise_cold_or_pageout_pte_range() mm/madvise.c queue_folios_pmd() mm/mempolicy.c vmf_insert_pfn_pmd() installs special PMDs holding a raw pfn that need not have a memmap entry at all. Both walkers can reach one and fault on the first folio field read. The PTE halves of both already use vm_normal_folio(); these two patches make the PMD halves match. The four callers of vmf_insert_pfn_pmd(), and which walker each reaches: drivers/vfio/pci/vfio_pci_core.c VM_PFNMAP mempolicy drivers/gpu/drm/drm_gem_shmem_helper.c VM_PFNMAP mempolicy drivers/gpu/drm/panthor/panthor_gem.c VM_PFNMAP mempolicy drivers/hv/mshv_vtl_main.c VM_MIXEDMAP both can_madv_lru_vma() rejects VM_PFNMAP, so only mshv_vtl_low reaches the madvise walker, and that needs CAP_SYS_ADMIN. queue_pages_walk_ops supplies its own ->test_walk, so walk_page_test()'s generic VM_PFNMAP skip never runs and vfio-pci is reachable by any process holding the device fd. Hence the different stable tags. One behaviour change: mbind(MPOL_MF_STRICT) over a PMD mapped VM_PFNMAP region now returns 0 rather than -EIO. The PTE loop already returned 0 there. drm_gem_shmem and panthor are where this is observable, since they PMD map pages that do have a memmap entry and so never faulted. Reproducer ========== No hardware needed. An out of tree module stands in for the drivers above: three misc devices, each with a ->huge_fault calling vmf_insert_pfn_pmd(), plus VM_HUGEPAGE so the fault path takes the PMD branch. /dev/pmdspec_mixed VM_MIXEDMAP, pfn at the 1 TiB mark, no memmap /dev/pmdspec_pfnmap VM_PFNMAP, pfn at the 1 TiB mark, no memmap /dev/pmdspec_real VM_PFNMAP, real alloc_pages(PMD_ORDER) on node 0 Userspace maps the device into a PMD aligned window, reads one byte to fault the PMD in, checks a module parameter to confirm it went in, then issues the operation. vng --run --user root --memory 4G --verbose \ --append "numa=fake=2" \ --exec "insmod pmdspec.ko && ./pmdspec_test " numa=fake=2 gives a node 1 to bind to; the module allocates its real page on node 0, which is what makes queue_folio_required() true. subtest operation parent series -------------------------------------------------------------------- madv_cold madvise(MADV_COLD) oops ret=0 madv_pageout madvise(MADV_PAGEOUT) oops ret=0 mbind_mixed mbind(MPOL_BIND, n1, MPOL_MF_MOVE) oops ret=0 mbind_pfnmap mbind(MPOL_BIND, n1, MPOL_MF_STRICT) oops ret=0 mbind_real mbind(MPOL_BIND, n1, MPOL_MF_STRICT) -EIO ret=0 Two things the table shows that are easy to miss in the code: - mbind_mixed passes only MPOL_MF_MOVE. MPOL_MF_STRICT is not needed for a VM_MIXEDMAP vma: walk_page_test() only skips VM_PFNMAP, and vma_migratable() is true for VM_MIXEDMAP. - mbind_real demonstrates the user visible change (-EIO -> 0) This patch (of 2): mmap a VM_PFNMAP region whose ->huge_fault installs a PMD through vmf_insert_pfn_pmd() - a vfio-pci MMIO BAR does this - then mbind(p, len, MPOL_BIND, &mask, maxnode, MPOL_MF_STRICT); With a stand-in module for the driver: BUG: unable to handle page fault for address: fffff96dc0000008 RIP: 0010:queue_folios_pte_range+0xaf/0x440 walk_pgd_range+0x52b/0xaf0 __walk_page_range+0x6a/0x1d0 walk_page_range_mm_unsafe+0x193/0x230 queue_pages_range+0x64/0xa0 do_mbind+0x25e/0x640 queue_folios_pmd(), inlined above, calls pmd_folio() on that PMD. The pfn is raw MMIO with no memmap entry, so the folio lands in unpopulated vmemmap. Neither guard stops the walk: walk_page_test() skips VM_PFNMAP, but queue_pages_walk_ops supplies ->test_walk, so it never runs queue_pages_test_walk() honours vma_migratable(), but only while MPOL_MF_STRICT is clear A VM_MIXEDMAP vma needs neither flag, being vma_migratable(), so plain mbind(MPOL_MF_MOVE) reaches this too - and there the bad folio carries on into migrate_folio_add() and folio_isolate_lru(). mshv_vtl_low is such a mapping. Use vm_normal_folio_pmd() and skip on NULL, as the PTE loop in queue_folios_pte_range() already does with vm_normal_folio(). This also filters the huge zero PMD, so its separate check is no longer needed. mbind(MPOL_MF_STRICT) over a PMD mapped VM_PFNMAP region now returns 0 rather than -EIO. The PTE loop already returned 0 there. Link: https://lore.kernel.org/20260926105110.2156652-1-gourry@gourry.net Link: https://lore.kernel.org/20260926105110.2156652-2-gourry@gourry.net Fixes: 3c8e44c9b369 ("mm: mark special bits for huge pfn mappings when inject") Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260817220810.1175596-1-gourry%40gourry.net Acked-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Assisted-by: LLM Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Vlastimil Babka Cc: Jann Horn Cc: Joshua Hahn Cc: Rakie Kim Cc: Ying Huang Cc: Peter Xu Cc: Jason Gunthorpe Cc: --- mm/mempolicy.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 95dba5d919e9..e9860fb9f73f 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -667,7 +667,8 @@ static inline bool queue_folio_required(struct folio *folio, return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT); } -static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) +static void queue_folios_pmd(pmd_t *pmd, unsigned long addr, + struct mm_walk *walk) { struct folio *folio; struct queue_pages *qp = walk->private; @@ -678,13 +679,9 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) qp->nr_failed++; return; } - folio = pmd_folio(pmdval); - if (folio_is_zone_device(folio)) + folio = vm_normal_folio_pmd(walk->vma, addr, pmdval); + if (!folio || folio_is_zone_device(folio)) return; - if (is_huge_zero_folio(folio)) { - walk->action = ACTION_CONTINUE; - return; - } if (!queue_folio_required(folio, qp)) return; if (!(qp->flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) || @@ -717,7 +714,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, ptl = pmd_trans_huge_lock(pmd, vma); if (ptl) { - queue_folios_pmd(pmd, walk); + queue_folios_pmd(pmd, addr, walk); spin_unlock(ptl); goto out; } From 0bd719cb607ef14bcd554e044bbd80dd901b9996 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sat, 26 Sep 2026 06:51:10 -0400 Subject: [PATCH 373/539] mm/madvise: use vm_normal_folio_pmd() in cold/pageout PMD range mmap a VM_MIXEDMAP region whose ->huge_fault installs a PMD through vmf_insert_pfn_pmd() - mshv_vtl_low does this, and needs CAP_SYS_ADMIN to open - then: madvise(p, PMD_SIZE, MADV_PAGEOUT); With a stand-in module for the driver: BUG: unable to handle page fault for address: fffff587c0000008 RIP: 0010:madvise_cold_or_pageout_pte_range+0x410/0x9b0 walk_pgd_range+0x52b/0xaf0 __walk_page_range+0x6a/0x1d0 walk_page_range_vma_unsafe+0x8e/0x120 madvise_pageout+0xb2/0x180 madvise_vma_behavior+0x46b/0xa90 do_madvise+0x108/0x190 __x64_sys_madvise+0x26/0x30 Nothing validates the pfn on the way in: can_madv_lru_vma() rejects VM_PFNMAP, but not VM_MIXEDMAP can_fault() *pfn = vmf->pgoff & ~(mask >> PAGE_SHIFT); vmf_insert_pfn_pmd() no pfn_valid() check pmd_folio() pfn_to_page() -> unpopulated vmemmap Even with a valid pfn the path is wrong. The mapping carries no rmap, so folio_maybe_mapped_shared() sees mapcount 0, and the walker goes on to folio_deactivate(), or folio_isolate_lru() plus reclaim_pages(), against a folio this mapping does not own. Use vm_normal_folio_pmd() and skip on NULL, as the PTE half of this same walker already does with vm_normal_folio(). This also filters the huge zero PMD, so its separate check is no longer needed. Link: https://lore.kernel.org/20260926105110.2156652-3-gourry@gourry.net Fixes: 3c8e44c9b369 ("mm: mark special bits for huge pfn mappings when inject") Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260817220810.1175596-1-gourry%40gourry.net Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Assisted-by: LLM Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Jann Horn Cc: Joshua Hahn Cc: Rakie Kim Cc: Ying Huang Cc: Peter Xu Cc: Jason Gunthorpe Cc: # v6.19+ --- mm/madvise.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 963337f93a7a..6c52f699222b 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -394,16 +394,15 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, return 0; orig_pmd = *pmd; - if (is_huge_zero_pmd(orig_pmd)) - goto huge_unlock; - if (unlikely(!pmd_present(orig_pmd))) { VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && !pmd_is_device_private_entry(orig_pmd)); goto huge_unlock; } - folio = pmd_folio(orig_pmd); + folio = vm_normal_folio_pmd(vma, addr, orig_pmd); + if (!folio) + goto huge_unlock; if (folio_is_zone_device(folio)) goto huge_unlock; From 5436132610d94167c44cfe308359d322dd53e2f2 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Tue, 22 Sep 2026 22:29:01 -0400 Subject: [PATCH 374/539] mm: refactor find_next_best_node to find_next_best_node_in Patch series "mm: refactor zonelist constructors and iterators", v3. find_next_best_node() picks the next-closest node when building a fallback list, and hardcodes N_MEMORY as the set it picks from. Refactor it into find_next_best_node_in(), which takes the candidate set explicitly. This makes the existing behaviour explicit at both mm/memory-tiers.c call sites - they select demotion targets in fallback order from N_MEMORY - and lets callers narrow that set. Then extract the per-node construction loop out of build_zonelists() into build_node_zonelist(), parameterised on the candidate nodemask and destination zonelist index. Together these allow a zonelist to be built over a candidate set other than N_MEMORY, into a zonelist other than FALLBACK, and iterated in fallback order over a caller-defined subset. These are prerequisites for generating a private node zonelist (nodes unreachable by default), but are otherwise general improvements to the existing interfaces so I'm proposing them separately. No functional change intended - purely refactor commits. This patch (of 2): find_next_best_node() picks the next-closest node for a fallback list from the full N_MEMORY set. Refactor it into find_next_best_node_in(), which takes an explicit candidates nodemask. This enables building fallback lists with non-N_MEMORY candidates. No functional change: every caller still selects from N_MEMORY. Link: https://lore.kernel.org/20260923022902.2433614-1-gourry@gourry.net Link: https://lore.kernel.org/20260923022902.2433614-2-gourry@gourry.net Signed-off-by: Gregory Price Signed-off-by: Andrew Morton Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Balbir Singh Reviewed-by: Zi Yan Reviewed-by: Zenghui Yu (Huawei) Cc: David Hildenbrand Cc: Lorenzo Stoakes Cc: Liam R. Howlett Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Joshua Hahn Cc: Rakie Kim Cc: Ying Huang Cc: Brendan Jackman Cc: Johannes Weiner --- mm/internal.h | 6 ++++-- mm/memory-tiers.c | 7 ++++--- mm/page_alloc.c | 13 ++++++++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 0dca33db068f..05179c4b2090 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1124,7 +1124,8 @@ extern int node_reclaim_mode; extern unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order); -extern int find_next_best_node(int node, nodemask_t *used_node_mask); +int find_next_best_node_in(int node, nodemask_t *used_node_mask, + const nodemask_t *candidates); #else #define node_reclaim_mode 0 @@ -1133,7 +1134,8 @@ static inline unsigned long node_reclaim(struct pglist_data *pgdat, { return 0; } -static inline int find_next_best_node(int node, nodemask_t *used_node_mask) +static inline int find_next_best_node_in(int node, nodemask_t *used_node_mask, + const nodemask_t *candidates) { return NUMA_NO_NODE; } diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 54851d8a195b..25e121851b58 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -370,7 +370,7 @@ int next_demotion_node(int node, const nodemask_t *allowed_mask) * closest demotion target. */ nodes_complement(mask, *allowed_mask); - return find_next_best_node(node, &mask); + return find_next_best_node_in(node, &mask, &node_states[N_MEMORY]); } static void disable_all_demotion_targets(void) @@ -450,7 +450,7 @@ static void establish_demotion_targets(void) memtier = list_next_entry(memtier, list); tier_nodes = get_memtier_nodemask(memtier); /* - * find_next_best_node, use 'used' nodemask as a skip list. + * find_next_best_node_in, use 'used' nodemask as a skip list. * Add all memory nodes except the selected memory tier * nodelist to skip list so that we find the best node from the * memtier nodelist. @@ -463,7 +463,8 @@ static void establish_demotion_targets(void) * in the preferred mask when allocating pages during demotion. */ do { - target = find_next_best_node(node, &tier_nodes); + target = find_next_best_node_in(node, &tier_nodes, + &node_states[N_MEMORY]); if (target == NUMA_NO_NODE) break; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index cfafc9be9e30..2ba0f540ee43 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5781,9 +5781,10 @@ static int numa_zonelist_order_handler(const struct ctl_table *table, int write, static int node_load[MAX_NUMNODES]; /** - * find_next_best_node - find the next node that should appear in a given node's fallback list + * find_next_best_node_in - find the next node that should appear in a given node's fallback list * @node: node whose fallback list we're appending * @used_node_mask: nodemask_t of already used nodes + * @candidates: nodemask_t of nodes eligible for selection * * We use a number of factors to determine which is the next node that should * appear on a given node's fallback list. The node should not have appeared @@ -5795,7 +5796,8 @@ static int node_load[MAX_NUMNODES]; * * Return: node id of the found node or %NUMA_NO_NODE if no node is found. */ -int find_next_best_node(int node, nodemask_t *used_node_mask) +int find_next_best_node_in(int node, nodemask_t *used_node_mask, + const nodemask_t *candidates) { int n, val; int min_val = INT_MAX; @@ -5805,12 +5807,12 @@ int find_next_best_node(int node, nodemask_t *used_node_mask) * Use the local node if we haven't already, but for memoryless local * node, we should skip it and fall back to other nodes. */ - if (!node_isset(node, *used_node_mask) && node_state(node, N_MEMORY)) { + if (!node_isset(node, *used_node_mask) && node_isset(node, *candidates)) { node_set(node, *used_node_mask); return node; } - for_each_node_state(n, N_MEMORY) { + for_each_node_mask(n, *candidates) { /* Don't want a node to appear more than once */ if (node_isset(n, *used_node_mask)) @@ -5895,7 +5897,8 @@ static void build_zonelists(pg_data_t *pgdat) prev_node = local_node; memset(node_order, 0, sizeof(node_order)); - while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { + while ((node = find_next_best_node_in(local_node, &used_mask, + &node_states[N_MEMORY])) >= 0) { /* * We don't want to pressure a particular node. * So adding penalty to the first node in same From d82701f6266abfcda931ce37ed98560945abf498 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Tue, 22 Sep 2026 22:29:02 -0400 Subject: [PATCH 375/539] mm/page_alloc: refactor build_node_zonelist() out of build_zonelists() Extract per-node fallback-list construction into build_node_zonelist(). Build each selected node directly into the destination zonelist so no intermediate node_order array or node count is needed. Print the fallback order as each node is added. This lets us build new zonelists from candidate nodemasks instead of just the default N_MEMORY node state list. Add a zlidx argument so callers explicitly select the destination zonelist. The existing caller continues to use ZONELIST_FALLBACK. No functional change: build_zonelists() builds and prints the same FALLBACK list over N_MEMORY with node_load updates as before. Link: https://lore.kernel.org/20260923022902.2433614-3-gourry@gourry.net Signed-off-by: Gregory Price Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zenghui Yu (Huawei) Cc: David Hildenbrand Cc: Lorenzo Stoakes Cc: Liam R. Howlett Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Joshua Hahn Cc: Rakie Kim Cc: Ying Huang Cc: Brendan Jackman Cc: Johannes Weiner --- mm/page_alloc.c | 63 ++++++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2ba0f540ee43..b261445519e8 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5845,31 +5845,6 @@ int find_next_best_node_in(int node, nodemask_t *used_node_mask, } -/* - * Build zonelists ordered by node and zones within node. - * This results in maximum locality--normal zone overflows into local - * DMA zone, if any--but risks exhausting DMA zone. - */ -static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, - unsigned nr_nodes) -{ - struct zoneref *zonerefs; - int i; - - zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; - - for (i = 0; i < nr_nodes; i++) { - int nr_zones; - - pg_data_t *node = NODE_DATA(node_order[i]); - - nr_zones = build_zonerefs_node(node, zonerefs); - zonerefs += nr_zones; - } - zonerefs->zone = NULL; - zonerefs->zone_idx = 0; -} - /* * Build __GFP_THISNODE zonelists */ @@ -5885,20 +5860,24 @@ static void build_thisnode_zonelists(pg_data_t *pgdat) zonerefs->zone_idx = 0; } -static void build_zonelists(pg_data_t *pgdat) +/* + * Build one zonelist ordered by node and zones within node. This results in + * maximum locality--normal zone overflows into local DMA zone, if any--but + * risks exhausting DMA zone. + */ +static void build_node_zonelist(pg_data_t *pgdat, const nodemask_t *candidates, + int zlidx) { - static int node_order[MAX_NUMNODES]; - int node, nr_nodes = 0; + struct zoneref *zonerefs = pgdat->node_zonelists[zlidx]._zonerefs; nodemask_t used_mask = NODE_MASK_NONE; - int local_node, prev_node; + int local_node = pgdat->node_id; + int prev_node = local_node; + int node; - /* NUMA-aware ordering of nodes */ - local_node = pgdat->node_id; - prev_node = local_node; + pr_info("Fallback order for Node %d: ", local_node); - memset(node_order, 0, sizeof(node_order)); while ((node = find_next_best_node_in(local_node, &used_mask, - &node_states[N_MEMORY])) >= 0) { + candidates)) >= 0) { /* * We don't want to pressure a particular node. * So adding penalty to the first node in same @@ -5908,18 +5887,22 @@ static void build_zonelists(pg_data_t *pgdat) node_distance(local_node, prev_node)) node_load[node] += 1; - node_order[nr_nodes++] = node; + zonerefs += build_zonerefs_node(NODE_DATA(node), zonerefs); + pr_cont("%d ", node); prev_node = node; } - build_zonelists_in_node_order(pgdat, node_order, nr_nodes); - build_thisnode_zonelists(pgdat); - pr_info("Fallback order for Node %d: ", local_node); - for (node = 0; node < nr_nodes; node++) - pr_cont("%d ", node_order[node]); + zonerefs->zone = NULL; + zonerefs->zone_idx = 0; pr_cont("\n"); } +static void build_zonelists(pg_data_t *pgdat) +{ + build_node_zonelist(pgdat, &node_states[N_MEMORY], ZONELIST_FALLBACK); + build_thisnode_zonelists(pgdat); +} + #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* * Return node id of node used for "local" allocations. From c3f25ea70432ac917e1881b08e62732f3cf33f97 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 11 Sep 2026 18:14:41 -0400 Subject: [PATCH 376/539] compiler.h: add ASSERT_STATIC_STORAGE() Patch series "Catch automatic storage in IDA and Maple Tree definitions". A 0day report [1] from the region allocation benchmark exposed a lockdep initialization bug: a stack-local Maple Tree used MTREE_INIT(), whose embedded lock has a static initializer. On the first allocation, lockdep rejected the lock address as a non-static class key and disabled locking validation. The IDA benchmark had the same issue, masked because it ran after Maple Tree had already disabled lockdep. The fix [2] switches the test to using mt_init_flags() and ida_init(). This series adds a compile-time check to the related DEFINE_IDA() and DEFINE_MTREE() declaration macros to catch the same class of mistake earlier. Patch 1 introduces ASSERT_STATIC_STORAGE(). It declares an unused static pointer initialized with the object's address, requiring that address to be a valid static initializer. Patches 2 and 3 apply the helper to IDA and Maple Tree definitions, respectively. The helper is mirrored in the tools compiler header. The existing automatic local IDAs and Maple Trees in the userspace radix-tree tests are converted to runtime initialization. The interval-tree span test keeps its existing mt_init_flags() call and uses a plain Maple Tree declaration. For example, an automatic local definition: void example(void) { DEFINE_IDA(ida); ida_destroy(&ida); } now produces: error: initializer element is not constant note: in expansion of macro 'ASSERT_STATIC_STORAGE' note: in expansion of macro 'DEFINE_IDA' File-scope definitions and static local definitions remain valid. Automatic local objects should use ida_init(), mt_init(), or mt_init_flags(). The check is limited to declaration macros. Direct uses of IDA_INIT(), MTREE_INIT(), and MTREE_INIT_EXT() remain unchanged. The helper cannot be inserted directly into those initializer expressions because it expands to a declaration. Validated by GCC and Clang checks accepting static storage and rejecting automatic storage The userspace IDR/IDA and Maple Tree test are passed as well. This patch (of 3): Static lock initializers rely on a persistent object address when lockdep assigns a lock-class key. Using such an initializer for an automatic local object can compile successfully but disable lockdep on the first lock acquisition. Add ASSERT_STATIC_STORAGE() for declaration macros that require static storage duration. It declares an unused static pointer initialized with the object's address. An automatic local object's address is not a valid static initializer, so the compiler rejects it. Mirror the helper in tools/include/linux/compiler.h because the userspace radix-tree tests include the kernel IDA and Maple Tree headers with the tools compiler definitions. For example: void example(void) { int object; ASSERT_STATIC_STORAGE(object); } GCC reports: error: initializer element is not constant name##_storage_check = &(name) ^ note: in expansion of macro 'ASSERT_STATIC_STORAGE' ASSERT_STATIC_STORAGE(object); File-scope objects and static local objects remain valid. The helper takes an object identifier and must be used as a declaration after that object has been declared. Link: https://lore.kernel.org/all/20260911155244.1406122-1-ynorov@nvidia.com/ Link: https://lore.kernel.org/20260911221444.1523311-2-ynorov@nvidia.com Link: https://download.01.org/0day-ci/archive/20260910/202609101106.771b567e-lkp@intel.com/ [1] Link: https://lore.kernel.org/all/20260911155244.1406122-1-ynorov@nvidia.com/ [2] Signed-off-by: Yury Norov Signed-off-by: Andrew Morton Cc: Alice Ryhl Cc: Andrew Ballance Cc: Christopher Li Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) --- include/linux/compiler.h | 5 +++++ tools/include/linux/compiler.h | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index cb2f6050bdf7..ef9036fa5413 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -275,6 +275,11 @@ static inline void *offset_to_ptr(const int *off) #define __ADDRESSABLE(sym) \ ___ADDRESSABLE(sym, __section(".discard.addressable")) +/* Enforce static storage duration. */ +#define ASSERT_STATIC_STORAGE(name) \ + static typeof(name) * const __always_unused \ + name##_storage_check = &(name) + /* * This returns a constant expression while determining if an argument is * a constant expression, most importantly without evaluating the argument. diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h index f2f54b038168..03ecf9086643 100644 --- a/tools/include/linux/compiler.h +++ b/tools/include/linux/compiler.h @@ -73,6 +73,11 @@ # define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b)) #endif +/* Enforce static storage duration. */ +#define ASSERT_STATIC_STORAGE(name) \ + static typeof(name) * const __always_unused \ + name##_storage_check = &(name) + /* * This returns a constant expression while determining if an argument is * a constant expression, most importantly without evaluating the argument. From 2233228b65291f8e9b04ee8be52bd2f694e12889 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 11 Sep 2026 18:14:42 -0400 Subject: [PATCH 377/539] idr: assert static storage for DEFINE_IDA() DEFINE_IDA() uses IDA_INIT(), which initializes the embedded XArray lock with a static spinlock initializer. For an automatic local IDA, lockdep cannot use the lock address as a persistent class key and reports "INFO: trying to register non-static key" before disabling itself. Apply ASSERT_STATIC_STORAGE() to DEFINE_IDA() so that this misuse is rejected at compile time. For example: void example(void) { DEFINE_IDA(ida); ida_destroy(&ida); } GCC reports: error: initializer element is not constant name##_storage_check = &(name) ^ note: in expansion of macro 'ASSERT_STATIC_STORAGE' ASSERT_STATIC_STORAGE(name) note: in expansion of macro 'DEFINE_IDA' DEFINE_IDA(ida); File-scope definitions and static DEFINE_IDA() within a function remain valid. Automatic local IDAs must instead be initialized with ida_init(). Direct uses of IDA_INIT() are not covered by this declaration check. Convert the five automatic local IDAs in the userspace radix-tree tests to ida_init() so they satisfy the new requirement. Validated file-scope and static local definitions with a kernel object build, and confirmed that an automatic local definition fails to compile. Link: https://lore.kernel.org/20260911221444.1523311-3-ynorov@nvidia.com Signed-off-by: Yury Norov Signed-off-by: Andrew Morton Cc: Alice Ryhl Cc: Andrew Ballance Cc: Christopher Li Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) --- include/linux/idr.h | 5 ++++- tools/testing/radix-tree/idr-test.c | 20 +++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/include/linux/idr.h b/include/linux/idr.h index 789e23e67444..e2a4b6298511 100644 --- a/include/linux/idr.h +++ b/include/linux/idr.h @@ -16,6 +16,7 @@ #include #include #include +#include struct idr { struct radix_tree_root idr_rt; @@ -269,7 +270,9 @@ struct ida { #define IDA_INIT(name) { \ .xa = XARRAY_INIT(name, IDA_INIT_FLAGS) \ } -#define DEFINE_IDA(name) struct ida name = IDA_INIT(name) +#define DEFINE_IDA(name) \ + struct ida name = IDA_INIT(name); \ + ASSERT_STATIC_STORAGE(name) int ida_alloc_range(struct ida *, unsigned int min, unsigned int max, gfp_t); void ida_free(struct ida *, unsigned int id); diff --git a/tools/testing/radix-tree/idr-test.c b/tools/testing/radix-tree/idr-test.c index 945144e98507..6fcba5b5870b 100644 --- a/tools/testing/radix-tree/idr-test.c +++ b/tools/testing/radix-tree/idr-test.c @@ -460,9 +460,11 @@ void ida_dump(struct ida *); */ void ida_check_nomem(void) { - DEFINE_IDA(ida); + struct ida ida; int id; + ida_init(&ida); + id = ida_alloc_min(&ida, 256, GFP_NOWAIT); IDA_BUG_ON(&ida, id != -ENOMEM); id = ida_alloc_min(&ida, 1UL << 30, GFP_NOWAIT); @@ -475,9 +477,11 @@ void ida_check_nomem(void) */ void ida_check_conv_user(void) { - DEFINE_IDA(ida); + struct ida ida; unsigned long i; + ida_init(&ida); + for (i = 0; i < 1000000; i++) { int id = ida_alloc(&ida, GFP_NOWAIT); if (id == -ENOMEM) { @@ -496,11 +500,13 @@ void ida_check_conv_user(void) void ida_check_random(void) { - DEFINE_IDA(ida); + struct ida ida; DECLARE_BITMAP(bitmap, 2048); unsigned int i; time_t s = time(NULL); + ida_init(&ida); + repeat: memset(bitmap, 0, sizeof(bitmap)); for (i = 0; i < 100000; i++) { @@ -522,9 +528,11 @@ void ida_check_random(void) void ida_alloc_free_test(void) { - DEFINE_IDA(ida); + struct ida ida; unsigned long i; + ida_init(&ida); + for (i = 0; i < 10000; i++) assert(ida_alloc_max(&ida, 20000, GFP_KERNEL) == i); assert(ida_alloc_range(&ida, 5, 30, GFP_KERNEL) < 0); @@ -576,10 +584,12 @@ static void *ida_leak_fn(void *arg) void ida_thread_tests(void) { - DEFINE_IDA(ida); + struct ida ida; pthread_t threads[20]; int i; + ida_init(&ida); + for (i = 0; i < ARRAY_SIZE(threads); i++) if (pthread_create(&threads[i], NULL, ida_random_fn, NULL)) { perror("creating ida thread"); From 778be1d8b57f3e0385324bb9d21ef048d88eb88f Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Fri, 11 Sep 2026 18:14:43 -0400 Subject: [PATCH 378/539] maple_tree: assert static storage for DEFINE_MTREE() DEFINE_MTREE() uses MTREE_INIT(), which initializes the tree's embedded lock with a static spinlock initializer. If the tree is an automatic local object, lockdep rejects its address as a non-static class key and disables locking validation on the first lock acquisition. Apply ASSERT_STATIC_STORAGE() to DEFINE_MTREE() to catch automatic local definitions at compile time. For example: void example(void) { DEFINE_MTREE(mt); mtree_destroy(&mt); } GCC reports: error: initializer element is not constant name##_storage_check = &(name) ^ note: in expansion of macro 'ASSERT_STATIC_STORAGE' ASSERT_STATIC_STORAGE(name) note: in expansion of macro 'DEFINE_MTREE' DEFINE_MTREE(mt); File-scope definitions and static local trees remain valid. Automatic local trees must instead use mt_init() or mt_init_flags(). Direct uses of MTREE_INIT() and MTREE_INIT_EXT() are unchanged. Replace the local DEFINE_MTREE() in the interval-tree span test with a plain declaration; the test already initializes the tree with mt_init_flags() before use. Convert the three local Maple Trees in the userspace radix-tree tests to mt_init(). Validated file-scope and static local definitions with a kernel object build, and confirmed that an automatic local definition fails to compile. The Maple Tree test and region allocation benchmark objects also build with lockdep enabled. Link: https://lore.kernel.org/20260911221444.1523311-4-ynorov@nvidia.com Signed-off-by: Yury Norov Signed-off-by: Andrew Morton Cc: Alice Ryhl Cc: Andrew Ballance Cc: Christopher Li Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) --- include/linux/maple_tree.h | 4 +++- lib/interval_tree_test.c | 2 +- tools/testing/radix-tree/maple.c | 12 +++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index e595ae5cd0ee..e30e57f5f3df 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -9,6 +9,7 @@ */ #include +#include #include #include @@ -297,7 +298,8 @@ struct maple_tree { #endif #define DEFINE_MTREE(name) \ - struct maple_tree name = MTREE_INIT(name, 0) + struct maple_tree name = MTREE_INIT(name, 0); \ + ASSERT_STATIC_STORAGE(name) #define mtree_lock(mt) spin_lock((&(mt)->ma_lock)) #define mtree_lock_nested(mas, subclass) \ diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index b0b07270ce7c..06f77fb3179f 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -244,7 +244,7 @@ static int span_iteration_check(void) unsigned long start, last; struct interval_tree_span_iter span, mas_span; - DEFINE_MTREE(tree); + struct maple_tree tree; MA_STATE(mas, &tree, 0, 0); diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c index d967e76a3c06..bfe4b8626c42 100644 --- a/tools/testing/radix-tree/maple.c +++ b/tools/testing/radix-tree/maple.c @@ -36022,10 +36022,12 @@ static noinline void __init check_erase_rebalance(struct maple_tree *mt) static noinline void __init check_mtree_dup(struct maple_tree *mt) { - DEFINE_MTREE(new); + struct maple_tree new; int i, j, ret, count = 0; unsigned int rand_seed = 17, rand; + mt_init(&new); + /* store a value at [0, 0] */ mt_init_flags(mt, 0); mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL); @@ -36319,7 +36321,9 @@ static inline int check_vma_modification(struct maple_tree *mt) void farmer_tests(void) { struct maple_node *node; - DEFINE_MTREE(tree); + struct maple_tree tree; + + mt_init(&tree); mt_dump(&tree, mt_dump_dec); @@ -36432,9 +36436,11 @@ static unsigned long get_last_index(struct ma_state *mas) static void test_spanning_store_regression(void) { unsigned long from = 0, to = 0; - DEFINE_MTREE(tree); + struct maple_tree tree; MA_STATE(mas, &tree, 0, 0); + mt_init(&tree); + /* * Build a 3-level tree. We require a parent node below the root node * and 2 leaf nodes under it, so we can span the entirety of the right From fcf93a839f8047dd0468c0205f96e881ff8c67c2 Mon Sep 17 00:00:00 2001 From: Yeoreum Yun Date: Fri, 11 Sep 2026 22:06:11 +0100 Subject: [PATCH 379/539] kselftest: mm: remove exclusion of building soft-dirty test in arm64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 0389c305ef56 (“selftests/mm: skip soft-dirty tests when CONFIG_MEM_SOFT_DIRTY is disabled”), the soft-dirty test is skipped when soft-dirty is not supported. There is therefore no reason to exclude the test from being built on arm64. Remove the arm64-specific exclusion. Link: https://lore.kernel.org/20260911210611.4001419-1-yeoreum.yun@arm.com Signed-off-by: Yeoreum Yun Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand (Arm) Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes (ARM) Tested-by: Zenghui Yu (Huawei) Cc: Alice Ryhl Cc: Andrew Ballance Cc: Christopher Li Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Yury Norov (NVIDIA) --- tools/testing/selftests/mm/Makefile | 3 --- tools/testing/selftests/mm/run_vmtests.sh | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 2d5366196e30..d3e9bd67904a 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -104,10 +104,7 @@ TEST_GEN_FILES += guard-regions TEST_GEN_FILES += merge TEST_GEN_FILES += rmap TEST_GEN_FILES += folio_split_race_test - -ifneq ($(ARCH),arm64) TEST_GEN_FILES += soft-dirty -endif ifeq ($(ARCH),x86_64) CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_32bit_program.c -m32) diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 2e5e7975ff4d..a1db51655702 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -408,10 +408,7 @@ then CATEGORY="pkey" run_test ./protection_keys_64 fi -if [ -x ./soft-dirty ] -then - CATEGORY="soft_dirty" run_test ./soft-dirty -fi +CATEGORY="soft_dirty" run_test ./soft-dirty CATEGORY="pagemap" run_test ./pagemap_ioctl From 9b20271285898e993c6a540aae6291af755da05a Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Sun, 13 Sep 2026 14:30:31 +0800 Subject: [PATCH 380/539] mm: zswap: return -ENOENT when the swap device is gone zswap_writeback_entry() returns -EEXIST when get_swap_device() finds no device. -EEXIST is the shrinker's "page already in swap cache" signal, which makes zswap_shrinker_scan() stop shrinking entirely. A NULL get_swap_device() instead means the device is being swapped off, so the entry is simply stale. Return -ENOENT so the shrinker skips the stale entry and keeps scanning. It affects all swap devices. Link: https://lore.kernel.org/20260913063031.1689420-1-hebaoquan@kylinos.cn Signed-off-by: Baoquan He Signed-off-by: Andrew Morton Acked-by: Nhat Pham Cc: Chengming Zhou Cc: Chris Li Cc: Johannes Weiner Cc: Kairui Song --- mm/zswap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/zswap.c b/mm/zswap.c index ff7c6742af50..584dd3063769 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1016,7 +1016,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, /* try to allocate swap cache folio */ si = get_swap_device(swpentry); if (IS_ERR_OR_NULL(si)) - return -EEXIST; + return -ENOENT; mpol = get_task_policy(current); folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol, From c315da11e3272d5026b613b933a48016f3af3a18 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:27:57 -0400 Subject: [PATCH 381/539] mm/zsmalloc: replace PG_private with pointer comparison Patch series "Remove PG_private by using page/folio->private checks instead", v5. This patchset removes PG_private to make space for upcoming PG_folio for identifying pages from a folio (more details in Note below). Instead of checking PG_private, all code is changed to check page/folio->private != NULL instead. Overview === Most code uses folio_attach/detach/change_private() functions, so folio refcount is increased and decreased when folio->private is set and reset, respectively. There is no need to change them. Changes are needed for exceptional users: 1. zsmalloc uses PG_private to indicate first component zpdesc page and page->private is used to store zspage in zpdesc. To remove PG_private, is_first_zpdesc() is replaced by pointer comparison. 2. kernel/events/ring_buffer.c stores page order in page->private. Replacing PG_private with page->private != NULL works. 3. drivers/xen/grant-table.c stores xen_page_foreign in page->private, where on 32-bit, a pointer to xen_page_foreign is stored; on 64-bit, page->private is used as xen_page_foreign. PG_private check is replaced by page->private != NULL on 32-bit for xen_page_foreign deallocation. On 64-bit, page->private is cleared unconditionally since {domid=0, gref=0} (xen_page_foreign can be 0) is valid. 4. fs/crypto/crypto.c stores a folio pointer in page->private, PG_private checks are replaced by page->private != NULL. 5. fs/erofs has two different uses: 5a. folio->private is used to form a reversed list of the outputs of readahead_folio(). readahead_folio_last() is added to output folios in reversed order, so that ->private is no longer needed. 5b. folio->private is used as an in-flight I/O counter. Convert the code to use folio_attach/detach/get_private() and add bias==1 to the counter to avoid folio->private being zero. 6. fs/nfs/write.c: folio refcount maintenance is in a bigger scope than folio->private. So folio_attach/detach/get_private() is not used. Nothing to change. 7. fs/f2fs uses attach_page_private() to first reset folio->private then immediately sets PAGE_PRIVATE_NOT_POINTER bit on it. Change it to use attach_page_private() to set PAGE_PRIVATE_NOT_POINTER bit directly to avoid folio->private == NULL gap inside set_page_private_##name(). 8. hugetlb uses folio_change_private(folio, NULL) without folio refcount maintenance. Change it to folio->private = NULL. After the above changes, PG_private ops are converted to page/folio->private ops. folio_has_attached_private() is added to check filesystem-only private data by excluding swapcache and hugetlb folios, because swapcache folios overlap swp_entry_t swap with ->private and hugetlb sets its own flags in ->private. Note === 1. KPF_PRIVATE is removed after PG_private is removed. 2. Documentation/mm/hugetlbfs_reserv.rst is outdated, so I did not remove PG_private related text. It should be rewritten. 3. PG_folio is planned to be set on every page from a folio in page_rmappable_folio(), so folios with any order (currently PG_large_rmappable is used to identify >0 order folios, but not order-0 folios) can be identified. Then vm_insert_*() can correctly reject all folios and rmap code will only see folios. Eventually, page_folio() will return NULL for non-folio pages by checking PG_folio, but before that all existing users that treat compound pages as folios will need to be converted. This patch (of 17): zsmalloc uses PG_private to indicate first zpdesc in a zspage chain. It is equivalent to check zpdesc == zspage->first_zpdesc. Replace is_first_zpdesc() with zpdesc == zspage->first_zpdesc in obj_allocated(). For get_first_zpdesc(), first_zpdesc is from zspage->first_zpdesc, so replace is_first_zpdesc() with first_zpdesc->zspage == zspage, the second requirement of a zspage chain, where all zpdescs point to the same zspage. is_first_zpdesc(), is only used in VM_BUG_ON_PAGE(), so performance impact should be negligible. While at it, change VM_BUG_ON() to VM_WARN_ON_ONCE_PAGE(). It prepares for a future commit that remove PG_private. No functional change intended. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-0-bb68b6a21869@nvidia.com Link: https://lore.kernel.org/20260920-remove-pg_private-v5-1-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Reviewed-by: Sergey Senozhatsky Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Assisted-by: LLM Cc: Minchan Kim --- mm/zpdesc.h | 2 +- mm/zsmalloc.c | 24 ++++++------------------ 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/mm/zpdesc.h b/mm/zpdesc.h index b8258dc78548..4fd81c2e8076 100644 --- a/mm/zpdesc.h +++ b/mm/zpdesc.h @@ -26,8 +26,8 @@ * with memcg_data. * * Page flags used: - * * PG_private identifies the first component page. * * PG_locked is used by page migration code. + * The first component page has zpdesc->zspage->first_zpdesc == zpdesc */ struct zpdesc { unsigned long flags; diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 11be37c43171..7ef80e0da626 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -290,11 +290,6 @@ struct zs_pool { atomic_t compaction_in_progress; }; -static inline void zpdesc_set_first(struct zpdesc *zpdesc) -{ - SetPagePrivate(zpdesc_page(zpdesc)); -} - static inline void zpdesc_inc_zone_page_state(struct zpdesc *zpdesc) { inc_zone_page_state(zpdesc_page(zpdesc), NR_ZSPAGES); @@ -476,11 +471,6 @@ static void record_obj(unsigned long handle, unsigned long obj) WRITE_ONCE(*(unsigned long *)handle, obj); } -static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc) -{ - return PagePrivate(zpdesc_page(zpdesc)); -} - /* Protected by class->lock */ static inline int get_zspage_inuse(struct zspage *zspage) { @@ -496,7 +486,8 @@ static struct zpdesc *get_first_zpdesc(struct zspage *zspage) { struct zpdesc *first_zpdesc = zspage->first_zpdesc; - VM_BUG_ON_PAGE(!is_first_zpdesc(first_zpdesc), zpdesc_page(first_zpdesc)); + /* the first zpdesc must point back to this zspage */ + VM_WARN_ON_ONCE_PAGE(first_zpdesc->zspage != zspage, zpdesc_page(first_zpdesc)); return first_zpdesc; } @@ -838,7 +829,8 @@ static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj, struct zspage *zspage = get_zspage(zpdesc); if (unlikely(ZsHugePage(zspage))) { - VM_BUG_ON_PAGE(!is_first_zpdesc(zpdesc), zpdesc_page(zpdesc)); + /* only first zpdesc holds the handle */ + VM_WARN_ON_ONCE_PAGE(zspage->first_zpdesc != zpdesc, zpdesc_page(zpdesc)); handle = zpdesc->handle; } else handle = *(unsigned long *)obj; @@ -853,9 +845,6 @@ static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj, static void reset_zpdesc(struct zpdesc *zpdesc) { - struct page *page = zpdesc_page(zpdesc); - - ClearPagePrivate(page); zpdesc->zspage = NULL; zpdesc->next = NULL; /* PageZsmalloc is sticky until the page is freed to the buddy. */ @@ -1006,8 +995,8 @@ static void create_page_chain(struct size_class *class, struct zspage *zspage, * 1. all pages are linked together using zpdesc->next * 2. each sub-page point to zspage using zpdesc->zspage * - * we set PG_private to identify the first zpdesc (i.e. no other zpdesc - * has this flag set). + * The first zpdesc has its zspage->first_zpdesc set to itself, no + * other zpdesc has this set. */ for (i = 0; i < nr_zpdescs; i++) { zpdesc = zpdescs[i]; @@ -1015,7 +1004,6 @@ static void create_page_chain(struct size_class *class, struct zspage *zspage, zpdesc->next = NULL; if (i == 0) { zspage->first_zpdesc = zpdesc; - zpdesc_set_first(zpdesc); if (unlikely(class->objs_per_zspage == 1 && class->pages_per_zspage == 1)) SetZsHugePage(zspage); From a0319334a9cc22d408007b444305394d20a553de Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:27:58 -0400 Subject: [PATCH 382/539] perf/ring_buffer: stop using PG_private as AUX page high-order marker A high-order AUX page sets PG_private on its first page and stores the order in first_page->private. Stop using PG_private and check first_page->private for AUX page order only in the ring buffer and its users. This is fine because page->private is 0 for order-0 AUX pages, matching the page order. It prepares for a future commit that remove PG_private. No functional change intended. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-2-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: Usama Arif Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Namhyung Kim Cc: Thomas Gleixner Cc: Borislav Petkov Cc: Dave Hansen Cc: Mark Rutland Cc: Alexander Shishkin Cc: Jiri Olsa Cc: Ian Rogers Cc: Adrian Hunter Cc: James Clark Cc: "H. Peter Anvin" --- arch/x86/events/intel/bts.c | 3 --- arch/x86/events/intel/pt.c | 6 ++---- kernel/events/ring_buffer.c | 7 +++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/arch/x86/events/intel/bts.c b/arch/x86/events/intel/bts.c index cbac54cb3a9e..5849392cf26d 100644 --- a/arch/x86/events/intel/bts.c +++ b/arch/x86/events/intel/bts.c @@ -66,9 +66,6 @@ static struct pmu bts_pmu; static int buf_nr_pages(struct page *page) { - if (!PagePrivate(page)) - return 1; - return 1 << page_private(page); } diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index 5754cd405562..49349afee611 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -781,8 +781,7 @@ static int topa_insert_pages(struct pt_buffer *buf, int cpu, gfp_t gfp) struct page *p; p = virt_to_page(buf->data_pages[buf->nr_pages]); - if (PagePrivate(p)) - order = page_private(p); + order = page_private(p); if (topa_table_full(topa)) { topa = topa_alloc(cpu, gfp); @@ -1296,8 +1295,7 @@ static int pt_buffer_try_single(struct pt_buffer *buf, int nr_pages) if (!intel_pt_validate_hw_cap(PT_CAP_single_range_output)) goto out; - if (PagePrivate(p)) - order = page_private(p); + order = page_private(p); if (1 << order != nr_pages) goto out; diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 1b1ffe0533e5..9d3d324f5127 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -635,11 +635,10 @@ static struct page *rb_alloc_aux_page(int node, int order) /* * Communicate the allocation size to the driver: * if we managed to secure a high-order allocation, - * set its first page's private to this order; - * !PagePrivate(page) means it's just a normal page. + * set its first page's private to this order, otherwise page's + * private remains zero. */ split_page(page, order); - SetPagePrivate(page); set_page_private(page, order); } @@ -650,7 +649,7 @@ static void rb_free_aux_page(struct perf_buffer *rb, int idx) { struct page *page = virt_to_page(rb->aux_pages[idx]); - ClearPagePrivate(page); + set_page_private(page, 0); __free_page(page); } From 6eecb67b69af634bbf27279dbef497580d20f9dd Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:27:59 -0400 Subject: [PATCH 383/539] xen/grant-table: stop setting PG_private on pages for grant mapping gnttab_alloc_pages() stores xen_page_foreign in page->private. On 32-bit, a pointer to an allocated xen_page_foreign is stored; on 64-bit, xen_page_forCcgn is stored inline. Checking page->private != NULL is enough to tell whether a xen_page_foreign needs to be freed on 32-bit and page->private is zeroed unconditionally on 64-bit. It prepares for a future commit that remove PG_private. No functional change intended. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-3-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Juergen Gross Cc: Stefano Stabellini Cc: Oleksandr Tyshchenko --- drivers/xen/grant-table.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index 076c1b0ab87f..f75b5cc71cfc 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c @@ -863,10 +863,10 @@ EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames); int gnttab_pages_set_private(int nr_pages, struct page **pages) { +#if BITS_PER_LONG < 64 int i; for (i = 0; i < nr_pages; i++) { -#if BITS_PER_LONG < 64 struct xen_page_foreign *foreign; foreign = kzalloc_obj(*foreign); @@ -874,9 +874,9 @@ int gnttab_pages_set_private(int nr_pages, struct page **pages) return -ENOMEM; set_page_private(pages[i], (unsigned long)foreign); -#endif - SetPagePrivate(pages[i]); } +#endif + /* Data is stored in page->private on 64-bit */ return 0; } @@ -1031,12 +1031,11 @@ void gnttab_pages_clear_private(int nr_pages, struct page **pages) int i; for (i = 0; i < nr_pages; i++) { - if (PagePrivate(pages[i])) { #if BITS_PER_LONG < 64 + if (page_private(pages[i])) kfree((void *)page_private(pages[i])); #endif - ClearPagePrivate(pages[i]); - } + set_page_private(pages[i], 0); } } EXPORT_SYMBOL_GPL(gnttab_pages_clear_private); From 70b4a9cc27f7140e14e73f61c2a053ffc18c4218 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:00 -0400 Subject: [PATCH 384/539] fscrypt: stop setting PG_private on bounce page The pointer to a plain text folio is stored in bound_page->private and cannot be NULL until the bounce_page is freed, making PG_private redundant. It prepares for a future commit that remove PG_private. No functional change intended. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-4-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: Eric Biggers Acked-by: Usama Arif Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Assisted-by: LLM Cc: "Theodore Y. Ts'o" Cc: Jaegeuk Kim --- fs/crypto/crypto.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 5286a124b0d9..aced5c50a460 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -65,7 +65,6 @@ void fscrypt_free_bounce_page(struct page *bounce_page) if (!bounce_page) return; set_page_private(bounce_page, (unsigned long)NULL); - ClearPagePrivate(bounce_page); mempool_free(bounce_page, fscrypt_bounce_page_pool); } EXPORT_SYMBOL(fscrypt_free_bounce_page); @@ -210,7 +209,6 @@ struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio, return ERR_PTR(err); } } - SetPagePrivate(ciphertext_page); set_page_private(ciphertext_page, (unsigned long)folio); return ciphertext_page; } From 36c9ce320c3cf0f415d11c173900c30db0ac883d Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:01 -0400 Subject: [PATCH 385/539] mm/hugetlb: use direct assignment instead of folio_change_private() folio_change_private() should be used along with folio_attach_private() and folio_detach_private(), where adding and remove ->private content requires folio refcount change. add_hugetlb_folio() simply sets folio->private to NULL without refcount manipulation. Change it to direct assignment to avoid semantic confusion. It prepares for a future commit that remove PG_private. No functional change intended. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-5-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: Usama Arif Reviewed-by: Gregory Price (Meta) Reviewed-by: Muchun Song Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Assisted-by: LLM Cc: Oscar Salvador --- mm/hugetlb.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 3403ce546606..e4c45698adba 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1446,11 +1446,8 @@ void add_hugetlb_folio(struct hstate *h, struct folio *folio, } __folio_set_hugetlb(folio); - folio_change_private(folio, NULL); - /* - * We have to set hugetlb_vmemmap_optimized again as above - * folio_change_private(folio, NULL) cleared it. - */ + /* Clear all folio->private flags except hugetlb_vmemmap_optimized. */ + folio->private = NULL; folio_set_hugetlb_vmemmap_optimized(folio); arch_clear_hugetlb_flags(folio); From b2c1a445f87a553f04ccaefd4e700c7e404a70e5 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:02 -0400 Subject: [PATCH 386/539] f2fs: stop using PG_private f2fs sets its PAGE_PRIVATE_* flags in page->private and checking page->private != NULL is equivalent to checking PG_private. Change PagePrivate() to page_private(). Meanwhile, in set_page_private_##name(), page->private is first set to 0/NULL before an PAGE_PRIVATE_* flag is set, but it can cause confusion when PG_private is removed and page->private != NULL is used instead. Change it to initialize page->private to PAGE_PRIVATE_NOT_POINTER instead and retain the original semantics. It prepares for a future commit that removes PG_private. No functional change intended. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-6-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: Usama Arif Acked-by: Chao Yu Reviewed-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Jaegeuk Kim --- fs/f2fs/f2fs.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 9940a6cecf1a..2f7ab5888b07 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -2691,7 +2691,7 @@ static inline bool folio_test_f2fs_##name(const struct folio *folio) \ } \ static inline bool page_private_##name(struct page *page) \ { \ - return PagePrivate(page) && \ + return page_private(page) && \ test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \ test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ } @@ -2710,9 +2710,9 @@ static inline void folio_set_f2fs_##name(struct folio *folio) \ } \ static inline void set_page_private_##name(struct page *page) \ { \ - if (!PagePrivate(page)) \ - attach_page_private(page, (void *)0); \ - set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \ + if (!page_private(page)) \ + attach_page_private(page, \ + (void *)BIT(PAGE_PRIVATE_NOT_POINTER)); \ set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ } From 7e907079f9a9220448d1c29f8da69dac139b26a1 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:03 -0400 Subject: [PATCH 387/539] f2fs: convert the ->private flag helpers to folio-only page-based ->private flag helpers are used in the compression path, where large folios are not enabled. They can use folio versions with page_folio(). The two remaining users in data.c and segment.c can use fio->folio instead of fio->page (two are in a union). Drop page-based helpers after the conversion and rename PAGE_PRIVATE_{GET,SET,CLEAR}_FUNC() and the PAGE_PRIVATE_* flags to F2FS_FOLIO_PRIVATE_* to match. Convert the folio/page union from f2fs_io_info union to folio only, since no page user is left. The folio helpers do a plain read-modify-write where the page ones used set_bit()/clear_bit(). It is fine because the converted code either holds folio lock or, in f2fs_compress_write_end_io(), matches what the non-compressed code does in f2fs_write_end_bio(). Link: https://lore.kernel.org/20260920-remove-pg_private-v5-7-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Co-developed-by: David Hildenbrand (Arm) Signed-off-by: David Hildenbrand (Arm) Signed-off-by: Andrew Morton Suggested-by: Tal Zussman Reviewed-by: Tal Zussman Reviewed-by: Lance Yang Reviewed-by: Chao Yu Assisted-by: LLM Cc: Jaegeuk Kim --- fs/f2fs/compress.c | 35 ++++++++++------ fs/f2fs/data.c | 2 +- fs/f2fs/f2fs.h | 99 ++++++++++++++++++---------------------------- fs/f2fs/segment.c | 2 +- 4 files changed, 63 insertions(+), 75 deletions(-) diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index ce88092d9ce2..09d9b8d0fdcc 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1064,13 +1064,15 @@ static void cancel_cluster_writeback(struct compress_ctx *cc, /* Cancel writeback and stay locked. */ for (i = 0; i < cc->cluster_size; i++) { + struct folio *folio = page_folio(cc->rpages[i]); + if (i < submitted) { inode_inc_dirty_pages(cc->inode); - lock_page(cc->rpages[i]); + folio_lock(folio); } - clear_page_private_gcing(cc->rpages[i]); - if (folio_test_writeback(page_folio(cc->rpages[i]))) - end_page_writeback(cc->rpages[i]); + folio_clear_f2fs_gcing(folio); + if (folio_test_writeback(folio)) + folio_end_writeback(folio); } } @@ -1078,11 +1080,15 @@ static void set_cluster_dirty(struct compress_ctx *cc) { int i; - for (i = 0; i < cc->cluster_size; i++) - if (cc->rpages[i]) { - set_page_dirty(cc->rpages[i]); - set_page_private_gcing(cc->rpages[i]); - } + for (i = 0; i < cc->cluster_size; i++) { + struct folio *folio; + + if (!cc->rpages[i]) + continue; + folio = page_folio(cc->rpages[i]); + folio_mark_dirty(folio); + folio_set_f2fs_gcing(folio); + } } static int prepare_compress_overwrite(struct compress_ctx *cc, @@ -1281,7 +1287,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, .op = REQ_OP_WRITE, .op_flags = wbc_to_write_flags(wbc), .old_blkaddr = NEW_ADDR, - .page = NULL, + .folio = NULL, .encrypted_page = NULL, .compressed_page = NULL, .io_type = io_type, @@ -1370,7 +1376,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, block_t blkaddr; blkaddr = f2fs_data_blkaddr(&dn); - fio.page = cc->rpages[i]; + fio.folio = page_folio(cc->rpages[i]); fio.old_blkaddr = blkaddr; /* cluster header */ @@ -1476,9 +1482,12 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) } for (i = 0; i < cic->nr_rpages; i++) { + struct folio *rfolio; + WARN_ON(!cic->rpages[i]); - clear_page_private_gcing(cic->rpages[i]); - end_page_writeback(cic->rpages[i]); + rfolio = page_folio(cic->rpages[i]); + folio_clear_f2fs_gcing(rfolio); + folio_end_writeback(rfolio); } page_array_free(sbi, cic->rpages, cic->nr_rpages); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 21f396ebe22c..ca8232a9095f 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2923,7 +2923,7 @@ bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) return true; if (fio) { - if (page_private_gcing(fio->page)) + if (folio_test_f2fs_gcing(fio->folio)) return true; if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 2f7ab5888b07..85937de3d701 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1357,10 +1357,7 @@ struct f2fs_io_info { blk_opf_t op_flags; /* req_flag_bits */ block_t new_blkaddr; /* new block address to be written */ block_t old_blkaddr; /* old block address before Cow */ - union { - struct page *page; /* page to be written */ - struct folio *folio; - }; + struct folio *folio; /* folio to be written */ struct page *encrypted_page; /* encrypted page */ struct page *compressed_page; /* compressed page */ struct list_head list; /* serialize IOs */ @@ -1613,27 +1610,27 @@ static inline void f2fs_set_bit(unsigned int nr, char *addr); static inline void f2fs_clear_bit(unsigned int nr, char *addr); /* - * Layout of f2fs page.private: + * Layout of f2fs folio->private: * * Layout A: lowest bit should be 1 * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... | - * bit 0 PAGE_PRIVATE_NOT_POINTER - * bit 1 PAGE_PRIVATE_ONGOING_MIGRATION - * bit 2 PAGE_PRIVATE_INLINE_INODE - * bit 3 PAGE_PRIVATE_REF_RESOURCE - * bit 4 PAGE_PRIVATE_ATOMIC_WRITE + * bit 0 F2FS_FOLIO_PRIVATE_NOT_POINTER + * bit 1 F2FS_FOLIO_PRIVATE_ONGOING_MIGRATION + * bit 2 F2FS_FOLIO_PRIVATE_INLINE_INODE + * bit 3 F2FS_FOLIO_PRIVATE_REF_RESOURCE + * bit 4 F2FS_FOLIO_PRIVATE_ATOMIC_WRITE * bit 5- f2fs private data * * Layout B: lowest bit should be 0 - * page.private is a wrapped pointer. + * folio->private is a wrapped pointer. */ enum { - PAGE_PRIVATE_NOT_POINTER, /* private contains non-pointer data */ - PAGE_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */ - PAGE_PRIVATE_INLINE_INODE, /* inode page contains inline data */ - PAGE_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */ - PAGE_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */ - PAGE_PRIVATE_MAX + F2FS_FOLIO_PRIVATE_NOT_POINTER, /* private contains non-pointer data */ + F2FS_FOLIO_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */ + F2FS_FOLIO_PRIVATE_INLINE_INODE, /* inode page contains inline data */ + F2FS_FOLIO_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */ + F2FS_FOLIO_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */ + F2FS_FOLIO_PRIVATE_MAX }; /* For compression */ @@ -2681,86 +2678,68 @@ release_quota: return -ENOSPC; } -#define PAGE_PRIVATE_GET_FUNC(name, flagname) \ +#define F2FS_FOLIO_PRIVATE_GET_FUNC(name, flagname) \ static inline bool folio_test_f2fs_##name(const struct folio *folio) \ { \ unsigned long priv = (unsigned long)folio->private; \ - unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) | \ - (1UL << PAGE_PRIVATE_##flagname); \ + unsigned long v = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) | \ + (1UL << F2FS_FOLIO_PRIVATE_##flagname); \ return (priv & v) == v; \ -} \ -static inline bool page_private_##name(struct page *page) \ -{ \ - return page_private(page) && \ - test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \ - test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ } -#define PAGE_PRIVATE_SET_FUNC(name, flagname) \ +#define F2FS_FOLIO_PRIVATE_SET_FUNC(name, flagname) \ static inline void folio_set_f2fs_##name(struct folio *folio) \ { \ - unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) | \ - (1UL << PAGE_PRIVATE_##flagname); \ + unsigned long v = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) | \ + (1UL << F2FS_FOLIO_PRIVATE_##flagname); \ if (!folio->private) \ folio_attach_private(folio, (void *)v); \ else { \ v |= (unsigned long)folio->private; \ folio->private = (void *)v; \ } \ -} \ -static inline void set_page_private_##name(struct page *page) \ -{ \ - if (!page_private(page)) \ - attach_page_private(page, \ - (void *)BIT(PAGE_PRIVATE_NOT_POINTER)); \ - set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ } -#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \ +#define F2FS_FOLIO_PRIVATE_CLEAR_FUNC(name, flagname) \ static inline void folio_clear_f2fs_##name(struct folio *folio) \ { \ unsigned long v = (unsigned long)folio->private; \ \ - v &= ~(1UL << PAGE_PRIVATE_##flagname); \ - if (v == (1UL << PAGE_PRIVATE_NOT_POINTER)) \ + v &= ~(1UL << F2FS_FOLIO_PRIVATE_##flagname); \ + if (v == (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER)) \ folio_detach_private(folio); \ else \ folio->private = (void *)v; \ -} \ -static inline void clear_page_private_##name(struct page *page) \ -{ \ - clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ - if (page_private(page) == BIT(PAGE_PRIVATE_NOT_POINTER)) \ - detach_page_private(page); \ } -PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER); -PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE); -PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE); +F2FS_FOLIO_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER); +F2FS_FOLIO_PRIVATE_GET_FUNC(inline, INLINE_INODE); +F2FS_FOLIO_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION); +F2FS_FOLIO_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE); -PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE); -PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE); -PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE); +F2FS_FOLIO_PRIVATE_SET_FUNC(reference, REF_RESOURCE); +F2FS_FOLIO_PRIVATE_SET_FUNC(inline, INLINE_INODE); +F2FS_FOLIO_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION); +F2FS_FOLIO_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE); -PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE); -PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE); -PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE); +F2FS_FOLIO_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE); +F2FS_FOLIO_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE); +F2FS_FOLIO_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION); +F2FS_FOLIO_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE); static inline unsigned long folio_get_f2fs_data(struct folio *folio) { unsigned long data = (unsigned long)folio->private; - if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data)) + if (!test_bit(F2FS_FOLIO_PRIVATE_NOT_POINTER, &data)) return 0; - return data >> PAGE_PRIVATE_MAX; + return data >> F2FS_FOLIO_PRIVATE_MAX; } static inline void folio_set_f2fs_data(struct folio *folio, unsigned long data) { - data = (1UL << PAGE_PRIVATE_NOT_POINTER) | (data << PAGE_PRIVATE_MAX); + data = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) | + (data << F2FS_FOLIO_PRIVATE_MAX); if (!folio_test_private(folio)) folio_attach_private(folio, (void *)data); diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 63b712d3d599..8c156e1fd37d 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -3803,7 +3803,7 @@ static int __get_segment_type_6(struct f2fs_io_info *fio) if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) return CURSEG_COLD_DATA_PINNED; - if (page_private_gcing(fio->page)) { + if (folio_test_f2fs_gcing(fio->folio)) { if (fio->sbi->am.atgc_enabled && (fio->io_type == FS_DATA_IO) && (fio->sbi->gc_mode != GC_URGENT_HIGH) && From ffbdf157ec6a9b43a1b776e0f8481517dec65c9c Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:04 -0400 Subject: [PATCH 388/539] erofs: mm/pagemap: add readahead_folio_last() to avoid folio->private erofs needs to traverse readahead folios in reverse order to achieve maximum performance by 1. reading all folios from readahead_folio(); 2. storing the prior folio pointer in folio->private; 3. traverse from the last folio to the first one. Add readahead_folio_last() to achieve the same function without using folio->private. __readahead_advance() helper shares readahead_control adjustment code among __readahead_folio(), readahead_folio_last(), and __readahead_batch() by checking new private member, _forward, of readahead_control. It prepares for a future commit that replaces PG_private checks with !folio->private checks. After switching the checks, erofs's use of folio->private without bumping folio refcount can cause unexpected outcomes, e.g., in filemap_release_folio(), try_to_free_buffers() becomes reachable. No functional change intended. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-8-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Assisted-by: LLM Cc: Yue Hu Cc: Jeffle Xu Cc: Sandeep Dhavale Cc: Hongbo Li Cc: Chunhai Guo Cc: Gao Xiang Cc: Chao Yu Cc: "Matthew Wilcox (Oracle)" Cc: Jan Kara --- fs/erofs/zdata.c | 13 +++------ include/linux/pagemap.h | 60 +++++++++++++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 6b07e73ee2aa..e981e371d6c2 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -1886,21 +1886,14 @@ static void z_erofs_readahead(struct readahead_control *rac) struct inode *realinode = erofs_real_inode(sharedinode, &need_iput); Z_EROFS_DEFINE_FRONTEND(f, realinode, sharedinode, readahead_pos(rac)); unsigned int nrpages = readahead_count(rac); - struct folio *head = NULL, *folio; + struct folio *folio; int err; trace_erofs_readahead(realinode, readahead_index(rac), nrpages, false); z_erofs_pcluster_readmore(&f, rac, true); - while ((folio = readahead_folio(rac))) { - folio->private = head; - head = folio; - } - - /* traverse in reverse order for best metadata I/O performance */ - while (head) { - folio = head; - head = folio_get_private(folio); + /* traverse from last to first for best metadata I/O performance */ + while ((folio = readahead_folio_last(rac))) { err = z_erofs_scan_folio(&f, folio, true); if (err && err != -EINTR) erofs_err(realinode->i_sb, "readahead error at folio %lu @ nid %llu", diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 939f3a5e973f..1e3462357aaa 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1415,6 +1415,7 @@ struct readahead_control { bool dropbehind; bool _workingset; unsigned long _pflags; + bool _forward; }; #define DEFINE_READAHEAD(ractl, f, r, m, i) \ @@ -1479,18 +1480,29 @@ void page_cache_async_readahead(struct address_space *mapping, page_cache_async_ra(&ractl, folio, req_count); } +/* + * Adjust readahead_control to ensure next folio comes from + * [_index, _index + _nr_pages) afterwards and reset _batch_count. + */ +static inline void __readahead_advance(struct readahead_control *rac) +{ + if (rac->_forward) + rac->_index += rac->_batch_count; + + rac->_nr_pages -= rac->_batch_count; + rac->_batch_count = 0; +} + static inline struct folio *__readahead_folio(struct readahead_control *ractl) { struct folio *folio; BUG_ON(ractl->_batch_count > ractl->_nr_pages); - ractl->_nr_pages -= ractl->_batch_count; - ractl->_index += ractl->_batch_count; + __readahead_advance(ractl); + ractl->_forward = true; - if (!ractl->_nr_pages) { - ractl->_batch_count = 0; + if (!ractl->_nr_pages) return NULL; - } folio = xa_load(&ractl->mapping->i_pages, ractl->_index); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); @@ -1516,6 +1528,39 @@ static inline struct folio *readahead_folio(struct readahead_control *ractl) return folio; } +/** + * readahead_folio_last - Get the next folio to read, from the tail. + * @ractl: The current readahead request. + * + * Like readahead_folio(), but walks the range back-to-front. The folio is + * returned locked with its refcount dropped; the caller unlocks it once I/O + * completes. Compound folios are returned once, at their head index. + * + * Context: The folio is locked. + * Return: A pointer to the next folio, or %NULL when done. + */ +static inline struct folio *readahead_folio_last(struct readahead_control *ractl) +{ + struct folio *folio; + + /* Drop the previously returned batch from the remaining range. */ + __readahead_advance(ractl); + ractl->_forward = false; + + if (!ractl->_nr_pages) + return NULL; + + /* xa_load() follows sibling entries, so a tail index returns the head */ + folio = xa_load(&ractl->mapping->i_pages, + ractl->_index + ractl->_nr_pages - 1); + VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); + + ractl->_batch_count = folio_nr_pages(folio); + + folio_put(folio); + return folio; +} + static inline unsigned int __readahead_batch(struct readahead_control *rac, struct page **array, unsigned int array_sz) { @@ -1524,9 +1569,8 @@ static inline unsigned int __readahead_batch(struct readahead_control *rac, struct folio *folio; BUG_ON(rac->_batch_count > rac->_nr_pages); - rac->_nr_pages -= rac->_batch_count; - rac->_index += rac->_batch_count; - rac->_batch_count = 0; + __readahead_advance(rac); + rac->_forward = true; xas_set(&xas, rac->_index); rcu_read_lock(); From 96fd9017e3807e158ace3af4eb255c58f52724cf Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:05 -0400 Subject: [PATCH 389/539] erofs: use folio_attach/detach_private() instead of direct assignment erofs_onlinefolio_init/split/end() use folio->private without setting PG_private or increasing folio refcount and it works. But after PG_private is replaced by checking folio->private in a future commit, it can break folio_expected_ref_count(), since the folio has private data without elevated refcount. Change them to use folio_attach/detach_private(). Folios during this process are locked as they are in the process of readahead, so no parallel migration/folio split can happen. Furthermore, because folio->private is used to store in-flight I/O counter and the counter reaches 0 when all I/O completes successfully without error or being dirty, ->private=0 causes folio_detach_private() to not drop the elevated folio refcount. Solve this issue by using bias=1 for the counter, so that ->private stays non NULL throughout every attach-to-detach process. Add a macro EROFS_ONLINEFOLIO_BIAS=1. While at it, fix the comment about ->private bit layout and add EROFS_ONLINEFOLIO_COUNT_MASK. It prepares for a future commit that removes PG_private. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-9-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Reviewed-by: Gao Xiang Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Assisted-by: LLM Cc: Chao Yu Cc: Yue Hu Cc: Jeffle Xu Cc: Sandeep Dhavale Cc: Hongbo Li Cc: Chunhai Guo --- fs/erofs/data.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/fs/erofs/data.c b/fs/erofs/data.c index be63b89f0862..8b150aebf0af 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -239,19 +239,23 @@ int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *map) /* * bit 30: I/O error occurred on this folio * bit 29: CPU has dirty data in D-cache (needs aliasing handling); - * bit 0 - 29: remaining parts to complete this folio + * bit 0 - 28: remaining parts to complete this folio, biased by 1 so that + * ->private stays non-NULL while the folio is attached */ #define EROFS_ONLINEFOLIO_EIO 30 #define EROFS_ONLINEFOLIO_DIRTY 29 +#define EROFS_ONLINEFOLIO_COUNT_MASK (BIT(EROFS_ONLINEFOLIO_DIRTY) - 1) +#define EROFS_ONLINEFOLIO_BIAS 1 void erofs_onlinefolio_init(struct folio *folio) { union { atomic_t o; void *v; - } u = { .o = ATOMIC_INIT(1) }; + } u = { .o = ATOMIC_INIT(1 + EROFS_ONLINEFOLIO_BIAS) }; - folio->private = u.v; /* valid only if file-backed folio is locked */ + /* valid only if file-backed folio is locked */ + folio_attach_private(folio, u.v); } void erofs_onlinefolio_split(struct folio *folio) @@ -265,14 +269,14 @@ void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty) do { orig = atomic_read((atomic_t *)&folio->private); - DBG_BUGON(orig <= 0); + DBG_BUGON((orig & EROFS_ONLINEFOLIO_COUNT_MASK) <= EROFS_ONLINEFOLIO_BIAS); v = dirty << EROFS_ONLINEFOLIO_DIRTY; v |= (orig - 1) | (!!err << EROFS_ONLINEFOLIO_EIO); } while (atomic_cmpxchg((atomic_t *)&folio->private, orig, v) != orig); - if (v & (BIT(EROFS_ONLINEFOLIO_DIRTY) - 1)) + if ((v & EROFS_ONLINEFOLIO_COUNT_MASK) != EROFS_ONLINEFOLIO_BIAS) return; - folio->private = 0; + folio_detach_private(folio); if (v & BIT(EROFS_ONLINEFOLIO_DIRTY)) flush_dcache_folio(folio); folio_end_read(folio, !(v & BIT(EROFS_ONLINEFOLIO_EIO))); From e01f7ddcb18a32cd2fca3806a45b86751efc08ff Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:06 -0400 Subject: [PATCH 390/539] mm/page-flags: check page/folio->private instead of PG_private After the changes of the prior commits, page/folio->private != NULL is now equivalent to checking PG_private. Stop checking PG_private on pages and folios and use page/folio->private instead, except swapcache and hugetlb folios, because the former uses a field (swp_entry_t swap) overlapping with ->private and the latter sets its flags in ->private. Exclude swapcache and hugetlb when the code is meant to check PG_private only. PG_swapcache and folio->swap.val cannot be set/clear as a whole, so excluding swapcache with folio_test_swapcache() is not reliable. Instead, use folio_test_swapbacked(), since PG_swapbacked is stable when a folio is added to/removed from swapcache. Add a helper, folio_has_attached_private(), for this check. folio_test_private() and PagePrivate() now read folio/page->private plainly instead of an atomic read of PG_private bit, so KCSAN complains about possible data races. Annotate them with data_race(). folio_expected_ref_count() can be called without the folio lock, so annotate folio->mapping with data_race() while at it. folio_set/clear_private() and Set/ClearPagePrivate() become no-ops. PG_private is no longer checked at page free time. They will be removed in an upcoming commit. Remove KPF_PRIVATE since PG_private is no longer used. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-10-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Steven Rostedt Cc: Masami Hiramatsu Cc: Lorenzo Stoakes Cc: "Matthew Wilcox (Oracle)" Cc: Jan Kara Cc: Johannes Weiner Cc: "Liam R. Howlett" Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Mathieu Desnoyers Cc: Baolin Wang Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Barry Song Cc: Lance Yang Cc: Usama Arif Cc: Matthew Brost Cc: Joshua Hahn Cc: Rakie Kim Cc: Byungchul Park Cc: Gregory Price Cc: Ying Huang Cc: Alistair Popple Cc: Qi Zheng Cc: Shakeel Butt Cc: Kairui Song Cc: Axel Rasmussen Cc: Yuanchu Xie Cc: Wei Xu --- fs/proc/page.c | 1 - include/linux/kernel-page-flags.h | 1 - include/linux/mm.h | 19 +++++++---- include/linux/page-flags.h | 57 +++++++++++++++++++++++++++---- include/trace/events/pagemap.h | 2 +- mm/huge_memory.c | 2 +- mm/migrate.c | 2 +- mm/page-writeback.c | 2 +- mm/vmscan.c | 2 +- tools/mm/page-types.c | 2 -- 10 files changed, 68 insertions(+), 22 deletions(-) diff --git a/fs/proc/page.c b/fs/proc/page.c index 260772b20bd9..f90e1030825e 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -232,7 +232,6 @@ u64 stable_page_flags(const struct page *page) u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved); u |= kpf_copy_bit(k, KPF_OWNER_2, PG_owner_2); - u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private); u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2); u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1); u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1); diff --git a/include/linux/kernel-page-flags.h b/include/linux/kernel-page-flags.h index 196778a087c4..fe5ab6e50bd7 100644 --- a/include/linux/kernel-page-flags.h +++ b/include/linux/kernel-page-flags.h @@ -11,7 +11,6 @@ #define KPF_RESERVED 32 #define KPF_MLOCKED 33 #define KPF_OWNER_2 34 -#define KPF_PRIVATE 35 #define KPF_PRIVATE_2 36 #define KPF_OWNER_PRIVATE 37 #define KPF_ARCH 38 diff --git a/include/linux/mm.h b/include/linux/mm.h index 30a3365bca82..6c8df7715eb2 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3005,9 +3005,9 @@ static inline bool folio_maybe_mapped_shared(struct folio *folio) * @folio: the folio * * Calculate the expected folio refcount, taking references from the pagecache, - * swapcache, PG_private and page table mappings into account. Useful in - * combination with folio_ref_count() to detect unexpected references (e.g., - * GUP or other temporary references). + * swapcache, private data (folio->private != NULL) and page table mappings into + * account. Useful in combination with folio_ref_count() to detect unexpected + * references (e.g., GUP or other temporary references). * * Does currently not consider references from the LRU cache. If the folio * was isolated from the LRU (which is the case during migration or split), @@ -3045,10 +3045,15 @@ static inline int folio_expected_ref_count(const struct folio *folio) ref_count += folio_test_swapcache(folio) << order; if (!folio_test_anon(folio)) { - /* One reference per page from the pagecache. */ - ref_count += !!folio->mapping << order; - /* One reference from PG_private. */ - ref_count += folio_test_private(folio); + /* + * One reference per page from the pagecache. + * Use data_race() since folio might not be locked. + */ + ref_count += !!data_race(folio->mapping) << order; + /* + * One reference from filesystem private data. + */ + ref_count += folio_has_attached_private(folio); } /* One reference per page table mapping. */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7080a6a1a79e..6d839f50bdcb 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -575,9 +575,31 @@ FOLIO_FLAG(swapbacked, FOLIO_HEAD_PAGE) /* * Private page markings that may be used by the filesystem that owns the page * for its own purposes. - * - PG_private and PG_private_2 cause release_folio() and co to be invoked + * - folio->private and PG_private_2 cause release_folio() and co to be invoked */ -PAGEFLAG(Private, private, PF_ANY) + +static __always_inline bool folio_test_private(const struct folio *folio) +{ + /* + * data_race() is added for readers without holding the folio lock. + * Only the NULL/non-NULL answer is used and both are valid while + * private is being attached or detached, so the race is benign. + */ + return data_race(folio->private); +} + +static __always_inline int PagePrivate(const struct page *page) +{ + /* See folio_test_private() for data_race() use */ + return !!data_race(page->private); +} + +/* no-ops during transition */ +static __always_inline void folio_set_private(struct folio *folio) { } +static __always_inline void folio_clear_private(struct folio *folio) { } +static __always_inline void SetPagePrivate(struct page *page) { } +static __always_inline void ClearPagePrivate(struct page *page) { } + FOLIO_FLAG(private_2, FOLIO_HEAD_PAGE) /* owner_2 can be set on tail pages for anon memory */ @@ -1169,7 +1191,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) */ #define PAGE_FLAGS_CHECK_AT_FREE \ (1UL << PG_lru | 1UL << PG_locked | \ - 1UL << PG_private | 1UL << PG_private_2 | \ + 1UL << PG_private_2 | \ 1UL << PG_writeback | 1UL << PG_reserved | \ 1UL << PG_active | \ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) @@ -1193,8 +1215,31 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) (0xffUL /* order */ | 1UL << PG_has_hwpoisoned | \ 1UL << PG_large_rmappable | 1UL << PG_partially_mapped) -#define PAGE_FLAGS_PRIVATE \ - (1UL << PG_private | 1UL << PG_private_2) +/** + * folio_has_attached_private - check if the folio has private data attached + * @folio: The folio to check. + * + * Use this in code that may encounter swapcache or hugetlb folios but only + * wants to detect attached private data. + * + * Return: true if the folio has private data attached. + */ +static inline bool folio_has_attached_private(const struct folio *folio) +{ + /* + * Swapcache stores swp_entry_t in folio->swap, a union with + * folio->private, and hugetlb stores its own flags in folio->private; + * both are excluded. + * + * NOTE: For swapcache, folio->swap.val PG_swapcache are not set as + * a whole, so folio_test_swapcache() is not reliable to exclude + * swapcache. Use folio_test_swapbacked() instead, since it remains set + * when a folio is added to/removed from swapcache. + */ + + return folio_test_private(folio) && !folio_test_swapbacked(folio) && + !folio_test_hugetlb(folio); +} /** * folio_has_private - Determine if folio has private stuff * @folio: The folio to be checked @@ -1204,7 +1249,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) */ static inline int folio_has_private(const struct folio *folio) { - return !!(folio->flags.f & PAGE_FLAGS_PRIVATE); + return folio_has_attached_private(folio) || folio_test_private_2(folio); } #undef PF_ANY diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h index 36c3a90f0acc..5d47b774633a 100644 --- a/include/trace/events/pagemap.h +++ b/include/trace/events/pagemap.h @@ -22,7 +22,7 @@ (folio_test_swapcache(folio) ? PAGEMAP_SWAPCACHE : 0) | \ (folio_test_swapbacked(folio) ? PAGEMAP_SWAPBACKED : 0) | \ (folio_test_mappedtodisk(folio) ? PAGEMAP_MAPPEDDISK : 0) | \ - (folio_test_private(folio) ? PAGEMAP_BUFFERS : 0) \ + (folio_has_attached_private(folio) ? PAGEMAP_BUFFERS : 0) \ ) TRACE_EVENT(mm_lru_insertion, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 194188c292af..c822e831554b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4845,7 +4845,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start, * will try to drop it before split and then check if the folio * can be split or not. So skip the check here. */ - if (!folio_test_private(folio) && + if (!folio_has_attached_private(folio) && folio_expected_ref_count(folio) != folio_ref_count(folio)) goto next; diff --git a/mm/migrate.c b/mm/migrate.c index a369d0c95c38..b7b92925a28c 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1327,7 +1327,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, * free the metadata, so the page can be freed. */ if (!src->mapping) { - if (folio_test_private(src)) { + if (folio_has_attached_private(src)) { try_to_free_buffers(src); goto out; } diff --git a/mm/page-writeback.c b/mm/page-writeback.c index eeab25d6ce36..499a35473e4f 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2705,7 +2705,7 @@ bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio) if (folio_test_set_dirty(folio)) return false; - __folio_mark_dirty(folio, mapping, !folio_test_private(folio)); + __folio_mark_dirty(folio, mapping, !folio_has_attached_private(folio)); if (mapping->host) { /* !PageAnon && !swapper_space */ diff --git a/mm/vmscan.c b/mm/vmscan.c index 80041e2b8049..dd6261c86279 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1029,7 +1029,7 @@ static void folio_check_dirty_writeback(struct folio *folio, *writeback = folio_test_writeback(folio); /* Verify dirty/writeback state if the filesystem supports it */ - if (!folio_test_private(folio)) + if (!folio_has_attached_private(folio)) return; mapping = folio_mapping(folio); diff --git a/tools/mm/page-types.c b/tools/mm/page-types.c index 7fc5a8be5997..47e4781c5fc3 100644 --- a/tools/mm/page-types.c +++ b/tools/mm/page-types.c @@ -73,7 +73,6 @@ #define KPF_RESERVED 32 #define KPF_MLOCKED 33 #define KPF_OWNER_2 34 -#define KPF_PRIVATE 35 #define KPF_PRIVATE_2 36 #define KPF_OWNER_PRIVATE 37 #define KPF_ARCH 38 @@ -131,7 +130,6 @@ static const char * const page_flag_names[] = { [KPF_RESERVED] = "r:reserved", [KPF_MLOCKED] = "m:mlocked", [KPF_OWNER_2] = "d:owner_2", - [KPF_PRIVATE] = "P:private", [KPF_PRIVATE_2] = "p:private_2", [KPF_OWNER_PRIVATE] = "O:owner_private", [KPF_ARCH] = "h:arch", From abf8efadeb21b741cbd0cc13d57b8f0a207df10d Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:07 -0400 Subject: [PATCH 391/539] treewide: remove folio_set/clear_private() usage They are no-ops now. Remove them. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-11-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Trond Myklebust Cc: Anna Schumaker Cc: "Matthew Wilcox (Oracle)" Cc: Jan Kara Cc: Matthew Brost Cc: Joshua Hahn Cc: Rakie Kim Cc: Byungchul Park Cc: Gregory Price Cc: Ying Huang Cc: Alistair Popple --- fs/nfs/write.c | 2 -- include/linux/pagemap.h | 4 +--- mm/migrate.c | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 623e7ef1f73d..b6967b528669 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -717,7 +717,6 @@ static void nfs_inode_add_request(struct nfs_page *req) nfs_lock_request(req); spin_lock(&mapping->i_private_lock); set_bit(PG_MAPPED, &req->wb_flags); - folio_set_private(folio); folio->private = req; spin_unlock(&mapping->i_private_lock); atomic_long_inc(&nfsi->nrequests); @@ -745,7 +744,6 @@ static void nfs_inode_remove_request(struct nfs_page *req) spin_lock(&mapping->i_private_lock); folio->private = NULL; - folio_clear_private(folio); clear_bit(PG_MAPPED, &req->wb_head->wb_flags); spin_unlock(&mapping->i_private_lock); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 1e3462357aaa..bcbb0afe1a68 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -594,7 +594,6 @@ static inline void folio_attach_private(struct folio *folio, void *data) { folio_get(folio); folio->private = data; - folio_set_private(folio); } /** @@ -629,9 +628,8 @@ static inline void *folio_detach_private(struct folio *folio) { void *data = folio_get_private(folio); - if (!folio_test_private(folio)) + if (!data) return NULL; - folio_clear_private(folio); folio->private = NULL; folio_put(folio); diff --git a/mm/migrate.c b/mm/migrate.c index b7b92925a28c..7e3a81f06974 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -835,7 +835,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) */ if (folio_test_swapcache(folio)) folio_clear_swapcache(folio); - folio_clear_private(folio); /* page->private contains hugetlb specific flags */ if (!folio_test_hugetlb(folio)) From 6d1a85b3067290c54f32782a21e660fa58876f0b Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:08 -0400 Subject: [PATCH 392/539] ceph: replace PagePrivate() with page_private() PagePrivate() is going to be removed along with PG_private and its implementation is the same as page_private(). Link: https://lore.kernel.org/20260920-remove-pg_private-v5-12-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Ilya Dryomov Cc: Alex Markuze Cc: Viacheslav Dubeyko --- fs/ceph/addr.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index e598b2d424ec..0c00e9636b51 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -70,9 +70,7 @@ static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned static inline struct ceph_snap_context *page_snap_context(struct page *page) { - if (PagePrivate(page)) - return (void *)page->private; - return NULL; + return (void *)page_private(page); } /* @@ -124,8 +122,8 @@ static bool ceph_dirty_folio(struct address_space *mapping, struct folio *folio) spin_unlock(&ci->i_ceph_lock); /* - * Reference snap context in folio->private. Also set - * PagePrivate so that we get invalidate_folio callback. + * Reference snap context in folio->private. Setting folio->private is + * what gets us the invalidate_folio callback. */ VM_WARN_ON_FOLIO(folio->private, folio); folio_attach_private(folio, snapc); From ebb7205efe614a8527f45b797f29d368a67b15a1 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sun, 20 Sep 2026 22:28:09 -0400 Subject: [PATCH 393/539] md: use folio_alloc_buffers() Remove the last user of alloc_page_buffers(). Use folio_alloc_buffers() instead, since alloc_page_buffers() is a wrap over it. Although the pages used in md-bitmap are not folios, as they are not mapped into userspace nor enter the page cache, but they still have buffer heads attached. Cleaning up the code to not use buffer heads is future work. [ziy@nvidia.com: reword commit message] Link: https://lore.kernel.org/20260920-remove-pg_private-v5-13-bb68b6a21869@nvidia.com Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) --- drivers/md/md-bitmap.c | 5 +++-- fs/buffer.c | 8 -------- include/linux/buffer_head.h | 1 - 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index b8325cb09a37..5f1637f974c1 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -560,6 +560,7 @@ static int read_file_page(struct file *file, unsigned long index, { int ret = 0; struct inode *inode = file_inode(file); + struct folio *folio = page_folio(page); struct buffer_head *bh; sector_t block, blk_cur; unsigned long blocksize = i_blocksize(inode); @@ -567,12 +568,12 @@ static int read_file_page(struct file *file, unsigned long index, pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE, (unsigned long long)index << PAGE_SHIFT); - bh = alloc_page_buffers(page, blocksize); + bh = folio_alloc_buffers(folio, blocksize, GFP_NOFS | __GFP_ACCOUNT); if (!bh) { ret = -ENOMEM; goto out; } - attach_page_private(page, bh); + folio_attach_private(folio, bh); blk_cur = index << (PAGE_SHIFT - inode->i_blkbits); while (bh) { block = blk_cur; diff --git a/fs/buffer.c b/fs/buffer.c index ed966fa73b1b..020af5dbe2d0 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -773,14 +773,6 @@ no_grow: } EXPORT_SYMBOL_GPL(folio_alloc_buffers); -struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size) -{ - gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT; - - return folio_alloc_buffers(page_folio(page), size, gfp); -} -EXPORT_SYMBOL_GPL(alloc_page_buffers); - static inline void link_dev_buffers(struct folio *folio, struct buffer_head *head) { diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index fd2c7115c054..6ce2db05c60f 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -197,7 +197,6 @@ void folio_set_bh(struct buffer_head *bh, struct folio *folio, unsigned long offset); struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size, gfp_t gfp); -struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size); struct buffer_head *create_empty_buffers(struct folio *folio, unsigned long blocksize, unsigned long b_state); void end_buffer_read_sync(struct buffer_head *bh, int uptodate); From 5d0f54ab925f02df9537a01618dfca00ff05ad2f Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sun, 20 Sep 2026 22:28:10 -0400 Subject: [PATCH 394/539] md: use folio APIs in free_page() Convert the page to a folio. This removes some of the last uses of a few page APIs (detach_page_private(), page_buffers()). Replaces two calls to compound_head() with one. Remove the early return in free_buffers() to avoid memory leak and uninitialized file bitmap pages.[1][2] [ziy@nvidia.com: drop early return] Link: https://lore.kernel.org/20260920-remove-pg_private-v5-14-bb68b6a21869@nvidia.com Link: https://sashiko.dev/#/patchset/20260913-remove-pg_private-v4-0-848550f7574e%40nvidia.com?part=2 [1] Link: https://lore.kernel.org/all/aqgfA0QFi98gXYG2@casper.infradead.org/ [2] Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) --- drivers/md/md-bitmap.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 5f1637f974c1..02a126968ad4 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -533,19 +533,15 @@ static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) static void free_buffers(struct page *page) { - struct buffer_head *bh; + struct folio *folio = page_folio(page); + struct buffer_head *bh = folio_detach_private(folio); - if (!PagePrivate(page)) - return; - - bh = page_buffers(page); while (bh) { struct buffer_head *next = bh->b_this_page; free_buffer_head(bh); bh = next; } - detach_page_private(page); - put_page(page); + folio_put(folio); } /* read a page from a file. From effd0cf5f52618e00aa67f01eb2a909d8742186a Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sun, 20 Sep 2026 22:28:11 -0400 Subject: [PATCH 395/539] md: remove the last use of page_buffers() Convert the page to a folio and use folio_buffers() instead. This does introduce one extra call to compound_head(), but will simplify a later conversion of md-bitmap to use folios instead of pages. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-15-bb68b6a21869@nvidia.com Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand (Arm) --- drivers/md/md-bitmap.c | 3 ++- include/linux/buffer_head.h | 7 +------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 02a126968ad4..8adf6ddca3d6 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -516,7 +516,8 @@ static void end_bitmap_write(struct bio *bio) static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) { - struct buffer_head *bh = page_buffers(page); + struct folio *folio = page_folio(page); + struct buffer_head *bh = folio_buffers(folio); while (bh && bh->b_blocknr) { atomic_inc(&bitmap->pending_writes); diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 6ce2db05c60f..4b0b7188472b 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -175,12 +175,7 @@ static inline unsigned long bh_offset(const struct buffer_head *bh) return (unsigned long)(bh)->b_data & (page_size(bh->b_page) - 1); } -/* If we *know* page->private refers to buffer_heads */ -#define page_buffers(page) \ - ({ \ - BUG_ON(!PagePrivate(page)); \ - ((struct buffer_head *)page_private(page)); \ - }) +/* If we *know* folio->private refers to buffer_heads */ #define folio_buffers(folio) folio_get_private(folio) void buffer_check_dirty_writeback(struct folio *folio, From bbe5d3b3d4020d2696718da5cf19121c77f66a44 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:12 -0400 Subject: [PATCH 396/539] treewide: remove PagePrivate() and PG_private from comments and docs PG_private and PagePrivate() are no longer used. Adjust related comments and documentations to refer to page/folio->private instead. hugetlbfs_reserv.rst is outdated and left unchanged. It should be rewritten. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-16-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Ilya Dryomov Cc: Alex Markuze Cc: Viacheslav Dubeyko Cc: Trond Myklebust Cc: Anna Schumaker Cc: Richard Weinberger Cc: Zhihao Cheng Cc: Lorenzo Stoakes Cc: "Liam R. Howlett" Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko --- Documentation/admin-guide/kdump/vmcoreinfo.rst | 2 +- Documentation/filesystems/vfs.rst | 6 +++--- fs/nfs/file.c | 4 ++-- fs/ubifs/file.c | 8 ++++---- include/linux/mm.h | 15 ++++++++------- include/linux/mm_types.h | 4 ++-- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Documentation/admin-guide/kdump/vmcoreinfo.rst b/Documentation/admin-guide/kdump/vmcoreinfo.rst index 7663c610fe90..5f1df6d08050 100644 --- a/Documentation/admin-guide/kdump/vmcoreinfo.rst +++ b/Documentation/admin-guide/kdump/vmcoreinfo.rst @@ -325,7 +325,7 @@ NR_FREE_PAGES On linux-2.6.21 or later, the number of free pages is in vm_stat[NR_FREE_PAGES]. Used to get the number of free pages. -PG_lru|PG_private|PG_swapcache|PG_swapbacked|PG_hwpoison|PG_head_mask +PG_lru|PG_swapcache|PG_swapbacked|PG_hwpoison|PG_head_mask -------------------------------------------------------------------------- Page attributes. These flags are used to filter various unnecessary for diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index d3a93eec3945..dec7816303c6 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -649,8 +649,8 @@ Writeback. The first can be used independently to the others. The VM can try to release clean pages in order to reuse them. To do this it can call -->release_folio on clean folios with the private -flag set. Clean pages without PagePrivate and with no external references +->release_folio on clean folios with folio->private set. Clean pages +without folio->private set and with no external references will be released without notice being given to the address_space. To achieve this functionality, pages need to be placed on an LRU with @@ -674,7 +674,7 @@ filemap_fdatawait_range, to wait for all writeback to complete. An address_space handler may attach extra information to a page, typically using the 'private' field in the 'struct page'. If such -information is attached, the PG_Private flag should be set. This will +information is attached, non-NULL 'private' field will cause various VM routines to make extra calls into the address_space handler to deal with that data. diff --git a/fs/nfs/file.c b/fs/nfs/file.c index e1bdd10b35f1..38f830a6467c 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -484,7 +484,7 @@ static int nfs_write_end(const struct kiocb *iocb, * Partially or wholly invalidate a page * - Release the private state associated with a page if undergoing complete * page invalidation - * - Called if either PG_private or PG_fscache is set on the page + * - Called if either folio->private or PG_fscache is set on the page * - Caller holds page lock */ static void nfs_invalidate_folio(struct folio *folio, size_t offset, @@ -555,7 +555,7 @@ static void nfs_check_dirty_writeback(struct folio *folio, * Attempt to clear the private state associated with a page when an error * occurs that requires the cached contents of an inode to be written back or * destroyed - * - Called if either PG_private or fscache is set on the page + * - Called if either page->private or fscache is set on the page * - Caller holds page lock * - Return 0 if successful, -error otherwise */ diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c index e73c28b12f97..aa0298ce451e 100644 --- a/fs/ubifs/file.c +++ b/fs/ubifs/file.c @@ -12,14 +12,14 @@ * This file implements VFS file and inode operations for regular files, device * nodes and symlinks as well as address space operations. * - * UBIFS uses 2 page flags: @PG_private and @PG_checked. @PG_private is set if + * UBIFS uses folio->private and page flag @PG_checked. folio->private is set if * the page is dirty and is used for optimization purposes - dirty pages are - * not budgeted so the flag shows that 'ubifs_write_end()' should not release + * not budgeted so it shows that 'ubifs_write_end()' should not release * the budget for this page. The @PG_checked flag is set if full budgeting is * required for the page e.g., when it corresponds to a file hole or it is * beyond the file size. The budgeting is done in 'ubifs_write_begin()', because * it is OK to fail in this function, and the budget is released in - * 'ubifs_write_end()'. So the @PG_private and @PG_checked flags carry + * 'ubifs_write_end()'. So the folio->private and the @PG_checked flag carry * information about how the page was budgeted, to make it possible to release * the budget properly. * @@ -1509,7 +1509,7 @@ static vm_fault_t ubifs_vm_page_mkwrite(struct vm_fault *vmf) * * At the moment we do not know whether the folio is dirty or not, so we * assume that it is not and budget for a new folio. We could look at - * the @PG_private flag and figure this out, but we may race with write + * folio->private and figure this out, but we may race with write * back and the folio state may change by the time we lock it, so this * would need additional care. We do not bother with this at the * moment, although it might be good idea to do. Instead, we allocate diff --git a/include/linux/mm.h b/include/linux/mm.h index 6c8df7715eb2..0a2a7fc4a442 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2049,20 +2049,21 @@ vm_fault_t finish_fault(struct vm_fault *vmf); * * A pagecache page contains an opaque `private' member, which belongs to the * page's address_space. Usually, this is the address of a circular list of - * the page's disk buffers. PG_private must be set to tell the VM to call - * into the filesystem to release these pages. + * the page's disk buffers. It tells the VM to call into the filesystem to + * release these pages. * * A folio may belong to an inode's memory mapping. In this case, * folio->mapping points to the inode, and folio->index is the file * offset of the folio, in units of PAGE_SIZE. * - * If pagecache pages are not associated with an inode, they are said to be - * anonymous pages. These may become associated with the swapcache, and in that - * case PG_swapcache is set, and page->private is an offset into the swapcache. + * If pagecache folios are not associated with an inode, they are said to be + * anonymous folios. These may become associated with the swapcache, and in that + * case PG_swapcache is set, and folio->private is an offset into the swapcache. * * In either case (swapcache or inode backed), the pagecache itself holds one - * reference to the page. Setting PG_private should also increment the - * refcount. The each user mapping also has a reference to the page. + * reference to the folio. Attaching filesystem private data via + * folio_attach_private() also increments the refcount. Each user mapping also + * has a reference to the folio. * * The pagecache pages are stored in a per-mapping radix tree, which is * rooted at mapping->i_pages, and indexed by offset. diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 2a3988178adf..0720a4e98286 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -108,7 +108,7 @@ struct page { }; /** * @private: Mapping-private opaque data. - * Usually used for buffer_heads if PagePrivate. + * Usually used for buffer_heads. * Used for swp_entry_t if swapcache flag set. * Indicates order in the buddy system if PageBuddy * or on pcp_llist. @@ -675,7 +675,7 @@ static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc) #define STRUCT_PAGE_MAX_SHIFT (order_base_2(sizeof(struct page))) /* - * page_private can be used on tail pages. However, PagePrivate is only + * page_private can be used on tail pages. However, it is only * checked by the VM on the head page. So page_private on the tail pages * should be used for data that's ancillary to the head page (eg attaching * buffer heads to tail pages after attaching buffer heads to the head page) From b26c65bedbff4012fcaf9e2aa47281fd9aed0ec3 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 20 Sep 2026 22:28:13 -0400 Subject: [PATCH 397/539] mm/page-flags: remove PG_private folio->private != NULL indicates a folio carries private data, replacing PG_private. All PG_private users are converted. Remove PG_private and reserve the space as PG_folio for future use. Unused PG_private functions are removed too. Link: https://lore.kernel.org/20260920-remove-pg_private-v5-17-bb68b6a21869@nvidia.com Signed-off-by: Zi Yan Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Baoquan He Cc: Mike Rapoport Cc: Pasha Tatashin Cc: Pratyush Yadav Cc: Jonathan Corbet Cc: "Matthew Wilcox (Oracle)" Cc: Jan Kara Cc: Steven Rostedt Cc: Masami Hiramatsu Cc: Dave Young Cc: Shuah Khan Cc: Lorenzo Stoakes Cc: "Liam R. Howlett" Cc: Vlastimil Babka Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Mathieu Desnoyers --- include/linux/page-flags.h | 18 +----------------- include/trace/events/mmflags.h | 2 +- kernel/vmcore_info.c | 1 - 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 6d839f50bdcb..b0ddc652e76c 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -44,10 +44,6 @@ * Consequently, PG_reserved for a page mapped into user space can indicate * the zero page, the vDSO, MMIO pages or device memory. * - * The PG_private bitflag is set on pagecache pages if they contain filesystem - * specific data (which is normally at page->private). It can be used by - * private allocations for its own usage. - * * During initiation of disk I/O, PG_locked is set. This bit is set before I/O * and cleared when writeback _starts_ or when read _completes_. PG_writeback * is set before writeback starts and cleared when it finishes. @@ -105,7 +101,7 @@ enum pageflags { PG_owner_2, /* Owner use. If pagecache, fs may use */ PG_arch_1, PG_reserved, - PG_private, /* If pagecache, has fs-private data */ + PG_folio, /* Do not use: reserved for folio identification */ PG_private_2, /* If pagecache, has fs aux data */ PG_reclaim, /* To be reclaimed asap */ PG_swapbacked, /* Page is backed by RAM/swap */ @@ -588,18 +584,6 @@ static __always_inline bool folio_test_private(const struct folio *folio) return data_race(folio->private); } -static __always_inline int PagePrivate(const struct page *page) -{ - /* See folio_test_private() for data_race() use */ - return !!data_race(page->private); -} - -/* no-ops during transition */ -static __always_inline void folio_set_private(struct folio *folio) { } -static __always_inline void folio_clear_private(struct folio *folio) { } -static __always_inline void SetPagePrivate(struct page *page) { } -static __always_inline void ClearPagePrivate(struct page *page) { } - FOLIO_FLAG(private_2, FOLIO_HEAD_PAGE) /* owner_2 can be set on tail pages for anon memory */ diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index ef9aa388b84f..3c153b3ad845 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -144,7 +144,7 @@ TRACE_DEFINE_ENUM(___GFP_LAST_BIT); DEF_PAGEFLAG_NAME(owner_2), \ DEF_PAGEFLAG_NAME(arch_1), \ DEF_PAGEFLAG_NAME(reserved), \ - DEF_PAGEFLAG_NAME(private), \ + DEF_PAGEFLAG_NAME(folio), \ DEF_PAGEFLAG_NAME(private_2), \ DEF_PAGEFLAG_NAME(writeback), \ DEF_PAGEFLAG_NAME(head), \ diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c index 8614430ca212..5a417f8a922a 100644 --- a/kernel/vmcore_info.c +++ b/kernel/vmcore_info.c @@ -216,7 +216,6 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES); VMCOREINFO_NUMBER(NR_FREE_PAGES); VMCOREINFO_NUMBER(PG_lru); - VMCOREINFO_NUMBER(PG_private); VMCOREINFO_NUMBER(PG_swapcache); VMCOREINFO_NUMBER(PG_swapbacked); #define PAGE_SLAB_MAPCOUNT_VALUE (PGTY_slab << 24) From bb7f5fafc9d0e41921d10c43fb1ca40c8bdb6a0c Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:32 +0200 Subject: [PATCH 398/539] mm/swap: fix off-by-one in swap cache replace sanity check Patch series "mm/huge_memory: clean up and decouple the anon and file split helpers", v6. The folio split path handles anon, page cache and swap cache folios in one routine. That mixing is what makes the swap cache split restrictions hard to lift and to review. We now support uniform split to order-0 only, and no mappingless swap cache folios. And it has left a fair number of dead or redundant checks behind. This series prepares for lifting those restrictions by cleaning up the code first: split the routine into an anon and a file helper, and keep all swap cache handling in the anon helper. The file helper never sees a swap cache folio, folio_check_splittable() rejects them up front. Apart from two bug fixes (patch 1 and 2) and a slight adjustment of anon splitting (patch 12), this is a pure cleanup. Testing: The in-tree split_huge_page_test selftest (uniform, non-uniform and in-folio-offset splits of anon and pagecache folios) passes 62/62 over 600 runs on the patched kernel. ftrace function_graph tracing filtered on __folio_split() was used to compare per-call durations between the base and the patched kernel on the same x86-64 box (interleaved runs across alternating reboots. 135 split calls per run, 600 test runs): Before: 68.52 us, stddev: 1.58 After: 67.38 us, stddev: 1.33 The patched kernel is slightly faster. The stack usage and object size change as the config and compiler change, but in general the stack usage is reduced and object size is basically unchanged. This patch (of 17): The DEBUG_VM sanity check in __swap_cache_replace_folio() iterates the old folio's range with "while (ci_off++ < ci_end)", so the loop body runs on the already-incremented offset: the first entry is skipped and one entry past the range is read. For a folio split that entry belongs to the first after-split folio and was just repointed by the replacement loop above, so the check would warn spuriously whenever sub-folio orders differ from the head folio's. Currently we don't support non-uniform swapcache split, but this still needs a fix to clean it up and prepare for non-uniform swap cache split. Use the same do-while pattern as the replacement loop. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-0-ba1b4ba72c6f@tencent.com Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-1-ba1b4ba72c6f@tencent.com Fixes: 8578e0c00dcf ("mm, swap: use the swap table for the swap cache and switch API") Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Acked-by: Zi Yan Reviewed-by: Barry Song Acked-by: David Hildenbrand (Arm) Reviewed-by: Yeoreum Yun Acked-by: Kiryl Shutsemau (Meta) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/swap_state.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/swap_state.c b/mm/swap_state.c index 625c185a1ca4..cef44aadee61 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -396,8 +396,9 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci, folio_order(old) != folio_order(new)) { ci_off = swp_cluster_offset(old->swap); ci_end = ci_off + folio_nr_pages(old); - while (ci_off++ < ci_end) + do { WARN_ON_ONCE(swp_tb_to_folio(__swap_table_get(ci, ci_off)) != old); + } while (++ci_off < ci_end); } } From 1bdcfcbeaddd14878ae1fc7a1f7257f8a981b1ca Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:33 +0200 Subject: [PATCH 399/539] mm/huge_memory: fix rejection of swap cache folios with a mapping A folio in the swap cache cannot be split if it has a mapping (shmem). The split code does a defensive check for this in __folio_freeze_and_split_unmapped, after the folio ref has been frozen and the NR_SHMEM_THPS/NR_FILE_THPS counters have been decremented. It rejects the split and returns -EINVAL without unfreezing the folio or restoring the counters. That error path is buggy, if it is ever taken. It leaves the folio frozen and stuck, skews the counters, and fires the VM_WARN_ON_ONCE_FOLIO for a state that is actually legitimate. Check for this case up front in folio_check_splittable and return -EBUSY before any state is modified, so the split routine always backs out cleanly. Also fix a bracket style issue that checkpatch.pl keeps complaining about. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-2-ba1b4ba72c6f@tencent.com Fixes: 00527733d0dc ("mm/huge_memory: add two new (not yet used) functions for folio_split()") Fixes: 714b056c8321 ("mm/huge_memory: convert VM_BUG* to VM_WARN* in __folio_split") Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Barry Song Acked-by: David Hildenbrand (Arm) Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c822e831554b..7c55bc5e8da8 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3933,6 +3933,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, int folio_check_splittable(struct folio *folio, unsigned int new_order, enum split_type split_type) { + const bool is_anon = folio_test_anon(folio); + const bool is_swapcache = folio_test_swapcache(folio); + VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); /* * Folios that just got truncated cannot get split. Signal to the @@ -3941,11 +3944,11 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, * TODO: this will also currently refuse folios without a mapping in the * swapcache (shmem or to-be-anon folios). */ - if (!folio->mapping && !folio_test_anon(folio)) + if (!folio->mapping && !is_anon) return -EBUSY; /* order-1 is not supported for anonymous THP. */ - if (folio_test_anon(folio) && new_order == 1) + if (is_anon && new_order == 1) return -EINVAL; /* @@ -3956,7 +3959,7 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, * swapcache folio split. Only uniform split to order-0 can be used * here. */ - if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && folio_test_swapcache(folio)) + if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && is_swapcache) return -EINVAL; if (is_huge_zero_folio(folio)) @@ -3965,6 +3968,15 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, if (folio_test_writeback(folio)) return -EBUSY; + /* + * A non-anon swapcache folio that still has a mapping can only be a + * shmem folio under SWAP IO, it's removed from either swap cache or + * shmem mapping afterward. There is little benefit in splitting them + * hence reject it here up front before touching anything. + */ + if (!is_anon && is_swapcache && folio->mapping) + return -EBUSY; + return 0; } @@ -4037,14 +4049,8 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n } } - if (folio_test_swapcache(folio)) { - if (mapping) { - VM_WARN_ON_ONCE_FOLIO(mapping, folio); - return -EINVAL; - } - + if (folio_test_swapcache(folio)) ci = swap_cluster_get_and_lock(folio); - } /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ if (do_lru) From d4a1754333a0c815c11efc0824573c093bd1a6f1 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:34 +0200 Subject: [PATCH 400/539] mm/huge_memory: invert folio_ref_freeze() check to reduce indentation Invert the folio_ref_freeze() success check in __folio_freeze_and_split_unmapped() to return early on failure, which removes one level of indentation from the entire success path. This is a pure refactoring with no functional change. It prepares the function to be split into separate helpers for anonymous and file-backed folios in a later patch. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-3-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Barry Song Acked-by: David Hildenbrand (Arm) Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 214 +++++++++++++++++++++++------------------------ 1 file changed, 106 insertions(+), 108 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 7c55bc5e8da8..9b2908271b69 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4014,122 +4014,120 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n pgoff_t end, int *nr_shmem_dropped) { struct folio *end_folio = folio_next(folio); + struct swap_cluster_info *ci = NULL; struct folio *new_folio, *next; + struct lruvec *lruvec; int ret = 0; VM_WARN_ON_ONCE(!mapping && end); - if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { - struct swap_cluster_info *ci = NULL; - struct lruvec *lruvec; - - /* Take off the deferred split queue while frozen and memcg set */ - folio_unqueue_deferred_split(folio); - - /* - * deferred_split_scan() takes the folio off the queue before it - * splits it, so the unqueue above finds an empty list and - * leaves PG_partially_mapped set. - * Clear it here: the flag does not survive the split. - */ - folio_reset_partially_mapped(folio); - - if (mapping) { - int nr = folio_nr_pages(folio); - - if (folio_test_pmd_mappable(folio) && - new_order < HPAGE_PMD_ORDER) { - if (folio_test_swapbacked(folio)) { - lruvec_stat_mod_folio(folio, - NR_SHMEM_THPS, -nr); - } else { - lruvec_stat_mod_folio(folio, - NR_FILE_THPS, -nr); - } - } - } - - if (folio_test_swapcache(folio)) - ci = swap_cluster_get_and_lock(folio); - - /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ - if (do_lru) - lruvec = folio_lruvec_lock(folio); - - ret = __split_unmapped_folio(folio, new_order, split_at, xas, - mapping, split_type); - - /* - * Unfreeze after-split folios and put them back to the right - * list. @folio should be kept frozon until page cache - * entries are updated with all the other after-split folios - * to prevent others seeing stale page cache entries. - * As a result, new_folio starts from the next folio of - * @folio. - */ - for (new_folio = folio_next(folio); new_folio != end_folio; - new_folio = next) { - unsigned long nr_pages = folio_nr_pages(new_folio); - - next = folio_next(new_folio); - - zone_device_private_split_cb(folio, new_folio); - - folio_ref_unfreeze(new_folio, - folio_cache_ref_count(new_folio) + 1); - - if (do_lru) - lru_add_split_folio(folio, new_folio, lruvec, list); - - /* - * Anonymous folio with swap cache. - * NOTE: shmem in swap cache is not supported yet. - */ - if (ci) { - __swap_cache_replace_folio(ci, folio, new_folio); - continue; - } - - /* Anonymous folio without swap cache */ - if (!mapping) - continue; - - /* Add the new folio to the page cache. */ - if (new_folio->index < end) { - __xa_store(&mapping->i_pages, new_folio->index, - new_folio, 0); - continue; - } - - VM_WARN_ON_ONCE(!nr_shmem_dropped); - /* Drop folio beyond EOF: ->index >= end */ - if (shmem_mapping(mapping) && nr_shmem_dropped) - *nr_shmem_dropped += nr_pages; - else if (folio_test_clear_dirty(new_folio)) - folio_account_cleaned( - new_folio, inode_to_wb(mapping->host)); - __filemap_remove_folio(new_folio, NULL); - folio_put_refs(new_folio, nr_pages); - } - - zone_device_private_split_cb(folio, NULL); - /* - * Unfreeze @folio only after all page cache entries, which - * used to point to it, have been updated with new folios. - * Otherwise, a parallel folio_try_get() can grab @folio - * and its caller can see stale page cache entries. - */ - folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); - - if (do_lru) - lruvec_unlock(lruvec); - - if (ci) - swap_cluster_unlock(ci); - } else { + if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) return -EAGAIN; + + /* Take off the deferred split queue while frozen and memcg set */ + folio_unqueue_deferred_split(folio); + + /* + * deferred_split_scan() takes the folio off the queue before it + * splits it, so the unqueue above finds an empty list and + * leaves PG_partially_mapped set. + * Clear it here: the flag does not survive the split. + */ + folio_reset_partially_mapped(folio); + + if (mapping) { + int nr = folio_nr_pages(folio); + + if (folio_test_pmd_mappable(folio) && + new_order < HPAGE_PMD_ORDER) { + if (folio_test_swapbacked(folio)) { + lruvec_stat_mod_folio(folio, + NR_SHMEM_THPS, -nr); + } else { + lruvec_stat_mod_folio(folio, + NR_FILE_THPS, -nr); + } + } } + if (folio_test_swapcache(folio)) + ci = swap_cluster_get_and_lock(folio); + + /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ + if (do_lru) + lruvec = folio_lruvec_lock(folio); + + ret = __split_unmapped_folio(folio, new_order, split_at, xas, + mapping, split_type); + + /* + * Unfreeze after-split folios and put them back to the right + * list. @folio should be kept frozon until page cache + * entries are updated with all the other after-split folios + * to prevent others seeing stale page cache entries. + * As a result, new_folio starts from the next folio of + * @folio. + */ + for (new_folio = folio_next(folio); new_folio != end_folio; + new_folio = next) { + unsigned long nr_pages = folio_nr_pages(new_folio); + + next = folio_next(new_folio); + + zone_device_private_split_cb(folio, new_folio); + + folio_ref_unfreeze(new_folio, + folio_cache_ref_count(new_folio) + 1); + + if (do_lru) + lru_add_split_folio(folio, new_folio, lruvec, list); + + /* + * Anonymous folio with swap cache. + * NOTE: shmem in swap cache is not supported yet. + */ + if (ci) { + __swap_cache_replace_folio(ci, folio, new_folio); + continue; + } + + /* Anonymous folio without swap cache */ + if (!mapping) + continue; + + /* Add the new folio to the page cache. */ + if (new_folio->index < end) { + __xa_store(&mapping->i_pages, new_folio->index, + new_folio, 0); + continue; + } + + VM_WARN_ON_ONCE(!nr_shmem_dropped); + /* Drop folio beyond EOF: ->index >= end */ + if (shmem_mapping(mapping) && nr_shmem_dropped) + *nr_shmem_dropped += nr_pages; + else if (folio_test_clear_dirty(new_folio)) + folio_account_cleaned( + new_folio, inode_to_wb(mapping->host)); + __filemap_remove_folio(new_folio, NULL); + folio_put_refs(new_folio, nr_pages); + } + + zone_device_private_split_cb(folio, NULL); + /* + * Unfreeze @folio only after all page cache entries, which + * used to point to it, have been updated with new folios. + * Otherwise, a parallel folio_try_get() can grab @folio + * and its caller can see stale page cache entries. + */ + folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); + + if (do_lru) + lruvec_unlock(lruvec); + + if (ci) + swap_cluster_unlock(ci); + return ret; } From 49c33ca3f8122ca83747d20c6d55a2a7183dc714 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:35 +0200 Subject: [PATCH 401/539] mm/huge_memory: split the routine for splitting anon and file folio No functional change intended. Before adding more logic, split __folio_freeze_and_split_unmapped() into an anon and a file variant so each path can evolve independently. The two paths shared little beyond the folio freeze call, the LRU locking, and the unfreeze skeleton, but differed in all other per-folio bookkeeping and routines. While splitting, some cleanups become easy to apply, and helped drop a few now-redundant checks. The zone_device_private_split_cb() calls are only kept in the anon variant, as device private folios can only back anonymous memory, and add a VM_WARN_ON_ONCE_FOLIO() at the entry of the file variant. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-4-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 128 +++++++++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 48 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 9b2908271b69..a9057449d3a4 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4007,11 +4007,9 @@ static void folio_reset_partially_mapped(struct folio *folio) MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); } -static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int new_order, - struct page *split_at, struct xa_state *xas, - struct address_space *mapping, bool do_lru, - struct list_head *list, enum split_type split_type, - pgoff_t end, int *nr_shmem_dropped) +static int __folio_freeze_split_anon(struct folio *folio, + unsigned int new_order, struct page *split_at, bool do_lru, + struct list_head *list, enum split_type split_type) { struct folio *end_folio = folio_next(folio); struct swap_cluster_info *ci = NULL; @@ -4019,8 +4017,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n struct lruvec *lruvec; int ret = 0; - VM_WARN_ON_ONCE(!mapping && end); - if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) return -EAGAIN; @@ -4035,24 +4031,75 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n */ folio_reset_partially_mapped(folio); - if (mapping) { - int nr = folio_nr_pages(folio); - - if (folio_test_pmd_mappable(folio) && - new_order < HPAGE_PMD_ORDER) { - if (folio_test_swapbacked(folio)) { - lruvec_stat_mod_folio(folio, - NR_SHMEM_THPS, -nr); - } else { - lruvec_stat_mod_folio(folio, - NR_FILE_THPS, -nr); - } - } - } - if (folio_test_swapcache(folio)) ci = swap_cluster_get_and_lock(folio); + if (do_lru) + lruvec = folio_lruvec_lock(folio); + + ret = __split_unmapped_folio(folio, new_order, split_at, NULL, + NULL, split_type); + + /* + * Unfreeze the after-split folios and put them back to the right + * place. Keep the head @folio frozen until the end: sub entries + * in swap cache must be updated first, so a concurrent + * swap_cache_get_folio() cannot return the head folio for a sub + * entry (folio_try_get() will fail on the head @folio until unfreeze). + */ + for (new_folio = folio_next(folio); new_folio != end_folio; + new_folio = next) { + next = folio_next(new_folio); + zone_device_private_split_cb(folio, new_folio); + folio_ref_unfreeze(new_folio, + folio_cache_ref_count(new_folio) + 1); + if (do_lru) + lru_add_split_folio(folio, new_folio, lruvec, list); + if (ci) + __swap_cache_replace_folio(ci, folio, new_folio); + } + + zone_device_private_split_cb(folio, NULL); + folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); + + if (do_lru) + lruvec_unlock(lruvec); + if (ci) + swap_cluster_unlock(ci); + + return ret; +} + +static int __folio_freeze_split_file(struct folio *folio, + unsigned int new_order, struct page *split_at, + struct xa_state *xas, struct address_space *mapping, + bool do_lru, struct list_head *list, + enum split_type split_type, pgoff_t end, int *nr_shmem_dropped) +{ + struct folio *end_folio = folio_next(folio); + struct folio *new_folio, *next; + struct lruvec *lruvec; + int ret; + + /* Currently device private folios can only back anonymous memory. */ + VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio); + + if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) + return -EAGAIN; + + if (folio_test_pmd_mappable(folio) && + new_order < HPAGE_PMD_ORDER) { + int nr = folio_nr_pages(folio); + + if (folio_test_swapbacked(folio)) { + lruvec_stat_mod_folio(folio, + NR_SHMEM_THPS, -nr); + } else { + lruvec_stat_mod_folio(folio, + NR_FILE_THPS, -nr); + } + } + /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ if (do_lru) lruvec = folio_lruvec_lock(folio); @@ -4062,7 +4109,7 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n /* * Unfreeze after-split folios and put them back to the right - * list. @folio should be kept frozon until page cache + * list. @folio should be kept frozen until page cache * entries are updated with all the other after-split folios * to prevent others seeing stale page cache entries. * As a result, new_folio starts from the next folio of @@ -4072,29 +4119,15 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n new_folio = next) { unsigned long nr_pages = folio_nr_pages(new_folio); + /* compute next before the folio can be freed below */ next = folio_next(new_folio); - zone_device_private_split_cb(folio, new_folio); - folio_ref_unfreeze(new_folio, folio_cache_ref_count(new_folio) + 1); if (do_lru) lru_add_split_folio(folio, new_folio, lruvec, list); - /* - * Anonymous folio with swap cache. - * NOTE: shmem in swap cache is not supported yet. - */ - if (ci) { - __swap_cache_replace_folio(ci, folio, new_folio); - continue; - } - - /* Anonymous folio without swap cache */ - if (!mapping) - continue; - /* Add the new folio to the page cache. */ if (new_folio->index < end) { __xa_store(&mapping->i_pages, new_folio->index, @@ -4113,7 +4146,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n folio_put_refs(new_folio, nr_pages); } - zone_device_private_split_cb(folio, NULL); /* * Unfreeze @folio only after all page cache entries, which * used to point to it, have been updated with new folios. @@ -4125,9 +4157,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n if (do_lru) lruvec_unlock(lruvec); - if (ci) - swap_cluster_unlock(ci); - return ret; } @@ -4269,7 +4298,10 @@ static int __folio_split(struct folio *folio, unsigned int new_order, /* block interrupt reentry in xa_lock and spinlock */ local_irq_disable(); - if (mapping) { + if (is_anon) { + ret = __folio_freeze_split_anon(folio, new_order, split_at, + true, list, split_type); + } else { /* * Check if the folio is present in page cache. * We assume all tail are present too, if folio is there. @@ -4280,10 +4312,11 @@ static int __folio_split(struct folio *folio, unsigned int new_order, ret = -EAGAIN; goto fail; } + ret = __folio_freeze_split_file(folio, new_order, split_at, &xas, mapping, + true, list, split_type, end, + &nr_shmem_dropped); } - ret = __folio_freeze_and_split_unmapped(folio, new_order, split_at, &xas, mapping, - true, list, split_type, end, &nr_shmem_dropped); fail: if (mapping) xas_unlock(&xas); @@ -4383,9 +4416,8 @@ int folio_split_unmapped(struct folio *folio, unsigned int new_order) return -EAGAIN; local_irq_disable(); - ret = __folio_freeze_and_split_unmapped(folio, new_order, &folio->page, NULL, - NULL, false, NULL, SPLIT_TYPE_UNIFORM, - 0, NULL); + ret = __folio_freeze_split_anon(folio, new_order, &folio->page, + false, NULL, SPLIT_TYPE_UNIFORM); local_irq_enable(); return ret; } From 60ae71303b0cb92d83a2b63e741a15dd58604d2c Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:36 +0200 Subject: [PATCH 402/539] mm/huge_memory: rename __split_unmapped_folio() to __split_frozen_folio() The helper splits a folio whose refcount is frozen: the frozen refcount is the state it relies on, while unmapping is arranged by the caller beforehand. The old name caused confusion and people may try to call the helper on non-frozen folios. Also add a VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio)) to self document that frozen implies unmapped. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-5-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Suggested-by: Zi Yan Reviewed-by: Zi Yan Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index a9057449d3a4..b66b855df6f8 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3810,8 +3810,8 @@ static void __split_folio_to_order(struct folio *folio, int old_order, } /** - * __split_unmapped_folio() - splits an unmapped @folio to lower order folios in - * two ways: uniform split or non-uniform split. + * __split_frozen_folio() - splits a frozen @folio to lower order folios + * in two ways: uniform split or non-uniform split. * @folio: the to-be-split folio * @new_order: the smallest order of the after split folios (since buddy * allocator like split generates folios with orders from @folio's @@ -3850,7 +3850,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order, * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be * split but not to @new_order, the caller needs to check) */ -static int __split_unmapped_folio(struct folio *folio, int new_order, +static int __split_frozen_folio(struct folio *folio, int new_order, struct page *split_at, struct xa_state *xas, struct address_space *mapping, enum split_type split_type) { @@ -3860,6 +3860,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, struct folio *old_folio = folio; int split_order; + /* Frozen implies unmapped, callers unmap before splitting. */ + VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio); + /* * split to new_order one order at a time. For uniform split, * folio is split to new_order directly. @@ -4037,8 +4040,8 @@ static int __folio_freeze_split_anon(struct folio *folio, if (do_lru) lruvec = folio_lruvec_lock(folio); - ret = __split_unmapped_folio(folio, new_order, split_at, NULL, - NULL, split_type); + ret = __split_frozen_folio(folio, new_order, split_at, NULL, + NULL, split_type); /* * Unfreeze the after-split folios and put them back to the right @@ -4104,8 +4107,8 @@ static int __folio_freeze_split_file(struct folio *folio, if (do_lru) lruvec = folio_lruvec_lock(folio); - ret = __split_unmapped_folio(folio, new_order, split_at, xas, - mapping, split_type); + ret = __split_frozen_folio(folio, new_order, split_at, xas, + mapping, split_type); /* * Unfreeze after-split folios and put them back to the right @@ -4169,9 +4172,9 @@ static int __folio_freeze_split_file(struct folio *folio, * @list: after-split folios will be put on it if non NULL * @split_type: perform uniform split or not (non-uniform split) * - * It calls __split_unmapped_folio() to perform uniform and non-uniform split. + * It calls __split_frozen_folio() to perform uniform and non-uniform split. * It is in charge of checking whether the split is supported or not and - * preparing @folio for __split_unmapped_folio(). + * preparing @folio for __split_frozen_folio(). * * After splitting, the after-split folio containing @lock_at remains locked * and others are unlocked: @@ -4274,7 +4277,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, i_mmap_lock_read(mapping); /* - *__split_unmapped_folio() may need to trim off pages beyond + * __split_frozen_folio() may need to trim off pages beyond * EOF: but on 32-bit, i_size_read() takes an irq-unsafe * seqlock, which cannot be nested inside the page tree lock. * So note end now: i_size itself may be changed at any moment, From f1140fa0006d4b4508de93a473fcb2b5a55aef0d Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:37 +0200 Subject: [PATCH 403/539] mm/huge_memory: consolidate irq and locking for folio split Let each split helper handle its own locking instead of relying on the caller, so both helpers manage their own irq and locking state. This lets __folio_split() drop its local irq handling and fail label, preparing for further cleanup. The file path now uses xas_lock_irq() instead of local_irq_disable() with xas_lock(). The two are equivalent on non-RT, and TRANSPARENT_HUGEPAGE cannot be enabled on RT anyway. This conversion also buys consistency: every other place in mm/ that freezes a folio while it is still reachable through the page cache already takes the lock this way. This was actually the last plain xas_lock() on mapping->i_pages left in mm. If we are going to support RT, spinning on frozen folio refs could be a problem, but it already exists in many places and should be fixed generically. The anon helper keeps a single local_irq_disable() as before, because it has to cover several plain spinlocks at once. The dropped xas_reset() was a no-op as the xa_state is not walked before the xas_load() under the lock. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-6-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Yeoreum Yun Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 58 ++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b66b855df6f8..5460cf63a84f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4020,8 +4020,12 @@ static int __folio_freeze_split_anon(struct folio *folio, struct lruvec *lruvec; int ret = 0; - if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) + local_irq_disable(); + + if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { + local_irq_enable(); return -EAGAIN; + } /* Take off the deferred split queue while frozen and memcg set */ folio_unqueue_deferred_split(folio); @@ -4069,6 +4073,7 @@ static int __folio_freeze_split_anon(struct folio *folio, lruvec_unlock(lruvec); if (ci) swap_cluster_unlock(ci); + local_irq_enable(); return ret; } @@ -4087,8 +4092,21 @@ static int __folio_freeze_split_file(struct folio *folio, /* Currently device private folios can only back anonymous memory. */ VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio); - if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) - return -EAGAIN; + xas_lock_irq(xas); + + /* + * Check if the folio is present in page cache. + * We assume all tail are present too, if folio is there. + */ + if (xas_load(xas) != folio) { + ret = -EAGAIN; + goto fail; + } + + if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { + ret = -EAGAIN; + goto fail; + } if (folio_test_pmd_mappable(folio) && new_order < HPAGE_PMD_ORDER) { @@ -4160,6 +4178,8 @@ static int __folio_freeze_split_file(struct folio *folio, if (do_lru) lruvec_unlock(lruvec); +fail: + xas_unlock_irq(xas); return ret; } @@ -4299,32 +4319,13 @@ static int __folio_split(struct folio *folio, unsigned int new_order, unmap_folio(folio); - /* block interrupt reentry in xa_lock and spinlock */ - local_irq_disable(); - if (is_anon) { + if (is_anon) ret = __folio_freeze_split_anon(folio, new_order, split_at, true, list, split_type); - } else { - /* - * Check if the folio is present in page cache. - * We assume all tail are present too, if folio is there. - */ - xas_lock(&xas); - xas_reset(&xas); - if (xas_load(&xas) != folio) { - ret = -EAGAIN; - goto fail; - } + else ret = __folio_freeze_split_file(folio, new_order, split_at, &xas, mapping, true, list, split_type, end, &nr_shmem_dropped); - } - -fail: - if (mapping) - xas_unlock(&xas); - - local_irq_enable(); if (nr_shmem_dropped) shmem_uncharge(mapping->host, nr_shmem_dropped); @@ -4408,8 +4409,6 @@ out_no_memcg: */ int folio_split_unmapped(struct folio *folio, unsigned int new_order) { - int ret = 0; - VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); @@ -4418,11 +4417,8 @@ int folio_split_unmapped(struct folio *folio, unsigned int new_order) if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) return -EAGAIN; - local_irq_disable(); - ret = __folio_freeze_split_anon(folio, new_order, &folio->page, - false, NULL, SPLIT_TYPE_UNIFORM); - local_irq_enable(); - return ret; + return __folio_freeze_split_anon(folio, new_order, &folio->page, + false, NULL, SPLIT_TYPE_UNIFORM); } /* From 66b7545ba84f40db1ed8b7c1f5add32c8b96014c Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:38 +0200 Subject: [PATCH 404/539] mm/huge_memory: move EOF trimming into the file split helper Instead of receiving @end and @nr_shmem_dropped from the caller, the file split helper now computes the EOF boundary and trims pages beyond it itself, as this is only needed for file split. This drops the redundant parameter passing and sanity check. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-7-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Yeoreum Yun Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 5460cf63a84f..8c621131ee06 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4082,16 +4082,29 @@ static int __folio_freeze_split_file(struct folio *folio, unsigned int new_order, struct page *split_at, struct xa_state *xas, struct address_space *mapping, bool do_lru, struct list_head *list, - enum split_type split_type, pgoff_t end, int *nr_shmem_dropped) + enum split_type split_type) { struct folio *end_folio = folio_next(folio); struct folio *new_folio, *next; + int nr_shmem_dropped = 0; struct lruvec *lruvec; + pgoff_t end; int ret; /* Currently device private folios can only back anonymous memory. */ VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio); + /* + * The loop below may need to trim off pages beyond + * EOF: but on 32-bit, i_size_read() takes an irq-unsafe + * seqlock, which cannot be nested inside the page tree lock. + * So note end now: i_size itself may be changed at any moment, + * but folio lock is good enough to serialize the trimming. + */ + end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); + if (shmem_mapping(mapping)) + end = shmem_fallocend(mapping->host, end); + xas_lock_irq(xas); /* @@ -4156,10 +4169,9 @@ static int __folio_freeze_split_file(struct folio *folio, continue; } - VM_WARN_ON_ONCE(!nr_shmem_dropped); /* Drop folio beyond EOF: ->index >= end */ - if (shmem_mapping(mapping) && nr_shmem_dropped) - *nr_shmem_dropped += nr_pages; + if (shmem_mapping(mapping)) + nr_shmem_dropped += nr_pages; else if (folio_test_clear_dirty(new_folio)) folio_account_cleaned( new_folio, inode_to_wb(mapping->host)); @@ -4180,6 +4192,8 @@ static int __folio_freeze_split_file(struct folio *folio, fail: xas_unlock_irq(xas); + if (nr_shmem_dropped) + shmem_uncharge(mapping->host, nr_shmem_dropped); return ret; } @@ -4216,9 +4230,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, struct anon_vma *anon_vma = NULL; int old_order = folio_order(folio); struct folio *new_folio, *next; - int nr_shmem_dropped = 0; enum ttu_flags ttu_flags = 0; - pgoff_t end = 0; int ret; VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); @@ -4295,17 +4307,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, anon_vma = NULL; i_mmap_lock_read(mapping); - - /* - * __split_frozen_folio() may need to trim off pages beyond - * EOF: but on 32-bit, i_size_read() takes an irq-unsafe - * seqlock, which cannot be nested inside the page tree lock. - * So note end now: i_size itself may be changed at any moment, - * but folio lock is good enough to serialize the trimming. - */ - end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); - if (shmem_mapping(mapping)) - end = shmem_fallocend(mapping->host, end); } /* @@ -4324,11 +4325,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, true, list, split_type); else ret = __folio_freeze_split_file(folio, new_order, split_at, &xas, mapping, - true, list, split_type, end, - &nr_shmem_dropped); - - if (nr_shmem_dropped) - shmem_uncharge(mapping->host, nr_shmem_dropped); + true, list, split_type); if (!ret && is_anon && !folio_is_device_private(folio)) ttu_flags = TTU_USE_SHARED_ZEROPAGE; From ef31409ecb561e97baae288987b4b9bfad1fc2ac Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:39 +0200 Subject: [PATCH 405/539] mm/huge_memory: move unmap and remap into the split helpers To prepare for further cleanup, move the unmap/remap handling from __folio_split() into the split helpers. Only anon folios need to be remapped, so remap_page() is now only called for anon splits and the anon check in remap_page() is redundant and can be removed. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-8-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8c621131ee06..8b041460d977 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3640,9 +3640,6 @@ static void remap_page(struct folio *folio, unsigned long nr, int flags) { int i = 0; - /* If unmap_folio() uses try_to_migrate() on file, remove this check */ - if (!folio_test_anon(folio)) - return; for (;;) { remove_migration_ptes(folio, folio, TTU_RMAP_LOCKED | flags); i += folio_nr_pages(folio); @@ -4016,15 +4013,23 @@ static int __folio_freeze_split_anon(struct folio *folio, { struct folio *end_folio = folio_next(folio); struct swap_cluster_info *ci = NULL; + const int old_order = folio_order(folio); struct folio *new_folio, *next; + enum ttu_flags ttu_flags = 0; struct lruvec *lruvec; + bool need_remap = false; int ret = 0; + if (folio_mapped(folio)) { + need_remap = true; + unmap_folio(folio); + } + local_irq_disable(); if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { - local_irq_enable(); - return -EAGAIN; + ret = -EAGAIN; + goto out_no_split; } /* Take off the deferred split queue while frozen and memcg set */ @@ -4073,7 +4078,13 @@ static int __folio_freeze_split_anon(struct folio *folio, lruvec_unlock(lruvec); if (ci) swap_cluster_unlock(ci); +out_no_split: local_irq_enable(); + if (need_remap) { + if (!ret && !folio_is_device_private(folio)) + ttu_flags = TTU_USE_SHARED_ZEROPAGE; + remap_page(folio, 1 << old_order, ttu_flags); + } return ret; } @@ -4105,6 +4116,8 @@ static int __folio_freeze_split_file(struct folio *folio, if (shmem_mapping(mapping)) end = shmem_fallocend(mapping->host, end); + unmap_folio(folio); + xas_lock_irq(xas); /* @@ -4189,7 +4202,6 @@ static int __folio_freeze_split_file(struct folio *folio, if (do_lru) lruvec_unlock(lruvec); - fail: xas_unlock_irq(xas); if (nr_shmem_dropped) @@ -4230,7 +4242,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, struct anon_vma *anon_vma = NULL; int old_order = folio_order(folio); struct folio *new_folio, *next; - enum ttu_flags ttu_flags = 0; int ret; VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); @@ -4318,8 +4329,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, goto out_unlock; } - unmap_folio(folio); - if (is_anon) ret = __folio_freeze_split_anon(folio, new_order, split_at, true, list, split_type); @@ -4327,11 +4336,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, ret = __folio_freeze_split_file(folio, new_order, split_at, &xas, mapping, true, list, split_type); - if (!ret && is_anon && !folio_is_device_private(folio)) - ttu_flags = TTU_USE_SHARED_ZEROPAGE; - - remap_page(folio, 1 << old_order, ttu_flags); - /* * Drop the mapping while the inode is still pinned. @folio stays * locked and present in the page cache until the loop below, so From 27d3df29ce49bedcdd6f0869da4dec9feb80bbe0 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:40 +0200 Subject: [PATCH 406/539] mm/huge_memory: rename remap_page() to remap_anon_folio() remap_page() now only has one caller, __folio_freeze_split_anon(), and is only ever called for anon folios: unmap_folio() currently leaves file folios unmapped after the split, so they need no remapping. Rename it to remap_anon_folio() to make that explicit, and add a VM_WARN_ON_FOLIO() documenting it. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-9-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Yeoreum Yun Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8b041460d977..09de66b6b76e 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3551,7 +3551,6 @@ static void unmap_folio(struct folio *folio) /* * Anon pages need migration entries to preserve them, but file * pages can simply be left unmapped, then faulted back on demand. - * If that is ever changed (perhaps for mlock), update remap_page(). */ if (folio_test_anon(folio)) try_to_migrate(folio, ttu_flags); @@ -3636,10 +3635,17 @@ bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); } -static void remap_page(struct folio *folio, unsigned long nr, int flags) +static void remap_anon_folio(struct folio *folio, unsigned long nr, int flags) { int i = 0; + /* + * unmap_folio() installs migration entries only for anon folios, + * so currently only anon folios need to be remapped. File folios + * stay unmapped after the split and are faulted back on demand. + */ + VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); + for (;;) { remove_migration_ptes(folio, folio, TTU_RMAP_LOCKED | flags); i += folio_nr_pages(folio); @@ -3723,7 +3729,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order, * * Note that for mapped sub-pages of an anonymous THP, * PG_anon_exclusive has been cleared in unmap_folio() and is stored in - * the migration entry instead from where remap_page() will restore it. + * the migration entry instead from where remap_anon_folio() will restore it. * We can still have PG_anon_exclusive set on effectively unmapped and * unreferenced sub-pages of an anonymous THP: we can simply drop * PG_anon_exclusive (-> PG_mappedtodisk) for these here. @@ -4083,7 +4089,7 @@ out_no_split: if (need_remap) { if (!ret && !folio_is_device_private(folio)) ttu_flags = TTU_USE_SHARED_ZEROPAGE; - remap_page(folio, 1 << old_order, ttu_flags); + remap_anon_folio(folio, 1 << old_order, ttu_flags); } return ret; From be2d44befea1c2c6144ad638bdbc60974223d878 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:41 +0200 Subject: [PATCH 407/539] mm/huge_memory: move the racy refcount check into unmap_folio() The check only exists to avoid the expensive PMD-splitting unmap of a folio that cannot be split anyway. Move it from __folio_split() and folio_split_unmapped() into one check in unmap_folio(), right before the PMD split. All split helpers get the same early check without repeating it. unmap_folio() now returns -EAGAIN if the check fails and the split helpers propagate the error. folio_split_unmapped() drops its own copy of the check: it works on already unmapped folios and the definitive folio_ref_freeze() in __folio_freeze_split_anon() still catches unexpected references. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-10-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Yeoreum Yun Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 09de66b6b76e..ad92c2b554a0 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3538,13 +3538,22 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, split_huge_pmd_if_needed(next, end); } -static void unmap_folio(struct folio *folio) +/* + * A return value of 0 does not mean that unmapping succeeded. It might + * still have failed, but remap_anon_folio() must be called afterwards, + * for anon folios. + */ +static int unmap_folio(struct folio *folio) { enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | TTU_BATCH_FLUSH; VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); + /* Racy check if we can split the page, before we split PMDs */ + if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) + return -EAGAIN; + if (folio_test_pmd_mappable(folio)) ttu_flags |= TTU_SPLIT_HUGE_PMD; @@ -3558,6 +3567,8 @@ static void unmap_folio(struct folio *folio) try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); try_to_unmap_flush(); + + return 0; } static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, @@ -4028,7 +4039,9 @@ static int __folio_freeze_split_anon(struct folio *folio, if (folio_mapped(folio)) { need_remap = true; - unmap_folio(folio); + ret = unmap_folio(folio); + if (ret) + return ret; } local_irq_disable(); @@ -4122,7 +4135,9 @@ static int __folio_freeze_split_file(struct folio *folio, if (shmem_mapping(mapping)) end = shmem_fallocend(mapping->host, end); - unmap_folio(folio); + ret = unmap_folio(folio); + if (ret) + return ret; xas_lock_irq(xas); @@ -4326,15 +4341,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, i_mmap_lock_read(mapping); } - /* - * Racy check if we can split the page, before unmap_folio() will - * split PMDs - */ - if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) { - ret = -EAGAIN; - goto out_unlock; - } - if (is_anon) ret = __folio_freeze_split_anon(folio, new_order, split_at, true, list, split_type); @@ -4373,7 +4379,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, free_folio_and_swap_cache(new_folio); } -out_unlock: if (anon_vma) { anon_vma_unlock_write(anon_vma); put_anon_vma(anon_vma); @@ -4421,9 +4426,6 @@ int folio_split_unmapped(struct folio *folio, unsigned int new_order) VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_anon(folio), folio); - if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) - return -EAGAIN; - return __folio_freeze_split_anon(folio, new_order, &folio->page, false, NULL, SPLIT_TYPE_UNIFORM); } From 550a041c774bba1b79766dd86dc8e4b4fcedf262 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:42 +0200 Subject: [PATCH 408/539] mm/huge_memory: move filemap management into the file split helper Only file split needs the filemap and xarray handling and related variables. Move them out of __folio_split() into the file helper so the helper is self-contained, and simplify the parameters. No functional change. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-11-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Yeoreum Yun Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 102 ++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 58 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index ad92c2b554a0..6457192747aa 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4110,16 +4110,42 @@ out_no_split: static int __folio_freeze_split_file(struct folio *folio, unsigned int new_order, struct page *split_at, - struct xa_state *xas, struct address_space *mapping, bool do_lru, struct list_head *list, enum split_type split_type) { + struct address_space *mapping = folio->mapping; + XA_STATE(xas, &mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); struct folio *new_folio, *next; int nr_shmem_dropped = 0; + unsigned int min_order; struct lruvec *lruvec; pgoff_t end; - int ret; + gfp_t gfp; + int ret = 0; + + min_order = mapping_min_folio_order(mapping); + if (new_order < min_order) + return -EINVAL; + + gfp = current_gfp_context(mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK); + if (!filemap_release_folio(folio, gfp)) + return -EBUSY; + + mapping_set_update(&xas, mapping); + + if (split_type == SPLIT_TYPE_UNIFORM) { + const int old_order = folio_order(folio); + + xas_set_order(&xas, folio->index, new_order); + xas_split_alloc(&xas, folio, old_order, gfp); + if (xas_error(&xas)) { + ret = xas_error(&xas); + goto fail_free; + } + } + + i_mmap_lock_read(mapping); /* Currently device private folios can only back anonymous memory. */ VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio); @@ -4137,15 +4163,15 @@ static int __folio_freeze_split_file(struct folio *folio, ret = unmap_folio(folio); if (ret) - return ret; + goto fail_mmap_unlock; - xas_lock_irq(xas); + xas_lock_irq(&xas); /* * Check if the folio is present in page cache. * We assume all tail are present too, if folio is there. */ - if (xas_load(xas) != folio) { + if (xas_load(&xas) != folio) { ret = -EAGAIN; goto fail; } @@ -4172,7 +4198,7 @@ static int __folio_freeze_split_file(struct folio *folio, if (do_lru) lruvec = folio_lruvec_lock(folio); - ret = __split_frozen_folio(folio, new_order, split_at, xas, + ret = __split_frozen_folio(folio, new_order, split_at, &xas, mapping, split_type); /* @@ -4224,9 +4250,19 @@ static int __folio_freeze_split_file(struct folio *folio, if (do_lru) lruvec_unlock(lruvec); fail: - xas_unlock_irq(xas); + xas_unlock_irq(&xas); +fail_mmap_unlock: if (nr_shmem_dropped) shmem_uncharge(mapping->host, nr_shmem_dropped); + /* + * Drop the mapping while the inode is still pinned. @folio stays + * locked and present in the page cache, so eviction cannot free + * the inode yet, nothing past this point may touch the inode or + * the mapping. + */ + i_mmap_unlock_read(mapping); +fail_free: + xas_destroy(&xas); return ret; } @@ -4255,11 +4291,9 @@ static int __folio_split(struct folio *folio, unsigned int new_order, struct page *split_at, struct page *lock_at, struct list_head *list, enum split_type split_type) { - XA_STATE(xas, &folio->mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); bool is_anon = folio_test_anon(folio); struct mem_cgroup *memcg, *old_memcg; - struct address_space *mapping = NULL; struct anon_vma *anon_vma = NULL; int old_order = folio_order(folio); struct folio *new_folio, *next; @@ -4306,60 +4340,15 @@ static int __folio_split(struct folio *folio, unsigned int new_order, goto out; } anon_vma_lock_write(anon_vma); - mapping = NULL; - } else { - unsigned int min_order; - gfp_t gfp; - - mapping = folio->mapping; - min_order = mapping_min_folio_order(mapping); - if (new_order < min_order) { - ret = -EINVAL; - goto out; - } - - gfp = current_gfp_context(mapping_gfp_mask(mapping) & - GFP_RECLAIM_MASK); - - if (!filemap_release_folio(folio, gfp)) { - ret = -EBUSY; - goto out; - } - - mapping_set_update(&xas, mapping); - - if (split_type == SPLIT_TYPE_UNIFORM) { - xas_set_order(&xas, folio->index, new_order); - xas_split_alloc(&xas, folio, old_order, gfp); - if (xas_error(&xas)) { - ret = xas_error(&xas); - goto out; - } - } - - anon_vma = NULL; - i_mmap_lock_read(mapping); } if (is_anon) ret = __folio_freeze_split_anon(folio, new_order, split_at, true, list, split_type); else - ret = __folio_freeze_split_file(folio, new_order, split_at, &xas, mapping, + ret = __folio_freeze_split_file(folio, new_order, split_at, true, list, split_type); - /* - * Drop the mapping while the inode is still pinned. @folio stays - * locked and present in the page cache until the loop below, so - * eviction cannot free the inode yet; @lock_at is not enough, it may - * be a tail beyond EOF that the split already dropped from the page - * cache. Nothing past this point may touch the inode or the mapping. - */ - if (mapping) { - i_mmap_unlock_read(mapping); - mapping = NULL; - } - /* * Unlock all after-split folios except the one containing * @lock_at page. If @folio is not split, it will be kept locked. @@ -4383,14 +4372,11 @@ static int __folio_split(struct folio *folio, unsigned int new_order, anon_vma_unlock_write(anon_vma); put_anon_vma(anon_vma); } - if (mapping) - i_mmap_unlock_read(mapping); out: /* restore to caller's old_memcg */ set_active_memcg(old_memcg); mem_cgroup_put(memcg); out_no_memcg: - xas_destroy(&xas); if (is_pmd_order(old_order)) count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); From fea0f0ed65e88ccb01c31450d0fec0d5754f3976 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:43 +0200 Subject: [PATCH 409/539] mm/huge_memory: move anon_vma handling into the anon split helper Only anon split needs the anon_vma, and it only needs it to unmap and remap. Move the folio_get_anon_vma()/anon_vma_lock_write() pair out of __folio_split() into the anon helper next to the folio_mapped() check that already gates unmap_folio(). This makes the anon_vma conditional on folio_mapped(), which is a behaviour change but should be fine. folio_get_anon_vma() returns NULL whenever !folio_mapped(), so an anon folio with folio_mapcount() == 0 used to get -EBUSY from split_huge_page() and is now split instead. A realistic case is a THP that has been fully swapped out and is still in the swap cache: swap PTEs do not contribute mapcount, so it is !folio_mapped() but still alive. That should be safe and right to have because: - folio_ref_freeze() below still rejects a folio that picked up any reference, a mapping or a GUP pin, in the meantime. - A parallel split is excluded by the folio lock. The anon_vma write lock was added to serialize split in commit 062f1af2170a ("mm: thp: acquire the anon_vma rwsem for write during split"), when split_huge_page() did not hold the folio lock throughout. commit e9b61f19858a ("thp: reintroduce split_huge_page()") later made the folio lock a caller requirement and added the folio_ref_freeze() scheme, so that has been covered ever since. - Unmapped path is already exercised by folio_split_unmapped(), and the swap cache split already runs well for a partially swapped-out mapped THP. - A !folio_mapped() folio cannot become mapped meanwhile: mapping it requires the folio lock. For mapped folios the anon_vma write lock is now released before __folio_split() unlocks the after-split sub-folios, where previously it was held across that loop; that window is harmless as the sub-folios stay folio-locked and referenced until it. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-12-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Yeoreum Yun Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Kiryl Shutsemau Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 57 +++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 6457192747aa..ec955488d280 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4032,16 +4032,37 @@ static int __folio_freeze_split_anon(struct folio *folio, struct swap_cluster_info *ci = NULL; const int old_order = folio_order(folio); struct folio *new_folio, *next; + struct anon_vma *anon_vma = NULL; enum ttu_flags ttu_flags = 0; struct lruvec *lruvec; - bool need_remap = false; int ret = 0; + /* + * Unmap/remap needs the anon_vma, so we first take a reference on + * it to prevent it from disappearing, and lock it for write here, + * letting unmap_folio() walk the rmap with TTU_RMAP_LOCKED. + * + * folio_mapped() is not stable here, but it can only change in + * one direction while the folio is locked. The mapcount can drop + * to zero at any time, zap_pte_range() takes no folio lock. It + * cannot go up: swapin, migration and uffd move all lock the folio + * before mapping it, and fork only copies PTEs that already exist. + * + * So if we see the folio mapped, the worst case is an empty rmap + * walk. If we see it unmapped, it stays unmapped and needs neither + * the reference nor the lock. Anything else needs a reference + * first and folio_ref_freeze() below catches it. + * + * Note that entirely swapped-out THPs are unmapped but can be split. + */ if (folio_mapped(folio)) { - need_remap = true; + anon_vma = folio_get_anon_vma(folio); + if (!anon_vma) + return -EBUSY; + anon_vma_lock_write(anon_vma); ret = unmap_folio(folio); if (ret) - return ret; + goto out_unlock; } local_irq_disable(); @@ -4099,11 +4120,16 @@ static int __folio_freeze_split_anon(struct folio *folio, swap_cluster_unlock(ci); out_no_split: local_irq_enable(); - if (need_remap) { + if (anon_vma) { if (!ret && !folio_is_device_private(folio)) ttu_flags = TTU_USE_SHARED_ZEROPAGE; remap_anon_folio(folio, 1 << old_order, ttu_flags); } +out_unlock: + if (anon_vma) { + anon_vma_unlock_write(anon_vma); + put_anon_vma(anon_vma); + } return ret; } @@ -4294,7 +4320,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, struct folio *end_folio = folio_next(folio); bool is_anon = folio_test_anon(folio); struct mem_cgroup *memcg, *old_memcg; - struct anon_vma *anon_vma = NULL; int old_order = folio_order(folio); struct folio *new_folio, *next; int ret; @@ -4325,23 +4350,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, memcg = get_mem_cgroup_from_folio(folio); old_memcg = set_active_memcg(memcg); - if (is_anon) { - /* - * The caller does not necessarily hold an mmap_lock that would - * prevent the anon_vma disappearing so we first we take a - * reference to it and then lock the anon_vma for write. This - * is similar to folio_lock_anon_vma_read except the write lock - * is taken to serialise against parallel split or collapse - * operations. - */ - anon_vma = folio_get_anon_vma(folio); - if (!anon_vma) { - ret = -EBUSY; - goto out; - } - anon_vma_lock_write(anon_vma); - } - if (is_anon) ret = __folio_freeze_split_anon(folio, new_order, split_at, true, list, split_type); @@ -4368,11 +4376,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, free_folio_and_swap_cache(new_folio); } - if (anon_vma) { - anon_vma_unlock_write(anon_vma); - put_anon_vma(anon_vma); - } -out: /* restore to caller's old_memcg */ set_active_memcg(old_memcg); mem_cgroup_put(memcg); From ce97e78f8b12a333521030e94c78fb0f9d359839 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:44 +0200 Subject: [PATCH 410/539] mm/huge_memory: move memcg switch into the file split helper The xarray node allocations in __folio_freeze_split_file() need to be charged to the folio's memcg, so move the memcg switch from __folio_split() into the helper. The anon split helper and the after-split folio freeing perform no chargeable allocations, so no memcg handling is left in __folio_split(). Rename its out_no_memcg label to out. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-13-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Acked-by: Zi Yan Acked-by: David Hildenbrand (Arm) Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index ec955488d280..46d00bf6e437 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4142,6 +4142,7 @@ static int __folio_freeze_split_file(struct folio *folio, struct address_space *mapping = folio->mapping; XA_STATE(xas, &mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); + struct mem_cgroup *memcg, *old_memcg; struct folio *new_folio, *next; int nr_shmem_dropped = 0; unsigned int min_order; @@ -4154,9 +4155,18 @@ static int __folio_freeze_split_file(struct folio *folio, if (new_order < min_order) return -EINVAL; + /* + * Switch to folio's memcg as xarray node allocation can happen and + * needs to charge to it. + */ + memcg = get_mem_cgroup_from_folio(folio); + old_memcg = set_active_memcg(memcg); + gfp = current_gfp_context(mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK); - if (!filemap_release_folio(folio, gfp)) - return -EBUSY; + if (!filemap_release_folio(folio, gfp)) { + ret = -EBUSY; + goto fail_free; + } mapping_set_update(&xas, mapping); @@ -4288,6 +4298,9 @@ fail_mmap_unlock: */ i_mmap_unlock_read(mapping); fail_free: + /* Restore the previously active memcg */ + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); xas_destroy(&xas); return ret; } @@ -4319,7 +4332,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, { struct folio *end_folio = folio_next(folio); bool is_anon = folio_test_anon(folio); - struct mem_cgroup *memcg, *old_memcg; int old_order = folio_order(folio); struct folio *new_folio, *next; int ret; @@ -4329,27 +4341,20 @@ static int __folio_split(struct folio *folio, unsigned int new_order, if (folio != page_folio(split_at) || folio != page_folio(lock_at)) { ret = -EINVAL; - goto out_no_memcg; + goto out; } if (new_order >= old_order) { ret = -EINVAL; - goto out_no_memcg; + goto out; } ret = folio_check_splittable(folio, new_order, split_type); if (ret) { VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio"); - goto out_no_memcg; + goto out; } - /* - * switch to folio's memcg as xarray node allocation can happen and - * needs to charge to it. - */ - memcg = get_mem_cgroup_from_folio(folio); - old_memcg = set_active_memcg(memcg); - if (is_anon) ret = __folio_freeze_split_anon(folio, new_order, split_at, true, list, split_type); @@ -4376,10 +4381,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, free_folio_and_swap_cache(new_folio); } - /* restore to caller's old_memcg */ - set_active_memcg(old_memcg); - mem_cgroup_put(memcg); -out_no_memcg: +out: if (is_pmd_order(old_order)) count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); From 1643b0b76fd81a6e1ce459d519bd814bbd457480 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:45 +0200 Subject: [PATCH 411/539] mm/huge_memory: drop the unused do_lru argument of the file split helper The only caller of __folio_freeze_split_file() always passes do_lru as true, so the argument and the branches gated on it are dead code. Drop it. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-14-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 46d00bf6e437..4bcd57540eea 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4136,8 +4136,7 @@ out_unlock: static int __folio_freeze_split_file(struct folio *folio, unsigned int new_order, struct page *split_at, - bool do_lru, struct list_head *list, - enum split_type split_type) + struct list_head *list, enum split_type split_type) { struct address_space *mapping = folio->mapping; XA_STATE(xas, &mapping->i_pages, folio->index); @@ -4231,9 +4230,7 @@ static int __folio_freeze_split_file(struct folio *folio, } /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ - if (do_lru) - lruvec = folio_lruvec_lock(folio); - + lruvec = folio_lruvec_lock(folio); ret = __split_frozen_folio(folio, new_order, split_at, &xas, mapping, split_type); @@ -4255,8 +4252,7 @@ static int __folio_freeze_split_file(struct folio *folio, folio_ref_unfreeze(new_folio, folio_cache_ref_count(new_folio) + 1); - if (do_lru) - lru_add_split_folio(folio, new_folio, lruvec, list); + lru_add_split_folio(folio, new_folio, lruvec, list); /* Add the new folio to the page cache. */ if (new_folio->index < end) { @@ -4282,9 +4278,7 @@ static int __folio_freeze_split_file(struct folio *folio, * and its caller can see stale page cache entries. */ folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); - - if (do_lru) - lruvec_unlock(lruvec); + lruvec_unlock(lruvec); fail: xas_unlock_irq(&xas); fail_mmap_unlock: @@ -4360,7 +4354,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, true, list, split_type); else ret = __folio_freeze_split_file(folio, new_order, split_at, - true, list, split_type); + list, split_type); /* * Unlock all after-split folios except the one containing From 925ca87557a6b3c156c0168f1db1a3ff4801222b Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:46 +0200 Subject: [PATCH 412/539] mm/huge_memory: clean up after-split folio freeing in __folio_split Replace free_folio_and_swap_cache() with an explicit folio_free_swap() and folio_put() in the after-split loop. free_folio_and_swap_cache() must trylock it again and re-check folio_mapped() before freeing the swap cache entries. If the trylock loses a race, the entries are left behind even though the folio reference is dropped. The sub folios are still locked here, so just directly call folio_free_swap() under the lock if it's unmapped, then unlock and drop the reference. This makes the swap cache freeing deterministic and the reference drop explicit. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-15-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 4bcd57540eea..2614a4676d7b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4325,7 +4325,8 @@ static int __folio_split(struct folio *folio, unsigned int new_order, struct list_head *list, enum split_type split_type) { struct folio *end_folio = folio_next(folio); - bool is_anon = folio_test_anon(folio); + const bool is_anon = folio_test_anon(folio); + const bool is_swapcache = folio_test_swapcache(folio); int old_order = folio_order(folio); struct folio *new_folio, *next; int ret; @@ -4365,14 +4366,16 @@ static int __folio_split(struct folio *folio, unsigned int new_order, if (new_folio == page_folio(lock_at)) continue; - folio_unlock(new_folio); /* * Subpages whose mapping has been zapped may be freed * earlier, but freeing them requires taking the - * lru_lock, so we defer put_page() on tail pages until + * lru_lock, so we defer folio_put() on tail pages until * after the split completes. */ - free_folio_and_swap_cache(new_folio); + if (is_swapcache && !folio_mapped(new_folio)) + folio_free_swap(new_folio); + folio_unlock(new_folio); + folio_put(new_folio); } out: @@ -4399,7 +4402,7 @@ out: * isolated from LRU (if applicable) * * Upon return, the folio is not remapped, split folios are not added to LRU, - * free_folio_and_swap_cache() is not called, and new folios remain locked. + * folio_free_swap() is not called, and new folios remain locked. * * Return: 0 on success, -EAGAIN if the folio cannot be split (e.g., due to * insufficient reference count or extra pins). From e294bbe8edbe664d2edf6839f6b7307b861b97aa Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:47 +0200 Subject: [PATCH 413/539] mm/huge_memory: count only swap cache refs in anon folio split Only __folio_freeze_split_anon() sees anon folios and swap cache folios now. The file split helper only handles page cache folios, which hold exactly folio_nr_pages() references. Rename folio_cache_ref_count() to folio_swapcache_ref_count() and drop the anon check so the helper counts what its name says. The file split helper now uses folio_nr_pages() directly. No feature change. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-16-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2614a4676d7b..349c75b832cd 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3997,10 +3997,10 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, return 0; } -/* Number of folio references from the pagecache or the swapcache. */ -static unsigned int folio_cache_ref_count(const struct folio *folio) +/* Number of folio references from the swapcache. */ +static unsigned int folio_swapcache_ref_count(const struct folio *folio) { - if (folio_test_anon(folio) && !folio_test_swapcache(folio)) + if (!folio_test_swapcache(folio)) return 0; return folio_nr_pages(folio); } @@ -4067,7 +4067,7 @@ static int __folio_freeze_split_anon(struct folio *folio, local_irq_disable(); - if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { + if (!folio_ref_freeze(folio, folio_swapcache_ref_count(folio) + 1)) { ret = -EAGAIN; goto out_no_split; } @@ -4104,7 +4104,7 @@ static int __folio_freeze_split_anon(struct folio *folio, next = folio_next(new_folio); zone_device_private_split_cb(folio, new_folio); folio_ref_unfreeze(new_folio, - folio_cache_ref_count(new_folio) + 1); + folio_swapcache_ref_count(new_folio) + 1); if (do_lru) lru_add_split_folio(folio, new_folio, lruvec, list); if (ci) @@ -4112,7 +4112,7 @@ static int __folio_freeze_split_anon(struct folio *folio, } zone_device_private_split_cb(folio, NULL); - folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); + folio_ref_unfreeze(folio, folio_swapcache_ref_count(folio) + 1); if (do_lru) lruvec_unlock(lruvec); @@ -4138,6 +4138,7 @@ static int __folio_freeze_split_file(struct folio *folio, unsigned int new_order, struct page *split_at, struct list_head *list, enum split_type split_type) { + const long old_nr_pages = folio_nr_pages(folio); struct address_space *mapping = folio->mapping; XA_STATE(xas, &mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); @@ -4211,22 +4212,16 @@ static int __folio_freeze_split_file(struct folio *folio, goto fail; } - if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { + if (!folio_ref_freeze(folio, old_nr_pages + 1)) { ret = -EAGAIN; goto fail; } - if (folio_test_pmd_mappable(folio) && - new_order < HPAGE_PMD_ORDER) { - int nr = folio_nr_pages(folio); - - if (folio_test_swapbacked(folio)) { - lruvec_stat_mod_folio(folio, - NR_SHMEM_THPS, -nr); - } else { - lruvec_stat_mod_folio(folio, - NR_FILE_THPS, -nr); - } + if (folio_test_pmd_mappable(folio) && new_order < HPAGE_PMD_ORDER) { + if (folio_test_swapbacked(folio)) + lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -old_nr_pages); + else + lruvec_stat_mod_folio(folio, NR_FILE_THPS, -old_nr_pages); } /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ @@ -4250,7 +4245,7 @@ static int __folio_freeze_split_file(struct folio *folio, next = folio_next(new_folio); folio_ref_unfreeze(new_folio, - folio_cache_ref_count(new_folio) + 1); + folio_nr_pages(new_folio) + 1); lru_add_split_folio(folio, new_folio, lruvec, list); @@ -4277,7 +4272,7 @@ static int __folio_freeze_split_file(struct folio *folio, * Otherwise, a parallel folio_try_get() can grab @folio * and its caller can see stale page cache entries. */ - folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); + folio_ref_unfreeze(folio, folio_nr_pages(folio) + 1); lruvec_unlock(lruvec); fail: xas_unlock_irq(&xas); From 28585128ead0bafbd75221aad022ce8cb738eb47 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Wed, 23 Sep 2026 23:32:48 +0200 Subject: [PATCH 414/539] mm/huge_memory: drop the redundant mapping argument of __split_frozen_folio The mapping parameter only served as a non-NULL check to detect whether page cache entries need updating. The xa_state pointer conveys exactly the same information: the anon split helper passes NULL and the file split helper passes &xas, which is non-NULL iff the folio is in the page cache. Use the xas pointer instead and drop the parameter, along with its kerneldoc entry. Link: https://lore.kernel.org/20260923-swap-thp-cleanup-v6-17-ba1b4ba72c6f@tencent.com Signed-off-by: Kairui Song Signed-off-by: Andrew Morton Reviewed-by: Zi Yan Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Cc: Lorenzo Stoakes Cc: Baolin Wang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Cc: Dev Jain Cc: Lance Yang Cc: Usama Arif Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: Baoquan He Cc: Barry Song Cc: Youngjun Park Cc: Shivam Kalra Cc: Kairui Song --- mm/huge_memory.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 349c75b832cd..8f8bf60a2264 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3833,7 +3833,6 @@ static void __split_folio_to_order(struct folio *folio, int old_order, * @split_at: in buddy allocator like split, the folio containing @split_at * will be split until its order becomes @new_order. * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller - * @mapping: @folio->mapping * @split_type: if the split is uniform or not (buddy allocator like split) * * @@ -3866,7 +3865,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order, */ static int __split_frozen_folio(struct folio *folio, int new_order, struct page *split_at, struct xa_state *xas, - struct address_space *mapping, enum split_type split_type) + enum split_type split_type) { const bool is_anon = folio_test_anon(folio); int old_order = folio_order(folio); @@ -3890,7 +3889,7 @@ static int __split_frozen_folio(struct folio *folio, int new_order, if (is_anon && split_order == 1) continue; - if (mapping) { + if (xas) { /* * uniform split has xas_split_alloc() called before * irq is disabled to allocate enough memory, whereas @@ -4089,8 +4088,7 @@ static int __folio_freeze_split_anon(struct folio *folio, if (do_lru) lruvec = folio_lruvec_lock(folio); - ret = __split_frozen_folio(folio, new_order, split_at, NULL, - NULL, split_type); + ret = __split_frozen_folio(folio, new_order, split_at, NULL, split_type); /* * Unfreeze the after-split folios and put them back to the right @@ -4226,8 +4224,7 @@ static int __folio_freeze_split_file(struct folio *folio, /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ lruvec = folio_lruvec_lock(folio); - ret = __split_frozen_folio(folio, new_order, split_at, &xas, - mapping, split_type); + ret = __split_frozen_folio(folio, new_order, split_at, &xas, split_type); /* * Unfreeze after-split folios and put them back to the right From 3cada562c2bf0acbe505da23b705f125cb8ea119 Mon Sep 17 00:00:00 2001 From: Guillaume Morin Date: Mon, 14 Sep 2026 17:36:21 +0200 Subject: [PATCH 415/539] selftests/mm: hugetlb_madv_vs_map: add underflow test Add a test that checks for underflows when a parent unmaps the page first. Also check that when the child exits the reserve count is correct. Link: https://lore.kernel.org/all/alEJkwn5VlTTH_ZX@bender.morinfr.org/ Link: https://lore.kernel.org/aqgUdbtumaO8RiIb@bender.morinfr.org Signed-off-by: Guillaume Morin Signed-off-by: Andrew Morton Reviewed-by: Breno Leitao Reviewed-by: Mike Rapoport Tested-by: Breno Leitao --- .../testing/selftests/mm/hugepage_settings.c | 9 ++ .../testing/selftests/mm/hugepage_settings.h | 1 + .../selftests/mm/hugetlb_madv_vs_map.c | 141 +++++++++++++++--- 3 files changed, 130 insertions(+), 21 deletions(-) diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/testing/selftests/mm/hugepage_settings.c index d7917dce3aba..584054736ce9 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -449,6 +449,15 @@ unsigned long hugetlb_free_pages(unsigned long size) return read_num(path); } +unsigned long hugetlb_nr_resv_pages(unsigned long size) +{ + char path[PATH_MAX]; + + hugetlb_sysfs_path(path, sizeof(path), size, "resv_hugepages"); + + return read_num(path); +} + static bool __hugetlb_setup(unsigned long size, unsigned long nr) { unsigned long free = hugetlb_free_pages(size); diff --git a/tools/testing/selftests/mm/hugepage_settings.h b/tools/testing/selftests/mm/hugepage_settings.h index 726c73c43c05..548e9d288d1d 100644 --- a/tools/testing/selftests/mm/hugepage_settings.h +++ b/tools/testing/selftests/mm/hugepage_settings.h @@ -98,6 +98,7 @@ unsigned long default_huge_page_size(void); unsigned long hugetlb_nr_pages(unsigned long size); void hugetlb_set_nr_pages(unsigned long size, unsigned long nr); unsigned long hugetlb_free_pages(unsigned long size); +unsigned long hugetlb_nr_resv_pages(unsigned long size); static inline void hugetlb_save_settings(void) { diff --git a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c index f94549efcc6f..0f15eff1da04 100644 --- a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c +++ b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c @@ -3,18 +3,6 @@ * A test case that must run on a system with one and only one huge page available. * # echo 1 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages * - * During setup, the test allocates the only available page, and starts three threads: - * - thread1: - * * madvise(MADV_DONTNEED) on the allocated huge page - * - thread 2: - * * Write to the allocated huge page - * - thread 3: - * * Try to allocated an extra huge page (which must not available) - * - * The test fails if thread3 is able to allocate a page. - * - * Touching the first page after thread3's allocation will raise a SIGBUS - * * Author: Breno Leitao */ #include @@ -22,6 +10,7 @@ #include #include #include +#include #include #include "vm_util.h" @@ -74,7 +63,21 @@ void *map_extra(void *unused) return NULL; } -int main(void) +/* During setup in main, the only available page was allocated. This test then + * starts three threads: + * + * - thread1: + * * madvise(MADV_DONTNEED) on the allocated huge page + * - thread 2: + * * Write to the allocated huge page + * - thread 3: + * * Try to allocated an extra huge page (which must not available) + * + * The test fails if thread3 is able to allocate a page. + * + * Touching the first page after thread3's allocation will raise a SIGBUS + */ +void test_madv_vs_map(void) { pthread_t thread1, thread2, thread3; void *ret; @@ -85,13 +88,6 @@ int main(void) */ int max = 10; - ksft_print_header(); - ksft_set_plan(1); - - if (!hugetlb_setup_default_exact(1)) - ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n", - hugetlb_free_default_pages()); - mmap_size = default_huge_page_size(); while (max--) { @@ -100,7 +96,7 @@ int main(void) -1, 0); if ((unsigned long)huge_ptr == -1) - ksft_exit_fail_msg("Failed to allocate huge page\n"); + ksft_exit_fail_perror("Failed to allocate huge page"); pthread_create(&thread1, NULL, madv, NULL); pthread_create(&thread2, NULL, touch, NULL); @@ -120,5 +116,108 @@ int main(void) } ksft_test_result_pass("No unexpected huge page allocations\n"); +} + +/* We create a child process, then unmap the page in the parent while the child + * waits and verify that there is no underflow of the reserved count. We also + * verify that after the child exits, the reserved count is properly restored. + */ +void test_underflow(void) +{ + pid_t pid; + int pipe_fds[2]; + unsigned long nr_reserved = 0; + + huge_ptr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + + if ((unsigned long)huge_ptr == -1) + ksft_exit_fail_perror("Failed to allocate huge page"); + + nr_reserved = hugetlb_nr_resv_pages(default_huge_page_size()); + if (nr_reserved != 1) + ksft_exit_fail_msg("Unexpected number of reserved pages: %lu, expected 1\n", + nr_reserved); + + /* Force the fault to ensure the reservation is consumed */ + *huge_ptr = 0; + nr_reserved = hugetlb_nr_resv_pages(default_huge_page_size()); + if (nr_reserved != 0) + ksft_exit_fail_msg("Unexpected number of reserved pages: %lu, expected 0\n", + nr_reserved); + + if (pipe(pipe_fds) != 0) + ksft_exit_fail_perror("pipe failed"); + + pid = fork(); + if (pid < 0) + ksft_exit_fail_perror("fork failed"); + + if (pid == 0) { + /* Child: Simply wait for the parent */ + char b; + + close(pipe_fds[1]); + if (read(pipe_fds[0], &b, 1) < 0) + ksft_perror("child read failed"); + /* Let the parent do the cleanup */ + _exit(0); + } + + /* Parent */ + close(pipe_fds[0]); + + /* First unmap, this will close the vma */ + if (munmap(huge_ptr, mmap_size) != 0) { + ksft_perror("munmap failed"); + goto err_cleanup; + } + + nr_reserved = hugetlb_nr_resv_pages(default_huge_page_size()); + if (nr_reserved == 0) { + ksft_test_result_pass("Underflow not present!\n"); + } else { + ksft_test_result_fail("Unexpected HugePages_Rsvd=%ld after munmap, should be 0\n", + nr_reserved); + goto err_cleanup; + } + /* Make the child exit, this should restore HugePages_Rsvd to 0 */ + if (write(pipe_fds[1], &nr_reserved, 1) < 0) { + /* If write failed, the child is likely already gone */ + ksft_exit_fail_perror("write failed"); + } + close(pipe_fds[1]); + if (waitpid(pid, NULL, 0) <= 0) + ksft_exit_fail_msg("waitpid failed\n"); + + nr_reserved = hugetlb_nr_resv_pages(default_huge_page_size()); + if (nr_reserved == 0) + ksft_test_result_pass("After the child dies, HugePages_Rsvd is properly set to 0\n"); + else + ksft_exit_fail_msg("Unexpected HugePages_Rsvd=%ld after the child termination munmap, should be 0\n", nr_reserved); + + return; + +err_cleanup: + if (write(pipe_fds[1], &nr_reserved, 1) < 0) + ksft_exit_fail_perror("write failed"); + if (waitpid(pid, NULL, 0) <= 0) + ksft_exit_fail_perror("waitpid failed"); + + ksft_exit_fail(); +} + +int main(void) +{ + ksft_print_header(); + ksft_set_plan(3); + + if (!hugetlb_setup_default_exact(1)) + ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n", + hugetlb_free_default_pages()); + + test_madv_vs_map(); + test_underflow(); + ksft_finished(); } From 60e58be27d5e16d35e0728298f994dcae9f208a5 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 14 Sep 2026 07:23:18 -0700 Subject: [PATCH 416/539] mm/damon/sysfs-schemes: read sysfs_filter->addr_range only once Patch series "mm/damon: move damos filter range arguments validation to core". DAMOS filter range arguments are being validated in the DAMON sysfs interface. Some of those have minor time-of-check to time-of-use (TOCTOU) bugs. In future, other callers might have duplicated validations with similar bugs. Fix the bugs and further refactor the code to move the validation to the core layer. Also add kunit test cases for the validations. Patches 1 and 2 fix the existing TOCTOU bugs. Patches 3 and 4 adds the validation in the core layer. Patch 5 drops the replicated validation in DAMON sysfs interface. Patch 6 further cleanup the code. Patches 7 and 8 extends kunit tests to test the validation. This patch (of 8): DAMON sysfs interface reads the user-provided address range arguments for addr type DAMOS filter twice. Once for validation, and once again for assignments to the variable that will be passed to the core layer. If the user updates the argument in parallel, an invalid address range could be passed to the core layer. Avoid it by doing the assignments first, and then validating the assigned variables before passing those to the core layer. User impact of the bug should be trivial. From the core layer's perspective, the invalid address range is not really invalid. It just works as having a weird address range. No critical issues such as a crash or a leak could happen. And sane users ain't do such parallel arguments update anyway. If they do, such racy behavior is arguably somewhat expected and deserved. That said, there is no reason to keep such races. Link: https://lore.kernel.org/20260914142327.92510-1-sj@kernel.org Link: https://lore.kernel.org/20260914142327.92510-2-sj@kernel.org Fixes: 2f1abcfccd86 ("mm/damon/sysfs-schemes: support address range type DAMOS filter") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Usama Arif Cc: --- mm/damon/sysfs-schemes.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 3de4d804e049..3c1c1cb387fe 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2831,12 +2831,12 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, return err; } } else if (filter->type == DAMOS_FILTER_TYPE_ADDR) { - if (sysfs_filter->addr_range.end < - sysfs_filter->addr_range.start) { + filter->addr_range = sysfs_filter->addr_range; + if (filter->addr_range.end < + filter->addr_range.start) { damos_destroy_filter(filter); return -EINVAL; } - filter->addr_range = sysfs_filter->addr_range; } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) { filter->target_idx = sysfs_filter->target_idx; } else if (filter->type == DAMOS_FILTER_TYPE_HUGEPAGE_SIZE) { From af19aff38d6faea35b10f26cca32d0e54f8aea5e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 14 Sep 2026 07:23:19 -0700 Subject: [PATCH 417/539] mm/damon/sysfs-schemes: read sysfs_filter->sz_range only once DAMON sysfs interface reads the user-provided size range arguments for hugepage_size type DAMOS filter twice. Once for validation, and once again for assignments to the variable that will be passed to the core layer. If the user updates the arguments in parallel, an invalid size range could be passed to the core layer. Avoid it by doing the assignments first, and then validating the assigned variables before passing those to the core layer. User impact of the bug should be trivial. From the core layer's perspective, the invalid size range is not really invalid. It just works as having a weird size range. No critical issues such as a crash or a leak could happen. And sane users ain't do such parallel arguments update anyway. If they do, such racy behavior is arguably somewhat expected and deserved. That said, there is no reason to keep such races. Link: https://lore.kernel.org/20260914142327.92510-3-sj@kernel.org Fixes: ea1f204ba29a ("mm/damon/sysfs-schemes: add files for setting damos_filter->sz_range") Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Usama Arif Cc: --- mm/damon/sysfs-schemes.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 3c1c1cb387fe..8c8ab82c8fac 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2840,13 +2840,12 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) { filter->target_idx = sysfs_filter->target_idx; } else if (filter->type == DAMOS_FILTER_TYPE_HUGEPAGE_SIZE) { - if (sysfs_filter->range_min > - sysfs_filter->range_max) { + filter->sz_range.min = sysfs_filter->range_min; + filter->sz_range.max = sysfs_filter->range_max; + if (filter->range_min > filter->range_max) { damos_destroy_filter(filter); return -EINVAL; } - filter->sz_range.min = sysfs_filter->range_min; - filter->sz_range.max = sysfs_filter->range_max; } else if (filter->type == DAMOS_FILTER_TYPE_PROBE_HITS_WSUM) { filter->range_min = sysfs_filter->range_min; filter->range_max = sysfs_filter->range_max; From 6f276422aded370174edea69549ebe6b1683ffcc Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 14 Sep 2026 07:23:20 -0700 Subject: [PATCH 418/539] mm/damon/core: return an error from damos_commit_filter_arg() damos_commit_filter_arg() is supposed to always succeed. It may not in future, for example, if the given filter is invalid. Prepare the case by modifying its signature to return an error when it failed. Also pipe the return value to its callers and let them handle the error. Link: https://lore.kernel.org/20260914142327.92510-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Kunwu Chan Cc: Brendan Higgins Cc: David Gow Cc: Usama Arif Cc: --- mm/damon/core.c | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 38383ced3dd6..1932a5025207 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1317,7 +1317,7 @@ static struct damos_filter *damos_nth_ops_filter(int n, struct damos *s) return NULL; } -static void damos_commit_filter_arg( +static int damos_commit_filter_arg( struct damos_filter *dst, struct damos_filter *src) { switch (dst->type) { @@ -1340,28 +1340,32 @@ static void damos_commit_filter_arg( default: break; } + return 0; } -static void damos_commit_filter( +static int damos_commit_filter( struct damos_filter *dst, struct damos_filter *src) { dst->type = src->type; dst->matching = src->matching; dst->allow = src->allow; - damos_commit_filter_arg(dst, src); + return damos_commit_filter_arg(dst, src); } static int damos_commit_core_filters(struct damos *dst, struct damos *src) { struct damos_filter *dst_filter, *next, *src_filter, *new_filter; - int i = 0, j = 0; + int i = 0, j = 0, err; damos_for_each_core_filter_safe(dst_filter, next, dst) { src_filter = damos_nth_core_filter(i++, src); - if (src_filter) - damos_commit_filter(dst_filter, src_filter); - else + if (src_filter) { + err = damos_commit_filter(dst_filter, src_filter); + if (err) + return err; + } else { damos_destroy_filter(dst_filter); + } } damos_for_each_core_filter_safe(src_filter, next, src) { @@ -1373,7 +1377,11 @@ static int damos_commit_core_filters(struct damos *dst, struct damos *src) src_filter->allow); if (!new_filter) return -ENOMEM; - damos_commit_filter_arg(new_filter, src_filter); + err = damos_commit_filter_arg(new_filter, src_filter); + if (err) { + damos_destroy_filter(new_filter); + return err; + } damos_add_filter(dst, new_filter); } return 0; @@ -1382,14 +1390,17 @@ static int damos_commit_core_filters(struct damos *dst, struct damos *src) static int damos_commit_ops_filters(struct damos *dst, struct damos *src) { struct damos_filter *dst_filter, *next, *src_filter, *new_filter; - int i = 0, j = 0; + int i = 0, j = 0, err; damos_for_each_ops_filter_safe(dst_filter, next, dst) { src_filter = damos_nth_ops_filter(i++, src); - if (src_filter) - damos_commit_filter(dst_filter, src_filter); - else + if (src_filter) { + err = damos_commit_filter(dst_filter, src_filter); + if (err) + return err; + } else { damos_destroy_filter(dst_filter); + } } damos_for_each_ops_filter_safe(src_filter, next, src) { @@ -1401,7 +1412,11 @@ static int damos_commit_ops_filters(struct damos *dst, struct damos *src) src_filter->allow); if (!new_filter) return -ENOMEM; - damos_commit_filter_arg(new_filter, src_filter); + err = damos_commit_filter_arg(new_filter, src_filter); + if (err) { + damos_destroy_filter(new_filter); + return err; + } damos_add_filter(dst, new_filter); } return 0; From 1bfb9384ae0ed52d61eb601b655014b7815ff849 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 14 Sep 2026 07:23:21 -0700 Subject: [PATCH 419/539] mm/damon/core: disallow max < min damos filter range arguments commit damos_commit_filter_arg() receives range arguments for a few types of DAMOS filters. It allows any range including max < min range. It is fine for the logic, but makes no sense to support it. Actually DAMON sysfs interface is doing the validation on its own. To avoid duplicated validations in multiple DAMON API callers, it would be better to do the validation in the core layer. Add a validation of the given range. Link: https://lore.kernel.org/20260914142327.92510-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Usama Arif Cc: --- mm/damon/core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 1932a5025207..5fdacb9dcee5 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1325,15 +1325,21 @@ static int damos_commit_filter_arg( dst->memcg_id = src->memcg_id; break; case DAMOS_FILTER_TYPE_ADDR: + if (src->addr_range.end < src->addr_range.start) + return -EINVAL; dst->addr_range = src->addr_range; break; case DAMOS_FILTER_TYPE_TARGET: dst->target_idx = src->target_idx; break; case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE: + if (src->sz_range.max < src->sz_range.min) + return -EINVAL; dst->sz_range = src->sz_range; break; case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: + if (src->range_max < src->range_min) + return -EINVAL; dst->range_min = src->range_min; dst->range_max = src->range_max; break; From 1d2397b2c414e3f1fb536666ccc21d4065a3d9a6 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 14 Sep 2026 07:23:22 -0700 Subject: [PATCH 420/539] mm/damon/sysfs-schemes: drop centralized filter range arg validations DAMON sysfs interface is validating wrong range arguments for DAMOS filters. Now the core layer is doing the same validation. Drop the duplicated validation in DAMON sysfs interface. Link: https://lore.kernel.org/20260914142327.92510-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Usama Arif Cc: --- mm/damon/sysfs-schemes.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 8c8ab82c8fac..a4ec5d54cfbd 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2832,27 +2832,14 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, } } else if (filter->type == DAMOS_FILTER_TYPE_ADDR) { filter->addr_range = sysfs_filter->addr_range; - if (filter->addr_range.end < - filter->addr_range.start) { - damos_destroy_filter(filter); - return -EINVAL; - } } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) { filter->target_idx = sysfs_filter->target_idx; } else if (filter->type == DAMOS_FILTER_TYPE_HUGEPAGE_SIZE) { filter->sz_range.min = sysfs_filter->range_min; filter->sz_range.max = sysfs_filter->range_max; - if (filter->range_min > filter->range_max) { - damos_destroy_filter(filter); - return -EINVAL; - } } else if (filter->type == DAMOS_FILTER_TYPE_PROBE_HITS_WSUM) { filter->range_min = sysfs_filter->range_min; filter->range_max = sysfs_filter->range_max; - if (filter->range_min > filter->range_max) { - damos_destroy_filter(filter); - return -EINVAL; - } } damos_add_filter(scheme, filter); From cec08aad3def79bdb7a795a05e7c9a11d577b4f8 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 14 Sep 2026 07:23:23 -0700 Subject: [PATCH 421/539] mm/damon/sysfs-schemes: use switch-case in add_scheme_filters() damon_sysfs_add_scheeme_filters() has long if-else chains for DAMOS filter types. Convert the code to use switch-case, which would be cleaner and more efficient. Link: https://lore.kernel.org/20260914142327.92510-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Usama Arif Cc: --- mm/damon/sysfs-schemes.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index a4ec5d54cfbd..bfb6f0bc3f21 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2822,7 +2822,8 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, if (!filter) return -ENOMEM; - if (filter->type == DAMOS_FILTER_TYPE_MEMCG) { + switch (filter->type) { + case DAMOS_FILTER_TYPE_MEMCG: err = damon_sysfs_memcg_path_to_id( sysfs_filter->memcg_path, &filter->memcg_id); @@ -2830,16 +2831,23 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme, damos_destroy_filter(filter); return err; } - } else if (filter->type == DAMOS_FILTER_TYPE_ADDR) { + break; + case DAMOS_FILTER_TYPE_ADDR: filter->addr_range = sysfs_filter->addr_range; - } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) { + break; + case DAMOS_FILTER_TYPE_TARGET: filter->target_idx = sysfs_filter->target_idx; - } else if (filter->type == DAMOS_FILTER_TYPE_HUGEPAGE_SIZE) { + break; + case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE: filter->sz_range.min = sysfs_filter->range_min; filter->sz_range.max = sysfs_filter->range_max; - } else if (filter->type == DAMOS_FILTER_TYPE_PROBE_HITS_WSUM) { + break; + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: filter->range_min = sysfs_filter->range_min; filter->range_max = sysfs_filter->range_max; + break; + default: + break; } damos_add_filter(scheme, filter); From a1c7fdaf165755fff928a6c25b69d7e0966b6910 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 14 Sep 2026 07:23:24 -0700 Subject: [PATCH 422/539] mm/damon/core-kunit: extend damos_commit_filter_for() for wrong input damos_commit_filter_for() supposes damos_commit_filter() to always succeed with given inputs. damos_commit_filter() could return an error for invalid inputs. Existing callers always pass only valid inputs, but they may pass invalid inputs in future, for test purposes. Extend the function to be able to be used for wrong inputs-caused error testing. Link: https://lore.kernel.org/20260914142327.92510-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Usama Arif Cc: --- mm/damon/tests/core-kunit.h | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index c01e6a75cadc..2b0931cf6fb3 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1116,9 +1116,15 @@ static void damos_test_commit_dests(struct kunit *test) } static void damos_test_commit_filter_for(struct kunit *test, - struct damos_filter *dst, struct damos_filter *src) + struct damos_filter *dst, struct damos_filter *src, + bool expect_fail) { - damos_commit_filter(dst, src); + int err; + + err = damos_commit_filter(dst, src); + KUNIT_EXPECT_EQ(test, err != 0, expect_fail); + if (expect_fail) + return; KUNIT_EXPECT_EQ(test, dst->type, src->type); KUNIT_EXPECT_EQ(test, dst->matching, src->matching); KUNIT_EXPECT_EQ(test, dst->allow, src->allow); @@ -1157,47 +1163,47 @@ static void damos_test_commit_filter(struct kunit *test) .type = DAMOS_FILTER_TYPE_ANON, .matching = true, .allow = true, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_MEMCG, .matching = false, .allow = false, .memcg_id = 123, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_YOUNG, .matching = true, .allow = true, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_HUGEPAGE_SIZE, .matching = false, .allow = false, .sz_range = {.min = 234, .max = 345}, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_UNMAPPED, .matching = true, .allow = true, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_ADDR, .matching = false, .allow = false, .addr_range = {.start = 456, .end = 567}, - }); + }, false); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_TARGET, .matching = true, .allow = true, .target_idx = 6, - }); + }, false); } static void damos_test_help_initailize_scheme(struct damos *scheme) From 409d2ab6d98209c30d45123920721710475a30b4 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 14 Sep 2026 07:23:25 -0700 Subject: [PATCH 423/539] mm/damon/core-kunit: test invalid damos filter commits Add test cases for testing the validation of damos filter arguments in commit time. Link: https://lore.kernel.org/20260914142327.92510-9-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: Usama Arif Cc: --- mm/damon/tests/core-kunit.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 2b0931cf6fb3..527abc257061 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1184,6 +1184,13 @@ static void damos_test_commit_filter(struct kunit *test) .allow = false, .sz_range = {.min = 234, .max = 345}, }, false); + damos_test_commit_filter_for(test, &dst, + &(struct damos_filter){ + .type = DAMOS_FILTER_TYPE_HUGEPAGE_SIZE, + .matching = false, + .allow = false, + .sz_range = {.min = 456, .max = 123}, + }, true); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_UNMAPPED, @@ -1197,6 +1204,13 @@ static void damos_test_commit_filter(struct kunit *test) .allow = false, .addr_range = {.start = 456, .end = 567}, }, false); + damos_test_commit_filter_for(test, &dst, + &(struct damos_filter){ + .type = DAMOS_FILTER_TYPE_ADDR, + .matching = false, + .allow = false, + .addr_range = {.start = 567, .end = 456}, + }, true); damos_test_commit_filter_for(test, &dst, &(struct damos_filter){ .type = DAMOS_FILTER_TYPE_TARGET, From 65c7e923ed1afa7329f8b910327ebf6ae10a5fb6 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Mon, 14 Sep 2026 07:19:46 -0700 Subject: [PATCH 424/539] selftests/damon: stop kdamond on error exits of no-op commit test Patch series "mm/damon: misc improvements in tests and documents". Add various improvements to DAMON tests and documents. Patch 1 and 2 from Zenghui Yu (Huawei) let DAMON selftest to clean up its state and output after running the tests. Patch 3 from Eva Kurchatova makes a few selftest be able to run with Python's safe path mode. Patch 4 from Kunwu Chan improves coverage of DAMON kunit tests. Finally, patches 5 and 6 from Liew Rui Yan clarify and fix typos on documents for DAMOS quota behaviors. Below are notes that can be removed in the final commit message. This is a batched repost of DAMON patches that were individually posted. Please refer to each patch for changelog. This patch (of 6): The sysfs_no_op_commit_break test starts a kdamond via sysfs, but its error paths (e.g., drgn not installed) exit without stopping it. The leaked kdamond then makes subsequent tests, e.g. lru_sort.sh and reclaim.sh, skip with "Another kdamond is running". Wrap the post-start logic in try-finally so that kdamonds.stop() is executed on every exit path. Link: https://lore.kernel.org/20260914141952.91465-1-sj@kernel.org Link: https://lore.kernel.org/20260914141952.91465-2-sj@kernel.org Fixes: 10725cd2b09a ("selftests/damon: test no-op commit broke DAMON status") Signed-off-by: Zenghui Yu (Huawei) Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton Assisted-by: GLM-5.3 OpenCode Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Eva Kurchatova Cc: Kunwu Chan Cc: Lian Wang Cc: Liew Rui Yan --- .../damon/sysfs_no_op_commit_break.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tools/testing/selftests/damon/sysfs_no_op_commit_break.py b/tools/testing/selftests/damon/sysfs_no_op_commit_break.py index 2c65cffe6b54..81774ddc55ac 100755 --- a/tools/testing/selftests/damon/sysfs_no_op_commit_break.py +++ b/tools/testing/selftests/damon/sysfs_no_op_commit_break.py @@ -47,26 +47,27 @@ def main(): print('kdamond start failed: %s' % err) exit(1) - before_commit_status, err = \ - dump_damon_status_dict(kdamonds.kdamonds[0].pid) - if err is not None: - print('before-commit status dump failed: %s' % err) - exit(1) + try: + before_commit_status, err = \ + dump_damon_status_dict(kdamonds.kdamonds[0].pid) + if err is not None: + print('before-commit status dump failed: %s' % err) + exit(1) - kdamonds.kdamonds[0].commit() + kdamonds.kdamonds[0].commit() - after_commit_status, err = \ - dump_damon_status_dict(kdamonds.kdamonds[0].pid) - if err is not None: - print('after-commit status dump failed: %s' % err) - exit(1) + after_commit_status, err = \ + dump_damon_status_dict(kdamonds.kdamonds[0].pid) + if err is not None: + print('after-commit status dump failed: %s' % err) + exit(1) - if before_commit_status != after_commit_status: - print(f'before: {json.dumps(before_commit_status, indent=2)}') - print(f'after: {json.dumps(after_commit_status, indent=2)}') - exit(1) - - kdamonds.stop() + if before_commit_status != after_commit_status: + print(f'before: {json.dumps(before_commit_status, indent=2)}') + print(f'after: {json.dumps(after_commit_status, indent=2)}') + exit(1) + finally: + kdamonds.stop() if __name__ == '__main__': main() From b5c7c37cd0407b186c1d8f25d886aecf011b8c74 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Mon, 14 Sep 2026 07:19:47 -0700 Subject: [PATCH 425/539] selftests/damon: ignore test-generated damon_dump_output sysfs.py and sysfs_no_op_commit_break.py dump the DAMON status collected via drgn_dump_damon_status.py into a damon_dump_output file in the working directory. Ignore it so that running the tests in-tree does not pollute git status. Link: https://lore.kernel.org/20260914141952.91465-3-sj@kernel.org Signed-off-by: Zenghui Yu (Huawei) Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Eva Kurchatova Cc: Jonathan Corbet Cc: Kunwu Chan Cc: Liam R. Howlett Cc: Lian Wang Cc: Liew Rui Yan Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/damon/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/damon/.gitignore b/tools/testing/selftests/damon/.gitignore index 2f0297657c81..226bd7c4e072 100644 --- a/tools/testing/selftests/damon/.gitignore +++ b/tools/testing/selftests/damon/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only access_memory access_memory_even +damon_dump_output From cd87d9b56bf371be7d9208da395708bf13a67457 Mon Sep 17 00:00:00 2001 From: Eva Kurchatova Date: Mon, 14 Sep 2026 07:19:48 -0700 Subject: [PATCH 426/539] selftests/damon: add script dir to sys.path for PYTHONSAFEPATH compatibility Running these tests under Python's safe path mode, either with -P on the interpreter command line or with PYTHONSAFEPATH set in the environment, stops the script's own directory being prepended to sys.path. Some distributions (RHEL, for example) build the tests with -P in the shebang. This breaks all 7 DAMON Python selftests that import the _damon_sysfs helper module located in the same directory: ModuleNotFoundError: No module named '_damon_sysfs' Fix this by explicitly adding the script's directory to sys.path before importing _damon_sysfs, following the same pattern used in commit c3b3eb565bd7 ("tools: ynl: add script dir to sys.path") which fixed the identical issue for the YNL tools. Link: https://lore.kernel.org/20260914141952.91465-4-sj@kernel.org Signed-off-by: Eva Kurchatova Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Kunwu Chan Cc: Liam R. Howlett Cc: Lian Wang Cc: Liew Rui Yan Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: "Zenghui Yu (Huawei)" --- tools/testing/selftests/damon/damon_nr_regions.py | 3 +++ tools/testing/selftests/damon/damos_apply_interval.py | 3 +++ tools/testing/selftests/damon/damos_quota.py | 3 +++ tools/testing/selftests/damon/damos_quota_goal.py | 3 +++ tools/testing/selftests/damon/damos_tried_regions.py | 3 +++ .../selftests/damon/sysfs_update_schemes_tried_regions_hang.py | 3 +++ .../damon/sysfs_update_schemes_tried_regions_wss_estimation.py | 3 +++ 7 files changed, 21 insertions(+) diff --git a/tools/testing/selftests/damon/damon_nr_regions.py b/tools/testing/selftests/damon/damon_nr_regions.py index 58f3291fed12..e55239813c65 100755 --- a/tools/testing/selftests/damon/damon_nr_regions.py +++ b/tools/testing/selftests/damon/damon_nr_regions.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def test_nr_regions(real_nr_regions, min_nr_regions, max_nr_regions): diff --git a/tools/testing/selftests/damon/damos_apply_interval.py b/tools/testing/selftests/damon/damos_apply_interval.py index 0f2f36584e48..0bf7768b2006 100755 --- a/tools/testing/selftests/damon/damos_apply_interval.py +++ b/tools/testing/selftests/damon/damos_apply_interval.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/damos_quota.py b/tools/testing/selftests/damon/damos_quota.py index 57c4937aaed2..879115a499bf 100755 --- a/tools/testing/selftests/damon/damos_quota.py +++ b/tools/testing/selftests/damon/damos_quota.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/damos_quota_goal.py b/tools/testing/selftests/damon/damos_quota_goal.py index 661e4ba4765a..fed033a0afdf 100755 --- a/tools/testing/selftests/damon/damos_quota_goal.py +++ b/tools/testing/selftests/damon/damos_quota_goal.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/damos_tried_regions.py b/tools/testing/selftests/damon/damos_tried_regions.py index d6472e6a6e08..6941f87c10b1 100755 --- a/tools/testing/selftests/damon/damos_tried_regions.py +++ b/tools/testing/selftests/damon/damos_tried_regions.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_hang.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_hang.py index 28c887a0108f..625761c243b5 100755 --- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_hang.py +++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_hang.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def main(): diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py index 16fdc6e7fc56..36e7ae5f826d 100755 --- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py +++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +import os import subprocess +import sys import time +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import _damon_sysfs def pass_wss_estimation(sz_region): From a3211bc2912020d51e0454b7b7a9c64684e0545d Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 14 Sep 2026 07:19:49 -0700 Subject: [PATCH 427/539] mm/damon/tests/core-kunit: improve nr_samples_per_aggr test isolation Test the zero sample interval case with a non-zero aggregation interval, and keep the zero/zero case to cover the zero-result fallback. Also make the overflow case use an explicit sample interval so that it does not depend on the zero sample interval fallback. Link: https://lore.kernel.org/20260914141952.91465-5-sj@kernel.org Signed-off-by: Kunwu Chan Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Lian Wang Reviewed-by: SJ Park Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Eva Kurchatova Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Liew Rui Yan Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: "Zenghui Yu (Huawei)" --- mm/damon/tests/core-kunit.h | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 527abc257061..5da84caf4124 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -627,12 +627,20 @@ static void damon_test_set_regions(struct kunit *test) static void damon_test_nr_samples_per_aggr(struct kunit *test) { - struct damon_attrs attrs = { + struct damon_attrs attrs; + + /* Zero sample interval is treated as one. */ + attrs = (struct damon_attrs){ + .sample_interval = 0, + .aggr_interval = 5000, + }; + KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), 5000); + + /* Zero sample and aggregation intervals cover the zero-result fallback. */ + attrs = (struct damon_attrs){ .sample_interval = 0, .aggr_interval = 0, }; - - /* Zero aggregation interval doesn't cause division by zero */ KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), 1); /* @@ -640,7 +648,10 @@ static void damon_test_nr_samples_per_aggr(struct kunit *test) * overflow */ if (ULONG_MAX > UINT_MAX) { - attrs.aggr_interval = (unsigned long)UINT_MAX + 1; + attrs = (struct damon_attrs){ + .sample_interval = 1, + .aggr_interval = (unsigned long)UINT_MAX + 1, + }; KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), UINT_MAX); } From 3e8a5d6952100b756ec7624fcde687270ccef363 Mon Sep 17 00:00:00 2001 From: Liew Rui Yan Date: Mon, 14 Sep 2026 07:19:50 -0700 Subject: [PATCH 428/539] Docs/mm/damon/design: clarify when qt_exceeds increases qt_exceeds counts how many times a scheme's quota has been exceeded. When using the temporal auto-tuning algorithm, the effective size quota becomes zero once the goal is [over-]achieved. In this case, the quotas are still set, so qt_exceeds keeps increasing once per quota reset interval while the goal stays achieved. Clarify this behavior in the design document. Link: https://lore.kernel.org/20260914141952.91465-6-sj@kernel.org Signed-off-by: Liew Rui Yan Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Eva Kurchatova Cc: Jonathan Corbet Cc: Kunwu Chan Cc: Liam R. Howlett Cc: Lian Wang Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: "Zenghui Yu (Huawei)" --- Documentation/mm/damon/design.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index cb116a82ff20..ad09416dfb21 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -696,7 +696,9 @@ There are two such tuning algorithms that users can select as they need. fast as possible, using maximum allowed quota, but only for a temporal short time. When the quota is under-achieved, this algorithm keeps tuning quota to a maximum allowed one. Once the quota is [over]-achieved, this sets the - quota zero. Useful for deterministic control required environments. + quota zero. Useful for deterministic control required environments. Note + that the zero quota is a valid quota, and therefore ``qt_exceeds`` :ref:`stat + ` will keep increasing in this case. The goal can be specified with five parameters, namely ``target_metric``, ``target_value``, ``current_value``, ``nid`` and ``path``. The auto-tuning From 763b77347bb58c56a486db867813f45305ba6759 Mon Sep 17 00:00:00 2001 From: Liew Rui Yan Date: Mon, 14 Sep 2026 07:19:51 -0700 Subject: [PATCH 429/539] Docs/mm/damon/design: fix typos in temporal auto-tuning algorithm section Correct incorrect references of "quota" to "goal" in the temporal auto-tuning algorithm description and fix the square bracket formatting. Link: https://lore.kernel.org/20260914141952.91465-7-sj@kernel.org Signed-off-by: Liew Rui Yan Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Eva Kurchatova Cc: Jonathan Corbet Cc: Kunwu Chan Cc: Liam R. Howlett Cc: Lian Wang Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: "Zenghui Yu (Huawei)" --- Documentation/mm/damon/design.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index ad09416dfb21..707170bcd1b3 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -694,8 +694,8 @@ There are two such tuning algorithms that users can select as they need. This is the default selection. If unsure, use this. - ``temporal``: More straightforward algorithm. Tries to achieve the goal as fast as possible, using maximum allowed quota, but only for a temporal short - time. When the quota is under-achieved, this algorithm keeps tuning quota to - a maximum allowed one. Once the quota is [over]-achieved, this sets the + time. When the goal is under-achieved, this algorithm keeps tuning quota to + a maximum allowed one. Once the goal is [over-]achieved, this sets the quota zero. Useful for deterministic control required environments. Note that the zero quota is a valid quota, and therefore ``qt_exceeds`` :ref:`stat ` will keep increasing in this case. From d350956d829f4b620faf294c86116de640f1518c Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sat, 12 Sep 2026 08:28:22 -0400 Subject: [PATCH 430/539] proc/task_mmu: handle special PMDs in clear_refs and pagemap mshv_vtl_low can install a special PMD from a PFN supplied in the mmap file offset. That PFN need not have a memmap entry. Reading /proc/PID/pagemap or writing /proc/PID/clear_refs for such a mapping oopses the kernel. With a test driver mapping the PFN at the 1 TiB mark: echo 1 > /proc/self/clear_refs BUG: unable to handle page fault for address: fffffb5b80000008 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page RIP: 0010:clear_refs_pte_range+0xb1/0x1e0 Call Trace: walk_pgd_range+0x50a/0xad0 __walk_page_range+0x6a/0x1d0 walk_page_range_mm_unsafe+0x193/0x230 clear_refs_write+0x18e/0x3f0 pread(pagemap_fd, buf, 512 * 8, addr / PAGE_SIZE * 8) BUG: unable to handle page fault for address: fffff57840000008 RIP: 0010:pagemap_pmd_range+0x3cf/0x6b0 Call Trace: walk_pgd_range+0x50a/0xad0 __walk_page_range+0x6a/0x1d0 walk_page_range_mm_unsafe+0x193/0x230 pagemap_read+0x1dc/0x350 clear_refs_pte_range() passes the PMD to pmd_folio(), while pagemap_pmd_range_thp() uses pmd_page() followed by page_folio(). Both paths assume the PFN has a struct page and dereference bad address. Use vm_normal_folio_pmd() and vm_normal_page_pmd() so clear_refs skips folio operations and pagemap skips folio-derived flags for special PMDs. This matches the PTE paths, which already use vm_normal_folio() and vm_normal_page(). User-facing notes: - Pagemap still reports PM_PRESENT and the PFN to a CAP_SYS_ADMIN reader. - For a special PMD backed by a valid memmap entry and for the huge zero PMD, PM_FILE changes from set to clear. - PM_MMAP_EXCLUSIVE is already clear in both cases. - The output for an anonymous THP is unchanged. Tested in QEMU with the test driver on broken and fixed kernels. Both proc operations oops before the fix and return 0 after it. This is technically reachable via mshv_vtl_low, so I tagged it stable. Link: https://lore.kernel.org/20260912122822.3348978-1-gourry@gourry.net Fixes: 3c8e44c9b369 ("mm: mark special bits for huge pfn mappings when inject") Signed-off-by: Gregory Price Signed-off-by: Andrew Morton Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260912034833.2952750-1-gourry%40gourry.net Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Jann Horn Cc: Jason Gunthorpe Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Pedro Falcato Cc: Peter Xu Cc: Vlastimil Babka Cc: # v6.19+ --- fs/proc/task_mmu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index e671b4fd8ded..78e00c6d4e2d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1704,7 +1704,9 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, if (!pmd_present(*pmd)) goto out; - folio = pmd_folio(*pmd); + folio = vm_normal_folio_pmd(vma, addr, *pmd); + if (!folio) + goto out; /* Clear accessed and referenced bits. */ pmdp_test_and_clear_young(vma, addr, pmd); @@ -2024,7 +2026,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, goto populate_pagemap; if (pmd_present(pmd)) { - page = pmd_page(pmd); + page = vm_normal_page_pmd(vma, addr, pmd); flags |= PM_PRESENT; if (pmd_soft_dirty(pmd)) From ebb07f8a8b0a8b651360d9ec6b2848b71e0534a0 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 15 Sep 2026 10:43:30 +0800 Subject: [PATCH 431/539] mm/vmalloc: group xa_init with vbq field initializations Patch series "mm/vmalloc: minor cleanups", v2. Small cleanup series for mm/vmalloc.c, no functional changes: - Group xa_init() with the other vmap_block_queue field initializations in vmalloc_init(), instead of after the unrelated vfree_deferred setup. - Extract vmap_insert_free_area() helper to deduplicate the allocate- and-insert pattern that appeared both inside the loop body and after the loop in vmap_init_free_space(). - Extract show_busy_info() from vmalloc_info_show(), mirroring the existing show_purge_info() pattern, so the top-level show function only orchestrates the two data sources. This patch (of 3): Move xa_init() next to the other vbq field initializations instead of after the unrelated vfree_deferred setup. Link: https://lore.kernel.org/20260915-vmalloc_study-v2-0-cc4dfe635e22@linux.dev Link: https://lore.kernel.org/20260915-vmalloc_study-v2-1-cc4dfe635e22@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Reviewed-by: Uladzislau Rezki (Sony) --- mm/vmalloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 3845c7bd72b4..e6d434523aec 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -5561,10 +5561,11 @@ void __init vmalloc_init(void) vbq = &per_cpu(vmap_block_queue, i); spin_lock_init(&vbq->lock); INIT_LIST_HEAD(&vbq->free); + xa_init(&vbq->vmap_blocks); + p = &per_cpu(vfree_deferred, i); init_llist_head(&p->list); INIT_WORK(&p->wq, delayed_vfree_work); - xa_init(&vbq->vmap_blocks); } /* From c10ed0f2315a766c4e0932840886f3dc0a17a9c9 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 15 Sep 2026 10:43:31 +0800 Subject: [PATCH 432/539] mm/vmalloc: extract vmap_insert_free_area helper The allocation and insertion of a free vmap_area is duplicated between the loop body and the tail of vmap_init_free_space. Factor it into a small helper so the main function only deals with computing the free gaps between busy regions. Link: https://lore.kernel.org/20260915-vmalloc_study-v2-2-cc4dfe635e22@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Reviewed-by: Uladzislau Rezki (Sony) Reviewed-by: Dev Jain --- mm/vmalloc.c | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index e6d434523aec..419e6ff9b70d 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -5419,11 +5419,23 @@ module_init(proc_vmalloc_init); #endif +static void __init vmap_insert_free_area(unsigned long start, unsigned long end) +{ + struct vmap_area *free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); + + if (!WARN_ON_ONCE(!free)) { + free->va_start = start; + free->va_end = end; + insert_vmap_area_augment(free, NULL, + &free_vmap_area_root, + &free_vmap_area_list); + } +} + static void __init vmap_init_free_space(void) { unsigned long vmap_start = 1; const unsigned long vmap_end = ULONG_MAX; - struct vmap_area *free; struct vm_struct *busy; /* @@ -5433,32 +5445,15 @@ static void __init vmap_init_free_space(void) * |<--------------------------------->| */ for (busy = vmlist; busy; busy = busy->next) { - if ((unsigned long) busy->addr - vmap_start > 0) { - free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); - if (!WARN_ON_ONCE(!free)) { - free->va_start = vmap_start; - free->va_end = (unsigned long) busy->addr; - - insert_vmap_area_augment(free, NULL, - &free_vmap_area_root, - &free_vmap_area_list); - } - } + if ((unsigned long) busy->addr - vmap_start > 0) + vmap_insert_free_area(vmap_start, + (unsigned long) busy->addr); vmap_start = (unsigned long) busy->addr + busy->size; } - if (vmap_end - vmap_start > 0) { - free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); - if (!WARN_ON_ONCE(!free)) { - free->va_start = vmap_start; - free->va_end = vmap_end; - - insert_vmap_area_augment(free, NULL, - &free_vmap_area_root, - &free_vmap_area_list); - } - } + if (vmap_end - vmap_start > 0) + vmap_insert_free_area(vmap_start, vmap_end); } static void vmap_init_nodes(void) From 3b2384226404cf388c3e52a38dac3acb08b17b45 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 15 Sep 2026 10:43:32 +0800 Subject: [PATCH 433/539] mm/vmalloc: extract show_busy_info from vmalloc_info_show Extract the busy vmap area iteration into show_busy_info, mirroring the existing show_purge_info pattern. Link: https://lore.kernel.org/20260915-vmalloc_study-v2-3-cc4dfe635e22@linux.dev Signed-off-by: Ye Liu Signed-off-by: Andrew Morton Reviewed-by: Uladzislau Rezki (Sony) --- mm/vmalloc.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 419e6ff9b70d..547864233128 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -5331,7 +5331,7 @@ static void show_purge_info(struct seq_file *m) } } -static int vmalloc_info_show(struct seq_file *m, void *p) +static void show_busy_info(struct seq_file *m) { struct vmap_node *vn; struct vmap_area *va; @@ -5401,12 +5401,18 @@ static int vmalloc_info_show(struct seq_file *m, void *p) spin_unlock(&vn->busy.lock); } + if (IS_ENABLED(CONFIG_NUMA)) + kfree(counters); +} + +static int vmalloc_info_show(struct seq_file *m, void *p) +{ + show_busy_info(m); + /* * As a final step, dump "unpurged" areas. */ show_purge_info(m); - if (IS_ENABLED(CONFIG_NUMA)) - kfree(counters); return 0; } From 43f046860e81d0a6244a5d91a585289e512f6eef Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 12 Sep 2026 13:48:59 +0200 Subject: [PATCH 434/539] mm/secretmem: fix the enable parameter description The module parameter is enable, but its MODULE_PARM_DESC() names the variable behind it, secretmem_enable. secretmem is built in, so the description only reaches modules.builtin.modinfo, where it is filed under secretmem_enable while the parmtype line and /sys/module/secretmem/parameters/ say enable. Use the parameter name in the description. Link: https://lore.kernel.org/20260912114859.88957-1-kmehltretter@gmail.com Fixes: 1507f51255c9 ("mm: introduce memfd_secret system call to create "secret" memory areas") Signed-off-by: Karl Mehltretter Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Assisted-by: LLM --- mm/secretmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/secretmem.c b/mm/secretmem.c index 6cbb8efc994a..7287a2866897 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -39,7 +39,7 @@ static bool secretmem_enable __ro_after_init = 1; module_param_named(enable, secretmem_enable, bool, 0400); -MODULE_PARM_DESC(secretmem_enable, +MODULE_PARM_DESC(enable, "Enable secretmem and memfd_secret(2) system call"); static atomic_t secretmem_users; From 5445a6359d717354dd76d509b8e693f13403ec61 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sat, 12 Sep 2026 07:08:32 -0400 Subject: [PATCH 435/539] mm/madvise: reclaim isolated folios if PTE restart fails MADV_PAGEOUT collects isolated folios on a local list before reclaiming them after the PTE walk. The reschedule path drops the PTE lock and then restarts the mapping with pte_offset_map_lock(). A concurrent operation can remove or replace the PTE table while the lock is dropped, causing pte_offset_map_lock() to return NULL. Returning directly in that case bypasses reclaim_pages(), leaving the collected folios off the LRU with elevated references. This results in a permanent memory leak. nr_isolated_anon increases reliably and does not decrease when the process dies. Route the failure through the existing cleanup path so any isolated folios are reclaimed or put back. Simplest userland pseudo-code reproducer: p = mmap(PMD_SIZE, ANONYMOUS); touch_every_page(p, PMD_SIZE); parallel { while (1) madvise(p, PMD_SIZE, MADV_PAGEOUT); while (1) { madvise(p, PMD_SIZE, MADV_DONTNEED); touch_every_page(p, PMD_SIZE); } } Reproduced in qemu trivially with some explicit widening of the race window. Link: https://lore.kernel.org/20260912110832.3203902-1-gourry@gourry.net Fixes: b2f557a21bc8 ("mm/madvise: add cond_resched() in madvise_cold_or_pageout_pte_range()") Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260821150912.183976-1-gourry@gourry.net Reviewed-by: Lorenzo Stoakes (ARM) Assisted-by: LLM Cc: David Hildenbrand Cc: Jann Horn Cc: Jiexun Wang Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: --- mm/madvise.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/madvise.c b/mm/madvise.c index 6c52f699222b..8943ae181144 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -462,7 +462,7 @@ regular_folio: restart: start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!start_pte) - return 0; + goto out; flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { @@ -566,6 +566,7 @@ restart: folio_deactivate(folio); } +out: if (start_pte) { lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); From adbb0adb434d55bd02b18c8de6356f954a077c79 Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Meta)" Date: Mon, 14 Sep 2026 21:36:39 +0100 Subject: [PATCH 436/539] selftests/cgroup: ignore memory.reclaim -EAGAIN for zswap writeback test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zswap_writeback_enabled test fails when a write to memory.reclaim returns -EAGAIN, which means less than the requested amount was reclaimed. attempt_writeback() propagates the -EAGAIN to the caller, and the test case is marked as failed even when zswap writeback did happen. This heavily depends on the performance of the backing swap device. Reclaim does not wait for writeback (on cgroup v2), does not count pages that are under writeback as reclaimed, and memory.reclaim gives up after MAX_RECLAIM_RETRIES passes without making progress. On a slow device where reclaim does not make any progress before writeback completes, a write to memory.reclaim fails. On a VM with zswap enabled, where IO delay was injected via dm-delay, the success rate of the zswap writeback test drops dramatically once the delay reaches 11 ms: 7% failures at 10 ms and 79% failures at 11 ms, n = 100. When zswap writeback is enabled, ignore -EAGAIN from memory.reclaim and determine pass/fail based on the zswpwb counter because that is what zswap_writeback_enabled actually wants to test. With this change, the test reliably passes even on a slow swap device (tested up to 1000 ms delay). This makes the test resilient against the performance of the swap device. Link: https://lore.kernel.org/20260914-test-zswap-wb-ignore-eagain-v1-1-6fb715c22cd8@kernel.org Fixes: 158863e5d7cc ("selftests: cgroup: add tests to verify the zswap writeback path") Signed-off-by: Harry Yoo (Meta) Signed-off-by: Andrew Morton Reviewed-by: SJ Park Acked-by: Nhat Pham Assisted-by: LLM Cc: Chengming Zhou Cc: Johannes Weiner Cc: Joshua Hahn Cc: Kiryl Shutsemau Cc: Michal Koutný Cc: Shuah Khan Cc: Tejun Heo Cc: Usama Arif --- tools/testing/selftests/cgroup/test_zswap.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 1ac779072775..d50acc1b83ac 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -346,7 +346,16 @@ static int attempt_writeback(const char *cgroup, void *arg) * it can't writeback to swap. */ ret = cg_write_numeric(cgroup, "memory.reclaim", memsize); - if (!wb_enabled) + + /* + * When writeback is enabled, memory.reclaim may still fail to reclaim + * the requested amount of memory due to a slow swap device. + * Ignore -EAGAIN here. The caller determines pass/fail based on the + * zswap writeback counter. + */ + if (wb_enabled && ret == -EAGAIN) + ret = 0; + else if (!wb_enabled) ret = (ret == -EAGAIN) ? 0 : -1; out: From b59925301211ccc351d98d70b11f6d1b2c63150f Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 15 Sep 2026 19:37:27 +0300 Subject: [PATCH 437/539] mm/hmm/test: reject overflowing page counts The number of pages comes directly from userspace. Shifting a value larger than ULONG_MAX >> PAGE_SHIFT discards its high bits before the existing end-address check. A request for a huge number of pages can therefore be accepted and processed as a much smaller request. Reject page counts that cannot be represented as a byte size before performing the shift. So far as ChatGPT and I can tell this doesn't cause an issue in practice but preventing this integer overflow is the correct thing to do. Link: https://lore.kernel.org/6f0d39089e938d4f53dc72632eb84aca34de7015.1789457465.git.error27@gmail.com Fixes: b2ef9f5a5cb3 ("mm/hmm/test: add selftest driver for HMM") Signed-off-by: Dan Carpenter Signed-off-by: Andrew Morton Reviewed-by: Alistair Popple Assisted-by: ChatGPT:gpt-5 Cc: Jason Gunthorpe Cc: Jerome Glisse Cc: Leon Romanovsky Cc: Ralph Campbell Cc: Wei Yongjun --- lib/test_hmm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 6911daa9f854..b409c42bfe6f 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -1624,6 +1624,8 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, if (cmd.addr & ~PAGE_MASK) return -EINVAL; + if (cmd.npages > ULONG_MAX >> PAGE_SHIFT) + return -EINVAL; if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) return -EINVAL; From 222636c8c1398d6ada686ee12d5170f716ee75fc Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 15 Sep 2026 07:33:50 -0700 Subject: [PATCH 438/539] mm/damon/api: introduce DAMON_FILTER_TYPE_HUGEPAGE_SIZE Patch series "mm/damon: introduce hugepage_size probe filter". Knowing whether a given memory is backed by a hugepage of specific size is useful for efficient utilization of hugepages. For easy monitoring of the information, introduce a new data attribute probe filter type, hugepage_size. It works similar to the DAMOS filter of the same name. It works for memory that is backed by a hugepage of a given size range. Patch 1 introduces the new probe filter type to DAMON API and extends related data structures. Patch 2 updates probe filter commit logic to handle the size range. Patch 3 Updates the filtering logic to support the new type. Patch 4 adds new DAMON sysfs files for the size range. Patch 5 updates DAMON sysfs interface to fully support the new filter type. Patches 6-8 updates design, usage and ABI documents for the new feature. This patch (of 8): Introduce a new data attribute probe filter type, hugepage_size. It will work for memory that is backed by a hugepage of a given size range. Add a new damon_filter_type enum DAMON_FILTER_TYPE_HUGEPAGE_SIZE to identify the type. Add two new fields in the damon_filter struct for saving the size range. Link: https://lore.kernel.org/20260915143359.91472-1-sj@kernel.org Link: https://lore.kernel.org/20260915143359.91472-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 4be7d1df8e71..bbb190b47401 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -783,12 +783,14 @@ struct damon_prep { * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. * @DAMON_FILTER_TYPE_PGIDLE_UNSET: Pgidle is unset. * @DAMON_FILTER_TYPE_PGIDLE_SET: Pgidle is set. + * @DAMON_FILTER_TYPE_HUGEPAGE_SIZE: Page is part of a hugepage. */ enum damon_filter_type { DAMON_FILTER_TYPE_ANON, DAMON_FILTER_TYPE_MEMCG, DAMON_FILTER_TYPE_PGIDLE_UNSET, DAMON_FILTER_TYPE_PGIDLE_SET, + DAMON_FILTER_TYPE_HUGEPAGE_SIZE, }; /** @@ -798,6 +800,8 @@ enum damon_filter_type { * @matching: Whether this filter is for the type-matching ones. * @allow: Whether the @type-@matching ones should pass this filter. * @memcg_id: Memcg id of the question if @type is DAMON_FILTER_MEMCG. + * @range_min: Minimum value of range arguments. + * @range_max: Maximum value of range arguments. */ struct damon_filter { enum damon_filter_type type; @@ -805,6 +809,10 @@ struct damon_filter { bool allow; union { u64 memcg_id; + struct { + unsigned long range_min; + unsigned long range_max; + }; }; /* private: */ /* Siblings list. */ From a76723d4b615bd7cf5be8a5a940fbcf9ff55bf18 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 15 Sep 2026 07:33:51 -0700 Subject: [PATCH 439/539] mm/damon/core: commit hugepage_size type damon filter Extend data attribute probe filters commit logic for the new hugepage_size filter type. Since it needs to carry the size range of the hugepage, update the logic to update the size range fields of the commit destination filter struct. While doing that, validate the given range and propagate an error if it is invalid. Add the error handling in the callers, too. Link: https://lore.kernel.org/20260915143359.91472-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 5fdacb9dcee5..e1b49c3d72b8 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1818,7 +1818,7 @@ static int damon_commit_preps(struct damon_probe *dst, struct damon_probe *src) return 0; } -static void damon_commit_filter(struct damon_filter *dst, +static int damon_commit_filter(struct damon_filter *dst, struct damon_filter *src) { dst->type = src->type; @@ -1828,23 +1828,33 @@ static void damon_commit_filter(struct damon_filter *dst, case DAMON_FILTER_TYPE_MEMCG: dst->memcg_id = src->memcg_id; break; + case DAMON_FILTER_TYPE_HUGEPAGE_SIZE: + if (src->range_max < src->range_min) + return -EINVAL; + dst->range_min = src->range_min; + dst->range_max = src->range_max; + break; default: break; } + return 0; } static int damon_commit_filters(struct damon_probe *dst, struct damon_probe *src) { struct damon_filter *dst_filter, *next, *src_filter, *new_filter; - int i = 0, j = 0; + int i = 0, j = 0, err; damon_for_each_filter_safe(dst_filter, next, dst) { src_filter = damon_nth_filter(i++, src); - if (src_filter) - damon_commit_filter(dst_filter, src_filter); - else + if (src_filter) { + err = damon_commit_filter(dst_filter, src_filter); + if (err) + return err; + } else { damon_destroy_filter(dst_filter); + } } damon_for_each_filter_safe(src_filter, next, src) { @@ -1859,6 +1869,14 @@ static int damon_commit_filters(struct damon_probe *dst, case DAMON_FILTER_TYPE_MEMCG: new_filter->memcg_id = src_filter->memcg_id; break; + case DAMON_FILTER_TYPE_HUGEPAGE_SIZE: + if (src_filter->range_max < src_filter->range_min) { + damon_destroy_filter(new_filter); + return -EINVAL; + } + new_filter->range_min = src_filter->range_min; + new_filter->range_max = src_filter->range_max; + break; default: break; } From 71fb2abf3254af5f6339ef7d31730a5a4228e28e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 15 Sep 2026 07:33:52 -0700 Subject: [PATCH 440/539] mm/damon/ops-common: support hugepage_size damon filter matching Update ops-common data attribute filter matching logic to support hugepage_size filter type. Link: https://lore.kernel.org/20260915143359.91472-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/ops-common.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index c36cc39cd2c7..77366f42b3e5 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -536,6 +536,7 @@ bool damon_ops_filter_match(struct damon_filter *filter, struct folio *folio) { bool matched = false; struct mem_cgroup *memcg; + size_t folio_sz; switch (filter->type) { case DAMON_FILTER_TYPE_ANON: @@ -558,6 +559,15 @@ bool damon_ops_filter_match(struct damon_filter *filter, struct folio *folio) matched = filter->memcg_id == mem_cgroup_id(memcg); rcu_read_unlock(); break; + case DAMON_FILTER_TYPE_HUGEPAGE_SIZE: + if (!folio) { + matched = false; + break; + } + folio_sz = folio_size(folio); + matched = filter->range_min <= folio_sz && + folio_sz <= filter->range_max; + break; default: break; } From e63813b89336803ffa8c4d9e8b5e08d99d302900 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 15 Sep 2026 07:33:53 -0700 Subject: [PATCH 441/539] mm/damon/sysfs: add min,max files under probe filter directory In future, DAMON sysfs interface will support data attribute probe filter types that have range arguments like the newly added hugepage_size type filter. To prepare such supports, add two new DAMON sysfs files, min and max, under the probe filter directory. Link: https://lore.kernel.org/20260915143359.91472-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 51fa506c879b..8e8d89b8ed98 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -975,6 +975,8 @@ struct damon_sysfs_filter { bool matching; bool allow; char *path; + unsigned long range_min; + unsigned long range_max; }; static struct damon_sysfs_filter *damon_sysfs_filter_alloc(void) @@ -1127,6 +1129,44 @@ static ssize_t path_store(struct kobject *kobj, return count; } +static ssize_t min_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_filter *filter = container_of(kobj, + struct damon_sysfs_filter, kobj); + + return sysfs_emit(buf, "%lu\n", filter->range_min); +} + +static ssize_t min_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_filter *filter = container_of(kobj, + struct damon_sysfs_filter, kobj); + int err = kstrtoul(buf, 0, &filter->range_min); + + return err ? err : count; +} + +static ssize_t max_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_filter *filter = container_of(kobj, + struct damon_sysfs_filter, kobj); + + return sysfs_emit(buf, "%lu\n", filter->range_max); +} + +static ssize_t max_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_filter *filter = container_of(kobj, + struct damon_sysfs_filter, kobj); + int err = kstrtoul(buf, 0, &filter->range_max); + + return err ? err : count; +} + static void damon_sysfs_filter_release(struct kobject *kobj) { struct damon_sysfs_filter *filter = container_of(kobj, @@ -1148,11 +1188,19 @@ static struct kobj_attribute damon_sysfs_filter_allow_attr = static struct kobj_attribute damon_sysfs_filter_path_attr = __ATTR_RW_MODE(path, 0600); +static struct kobj_attribute damon_sysfs_filter_min_attr = + __ATTR_RW_MODE(min, 0600); + +static struct kobj_attribute damon_sysfs_filter_max_attr = + __ATTR_RW_MODE(max, 0600); + static struct attribute *damon_sysfs_filter_attrs[] = { &damon_sysfs_filter_type_attr.attr, &damon_sysfs_filter_matching_attr.attr, &damon_sysfs_filter_allow_attr.attr, &damon_sysfs_filter_path_attr.attr, + &damon_sysfs_filter_min_attr.attr, + &damon_sysfs_filter_max_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_filter); From 270b82dc6f9cb8ece180b2b9e57e5348fbc89e72 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 15 Sep 2026 07:33:54 -0700 Subject: [PATCH 442/539] mm/damon/sysfs: support hugepage_size probe filter Extend DAMON sysfs interface to support hugepage_size probe filter. Allows hugepage_size user string input to the filter type file. Pass the size range argument that users set via min/max files under the probe filter directory to the DAMON core. Link: https://lore.kernel.org/20260915143359.91472-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/sysfs.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 8e8d89b8ed98..43519afb9eb7 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1007,6 +1007,10 @@ damon_sysfs_filter_type_names[] = { .type = DAMON_FILTER_TYPE_PGIDLE_SET, .name = "pgidle_set", }, + { + .type = DAMON_FILTER_TYPE_HUGEPAGE_SIZE, + .name = "hugepage_size", + }, }; static ssize_t type_show(struct kobject *kobj, @@ -2273,6 +2277,9 @@ static int damon_sysfs_set_filters(struct damon_probe *probe, damon_destroy_filter(filter); return err; } + } else if (filter->type == DAMON_FILTER_TYPE_HUGEPAGE_SIZE) { + filter->range_min = sys_filter->range_min; + filter->range_max = sys_filter->range_max; } damon_add_filter(probe, filter); } From e818662fb61bd82dd4ebcd4d574d7a24fea2b2d0 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 15 Sep 2026 07:33:55 -0700 Subject: [PATCH 443/539] Docs/mm/damon/design: update for hugepage_size probe filter Update DAMON design document for the newly added hugepage_size data attribute probe filter type. Link: https://lore.kernel.org/20260915143359.91472-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 707170bcd1b3..0a86792f90a1 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -300,6 +300,8 @@ filter types. Currently below filter types are supported. - ``pgidle_unset``: Matches if the page for the memory is marked as not access-idle. - ``pgidle_set``: Matches if the page for the memory is marked as access-idle. +- ``hugepage_size``: Matches if the page for the memory is a part of a hugepage + of a given size range. If such probes are registered, DAMON executes the probes for each region's sampling memory when it does the access :ref:`sampling From 595f2ab773aab813c8ff822eafe251239eba94b8 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 15 Sep 2026 07:33:56 -0700 Subject: [PATCH 444/539] Docs/admin-guide/mm/damon/usage: update for hugepage_size Update DAMON usage document for the newly added hugepage_size data attribute filter type. [akpm@linux-foundation.org: s/files acts/file acts/, per SJ] [akpm@linux-foundation.org: fix the fix] Link: https://lore.kernel.org/20260915143359.91472-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/admin-guide/mm/damon/usage.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index d3e37400367b..ba4725544856 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -78,7 +78,7 @@ comma (","). │ │ │ │ │ │ │ │ │ 0/prep_action │ │ │ │ │ │ │ │ │ ... │ │ │ │ │ │ │ │ filters/nr_filters - │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path + │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path,min,max │ │ │ │ │ │ │ │ │ ... │ │ │ │ │ │ │ ... │ │ │ │ │ :ref:`targets `/nr_targets @@ -308,6 +308,8 @@ Writing a number (``N``) to the file creates the number of child directories named ``0`` to ``N-1``. Each directory represents each filter and works in a way similar to that for :ref:`DAMOS filter `. When the filter ``type`` is ``memcg``, ``path`` file acts as ``memcg_path`` for :ref:`DAMOS +filter `. When the filter ``type`` is ``hugepage_size``, +``min`` and ``max`` files act as files of the same names for :ref:`DAMOS filter `. .. _sysfs_targets: From 37158d8eb89d2dd1ec7443e5e457741ded233f2a Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 15 Sep 2026 07:33:57 -0700 Subject: [PATCH 445/539] Docs/ABI/damon: update for hugepage_size probe filter For the newly added hugepage_size data attribute probe filter, two new sysfs files are added for the size range. Update DAMON ABI document for the new files. Link: https://lore.kernel.org/20260915143359.91472-9-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index ad21f58f3c91..55df688ea596 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -206,6 +206,20 @@ Description: If 'memcg' is written to the 'type' file, writing to and reading from this file sets and gets the path to the memory cgroup of the interest. +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//min +Date: Sep 2026 +Contact: SJ Park +Description: If 'hugepage_size' is written to the 'type' file, writing to and + reading from this file sets and gets the minimum size of the + huge page of the interest. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//max +Date: Sep 2026 +Contact: SJ Park +Description: If 'hugepage_size' is written to the 'type' file, writing to and + reading from this file sets and gets the maximum size of the + huge page of the interest. + What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//matching Date: May 2026 Contact: SJ Park From bcf7b0c3a06f9e150967e29a82f9ec00e9d26b70 Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Thu, 17 Sep 2026 13:40:15 +0800 Subject: [PATCH 446/539] mm/huge_memory: simplify pgtable deposit detection Whether a PMD has a deposited PTE page table only depends on whether the architecture requires deposits or the VMA is anonymous. Implement this rule directly in vma_has_deposited_pgtable(), avoiding mistaking a non-anonymous raw PFN PMD for one with a deposit and attempting to withdraw a page table that was never deposited. There is no known in-tree workload that triggers this. Mostly a defensive/simplifying change. Link: https://lore.kernel.org/20260917054015.23553-1-lance.yang@linux.dev Signed-off-by: Lance Yang Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Acked-by: Zi Yan Reviewed-by: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Liam R. Howlett Cc: Ryan Roberts --- mm/huge_memory.c | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8f8bf60a2264..26523137b758 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2520,25 +2520,13 @@ static struct folio *normal_or_softleaf_folio_pmd(struct vm_area_struct *vma, return pmd_to_softleaf_folio(pmdval); } -static bool has_deposited_pgtable(struct vm_area_struct *vma, pmd_t pmdval, - struct folio *folio) +static bool vma_has_deposited_pgtable(struct vm_area_struct *vma) { - /* Some architectures require unconditional depositing. */ - if (arch_needs_pgtable_deposit()) - return true; - /* - * Huge zero always deposited except for DAX which handles itself, see - * set_huge_zero_folio(). + * PMDs in anonymous VMAs always have a deposited page table. PMDs in + * other VMAs only have one when required by the architecture. */ - if (is_huge_zero_pmd(pmdval)) - return !vma_is_dax(vma); - - /* - * Otherwise, only anonymous folios are deposited, see - * __do_huge_pmd_anonymous_page(). - */ - return folio && folio_test_anon(folio); + return arch_needs_pgtable_deposit() || vma_is_anonymous(vma); } /** @@ -2578,7 +2566,7 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, is_present = pmd_present(orig_pmd); folio = normal_or_softleaf_folio_pmd(vma, addr, orig_pmd, is_present); - has_deposit = has_deposited_pgtable(vma, orig_pmd, folio); + has_deposit = vma_has_deposited_pgtable(vma); if (folio) zap_huge_pmd_folio(mm, vma, orig_pmd, folio, is_present); if (has_deposit) From 25aa09142eccd44ebfc537f5b1aaed1735e10dc4 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 18 Sep 2026 12:50:13 +0200 Subject: [PATCH 447/539] mm/vmalloc: use %p for pointer formatting Commit 45ec16908e84e ("mm: use %pK for /proc/vmallocinfo") introduced the %pK in order not to leak kernel pointers. Since commit ad67b74d2469d ("printk: hash addresses printed with %p") pointers are hashed by default and the behaviour can be controlled by `hash_pointers' boot argument. The policy on %p is to not introduce new ones. Removing %pK makes it possible to remove its handling from the library. Looking at the output, having the pointer in the output makes it possible to distinguish the individual entries. Use %p instead %pK. /proc/vmallocinfo has mode 0400, so this change doesn't make kernel pointer information available to unprivileged userspace. Link: https://lore.kernel.org/20260918105013.UpdykT6j@linutronix.de Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Andrew Morton Reviewed-by: SJ Park Reviewed-by: Uladzislau Rezki (Sony) --- mm/vmalloc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 547864233128..a009cc73f1b7 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -5323,7 +5323,7 @@ static void show_purge_info(struct seq_file *m) for_each_vmap_node(vn) { spin_lock(&vn->lazy.lock); list_for_each_entry(va, &vn->lazy.head, list) { - seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n", + seq_printf(m, "0x%p-0x%p %7ld unpurged vm_area\n", (void *)va->va_start, (void *)va->va_end, va_size(va)); } @@ -5346,7 +5346,7 @@ static void show_busy_info(struct seq_file *m) list_for_each_entry(va, &vn->busy.head, list) { if (!va->vm) { if (va->flags & VMAP_RAM) - seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n", + seq_printf(m, "0x%p-0x%p %7ld vm_map_ram\n", (void *)va->va_start, (void *)va->va_end, va_size(va)); @@ -5360,7 +5360,7 @@ static void show_busy_info(struct seq_file *m) /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ smp_rmb(); - seq_printf(m, "0x%pK-0x%pK %7ld", + seq_printf(m, "0x%p-0x%p %7ld", v->addr, v->addr + v->size, v->size); if (v->caller) From 12e828f463f390b14c88fe26b8790fc9edce985c Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Tue, 15 Sep 2026 15:55:24 +0530 Subject: [PATCH 448/539] mm/gup_test: safely calculate GUP batch size __gup_test_ioctl() calculates the end of a GUP batch using: next = addr + nr * PAGE_SIZE; If nr is too large, it can cause next to overflow and wrap around. If it wraps, the next > end check is bypassed and a large value of nr is passed to the GUP call, even though the pages array was allocated according to gup->size. This can lead to out of bounds writes. Also, when fewer than PAGE_SIZE bytes remain, the calculated batch contains zero pages. The code still calls GUP functions with pages + i, which can point past the allocated array. Calculate nr by taking the minimum of the number of pages per call and the pages remaining in the address range. Reject zero sized and non page aligned gup->size values and zero nr_pages_per_call value. Link: https://lore.kernel.org/20260915102524.125758-1-sarthak.sharma@arm.com Fixes: 64c349f4ae78 ("mm: add infrastructure for get_user_pages_fast() benchmarking") Signed-off-by: Sarthak Sharma Signed-off-by: Andrew Morton Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu --- mm/gup_test.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mm/gup_test.c b/mm/gup_test.c index 185ba3bb8ed1..ba74bf3f4104 100644 --- a/mm/gup_test.c +++ b/mm/gup_test.c @@ -116,7 +116,8 @@ static int __gup_test_ioctl(unsigned int cmd, bool needs_mmap_lock = cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK; - if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX) + if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX || !gup->size || + !gup->nr_pages_per_call || !PAGE_ALIGNED(gup->size)) return -EINVAL; if (check_add_overflow((unsigned long)gup->addr, (unsigned long)gup->size, &end)) @@ -139,11 +140,9 @@ static int __gup_test_ioctl(unsigned int cmd, if (nr != gup->nr_pages_per_call) break; + nr = min_t(unsigned long, nr, (end - addr) / PAGE_SIZE); + next = addr + nr * PAGE_SIZE; - if (next > end) { - next = end; - nr = (next - addr) / PAGE_SIZE; - } switch (cmd) { case GUP_FAST_BENCHMARK: From 5b306375a4aa3d002a0e0e5c508b21e581286cf7 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Tue, 15 Sep 2026 18:15:56 +0800 Subject: [PATCH 449/539] mm/mglru: restore accidentally removed seq < max_seq check Since commit 798c0330c2ca ("mm/mglru: rework aging feedback"), the following sanity check was accidentally removed: if (seq < max_seq) return 0; That means we can perform aging for any value less than or equal to max_gen_nr. This has been inconsistent with Documentation/admin-guide/mm/multigen_lru.rst, which states: Users can write the following command to ``lru_gen`` to create a new generation ``max_gen_nr+1``: ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]`` The correct semantics are that writing a value smaller than max_gen_nr should return 0, since the requested generation already exists. Link: https://lore.kernel.org/20260915101556.50467-1-baohua@kernel.org Fixes: 798c0330c2ca ("mm/mglru: rework aging feedback") Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Andrew Morton Reported-by: Chuanhua Han Reviewed-by: Kairui Song Reviewed-by: Baolin Wang Cc: Axel Rasmussen Cc: Baoquan He Cc: David Hildenbrand Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Cc: Yu Zhao --- mm/vmscan.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/vmscan.c b/mm/vmscan.c index dd6261c86279..e09da1ebec0e 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -5839,6 +5839,9 @@ static int run_aging(struct lruvec *lruvec, unsigned long seq, { DEFINE_MAX_SEQ(lruvec); + if (seq < max_seq) + return 0; + if (seq > max_seq) return -EINVAL; From 1a4a3f756cd7ecec834b2c73394a2fb9c4a8c6f0 Mon Sep 17 00:00:00 2001 From: Zhenghui Hao Date: Tue, 15 Sep 2026 16:20:29 +0800 Subject: [PATCH 450/539] mm/hugetlb: fix misspelled parameter names in comment The comment above hugepages_setup() refers to "hugepagsz" and "default_hugepagsz", but the parameters parsed by this code are "hugepagesz" and "default_hugepagesz". Fix the spelling and also drop the redundant spaces so the comment reads normally. No functional change. Link: https://lore.kernel.org/tencent_034C6FC23D4817C40657E5F17F64E260A009@qq.com Signed-off-by: Zhenghui Hao Signed-off-by: Andrew Morton Acked-by: Muchun Song Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Oscar Salvador --- mm/hugetlb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e4c45698adba..b017ae1dfd6f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4319,9 +4319,9 @@ static __init void hugetlb_parse_params(void) /* * hugepages command line processing - * hugepages normally follows a valid hugepagsz or default_hugepagsz - * specification. If not, ignore the hugepages value. hugepages can also - * be the first huge page command line option in which case it implicitly + * hugepages normally follows a valid hugepagesz or default_hugepagesz + * specification. If not, ignore the hugepages value. hugepages can also + * be the first huge page command line option in which case it implicitly * specifies the number of huge pages for the default size. */ static int __init hugepages_setup(char *s) From 1372648980b8524a5747a2732e90b20d3db10c3c Mon Sep 17 00:00:00 2001 From: Qiqi Liu Date: Tue, 15 Sep 2026 15:49:28 +0800 Subject: [PATCH 451/539] mm/page_alloc: apply per-task GFP context in bulk allocator alloc_pages_bulk_noprof() does not call current_gfp_context(), so per-task scoped allocation constraints (PF_MEMALLOC_NOIO, PF_MEMALLOC_NOFS, PF_MEMALLOC_PIN) are not applied on the bulk fast path. By ignoring PF_MEMALLOC_NOIO and PF_MEMALLOC_NOFS, the allocation can theoretically result in a deadlock. Ignoring PF_MEMALLOC_PIN also has consequences: without clearing __GFP_MOVABLE, prepare_alloc_pages() selects MIGRATE_MOVABLE for the PCP list, and a task with PF_MEMALLOC_PIN set receives movable pages from the bulk allocator. Once pinned, these pages can no longer be migrated but remain in MOVABLE pageblocks, violating the mobility contract. This can increase fragmentation and interfere with compaction or contiguous-memory allocations, eventually surfacing as higher allocation latency or allocation failures under memory pressure. Found via review of the bulk allocation tracepoint hooks [1]. Link: https://sashiko.dev/#/patchset/20260907120949.418450-1-liuqiqi%40kylinos.cn Link: https://lore.kernel.org/all/20260907120949.418450-1-liuqiqi@kylinos.cn/ [1] Link: https://lore.kernel.org/20260915074928.327471-1-liuqiqi@kylinos.cn Fixes: 387ba26fb1cb ("mm/page_alloc: add a bulk page allocator") Signed-off-by: Qiqi Liu Signed-off-by: Andrew Morton Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Zi Yan --- mm/page_alloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b261445519e8..3265f7ff909a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5189,6 +5189,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */ gfp &= gfp_allowed_mask; + gfp = current_gfp_context(gfp); if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &gfp, &alloc_flags)) goto out; From 5a6c9cf117fca3c38a98fb2a2d5549795c8b0b73 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sat, 12 Sep 2026 07:05:40 -0400 Subject: [PATCH 452/539] mm/madvise: use folio_trylock() in the cold/pageout PMD split MADV_COLD or MADV_PAGEOUT over part of a PMD splits the THP in madvise_cold_or_pageout_pte_range(). Two threads doing that to the same THP create spurious failures. CPU0 CPU1 ---- ---- folio_get() spin_unlock(ptl) folio_lock() folio_get() spin_unlock(ptl) folio_lock() <- blocks, keeps its ref split_folio() folio_expected_ref_count(folio) != folio_ref_count(folio) - 1 -EAGAIN CPU1 cannot drop its reference until it gets the lock CPU0 holds, so CPU0's split always fails. folio_trylock() makes CPU1 leave without ever taking a reference. The PTE branch of this same function already does this, as do madvise_free_pte_range() and madvise_free_huge_pmd(). Reproducer: 400 rounds of eight threads calling MADV_COLD on half of each of eight THPs, re-formed with MADV_COLLAPSE between rounds. From /proc/vmstat: thp_split_page thp_split_page_failed before 3186 860 after 3200 0 The short before count is rounds where every thread failed and the advice was dropped for that THP entirely. On failure the walker returns 0 and nothing retries. The PMD path becomes best effort when the folio lock is held elsewhere - same as the PTE path. Link: https://lore.kernel.org/20260912110540.3203010-1-gourry@gourry.net Signed-off-by: Gregory Price (Meta) Signed-off-by: Andrew Morton Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260817220810.1175596-1-gourry%40gourry.net Acked-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Assisted-by: LLM Cc: Jann Horn Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: --- mm/madvise.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/madvise.c b/mm/madvise.c index 8943ae181144..bef4de4425f2 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -417,9 +417,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (next - addr != HPAGE_PMD_SIZE) { int err; + if (!folio_trylock(folio)) + goto huge_unlock; folio_get(folio); spin_unlock(ptl); - folio_lock(folio); err = split_folio(folio); folio_unlock(folio); folio_put(folio); From 50d96b923593fea370fd125b8269a856b0974f6c Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:05 -0400 Subject: [PATCH 453/539] mm: memcontrol: take a const folio in folio_memcg() and friends Patch series "mm: memcontrol: constify the read side of the read side of the memcg API", v3. The memcg accessors, lruvec helpers, and stat readers only read from the memcg, folio, or lruvec they are given, but take non-const pointers. Constify them, along with the page_counter readers and the swap I/O blkg helpers along the way. This patch (of 11): The folio_memcg() family only reads from the folio, and everything it calls already takes a const folio. Constify it, along with the page wrappers built on top of it and get_obj_cgroup_from_folio(), which only reads the folio's objcg. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-0-c239a6010b58@columbia.edu Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-1-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/memcontrol.h | 50 +++++++++++++++++++------------------- mm/memcontrol.c | 8 +++--- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 46bf724cae7a..5a5ca6814782 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -410,7 +410,7 @@ static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) * or NULL. This function assumes that the folio is known to have a * proper object cgroup pointer. */ -static inline struct obj_cgroup *folio_objcg(struct folio *folio) +static inline struct obj_cgroup *folio_objcg(const struct folio *folio) { unsigned long memcg_data = folio->memcg_data; @@ -448,7 +448,7 @@ static inline struct obj_cgroup *folio_objcg(struct folio *folio) * Note: The caller should hold an rcu read lock or cgroup_mutex to protect * memcg associated with a folio from being released. */ -static inline struct mem_cgroup *folio_memcg(struct folio *folio) +static inline struct mem_cgroup *folio_memcg(const struct folio *folio) { struct obj_cgroup *objcg = folio_objcg(folio); @@ -461,7 +461,7 @@ static inline struct mem_cgroup *folio_memcg(struct folio *folio) * * Returns true if folio is charged to a memory cgroup, otherwise returns false. */ -static inline bool folio_memcg_charged(struct folio *folio) +static inline bool folio_memcg_charged(const struct folio *folio) { return folio->memcg_data != 0; } @@ -481,7 +481,7 @@ static inline bool folio_memcg_charged(struct folio *folio) * A caller should hold an rcu read lock to protect memcg associated with a * page from being released. */ -static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) +static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio) { /* * Because folio->memcg_data might be changed asynchronously @@ -498,11 +498,11 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) return obj_cgroup_memcg(objcg); } -static inline struct mem_cgroup *page_memcg_check(struct page *page) +static inline struct mem_cgroup *page_memcg_check(const struct page *page) { if (PageTail(page)) return NULL; - return folio_memcg_check((struct folio *)page); + return folio_memcg_check((const struct folio *)page); } static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) @@ -527,14 +527,14 @@ retry: * that the folio has an associated memory cgroup. It's not safe to call * this function against some types of folios, e.g. slab folios. */ -static inline bool folio_memcg_kmem(struct folio *folio) +static inline bool folio_memcg_kmem(const struct folio *folio) { VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page); VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio); return folio->memcg_data & MEMCG_DATA_KMEM; } -static inline bool PageMemcgKmem(struct page *page) +static inline bool PageMemcgKmem(const struct page *page) { return folio_memcg_kmem(page_folio(page)); } @@ -777,7 +777,7 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm); struct mem_cgroup *get_mem_cgroup_from_current(void); -struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio); +struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio); struct lruvec *folio_lruvec_lock(struct folio *folio); struct lruvec *folio_lruvec_lock_irq(struct folio *folio); @@ -905,8 +905,8 @@ static inline bool mm_match_cgroup(struct mm_struct *mm, return match; } -struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio); -ino_t page_cgroup_ino(struct page *page); +struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio); +ino_t page_cgroup_ino(const struct page *page); static inline bool mem_cgroup_online(struct mem_cgroup *memcg) { @@ -956,7 +956,7 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg); void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, int val); -static inline void mod_memcg_page_state(struct page *page, +static inline void mod_memcg_page_state(const struct page *page, enum memcg_stat_item idx, int val) { struct mem_cgroup *memcg; @@ -990,7 +990,7 @@ void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val); void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count); -static inline void count_memcg_folio_events(struct folio *folio, +static inline void count_memcg_folio_events(const struct folio *folio, enum vm_event_item idx, unsigned long nr) { struct mem_cgroup *memcg; @@ -1084,22 +1084,22 @@ static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) #define root_mem_cgroup (NULL) -static inline struct mem_cgroup *folio_memcg(struct folio *folio) +static inline struct mem_cgroup *folio_memcg(const struct folio *folio) { return NULL; } -static inline bool folio_memcg_charged(struct folio *folio) +static inline bool folio_memcg_charged(const struct folio *folio) { return false; } -static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) +static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio) { return NULL; } -static inline struct mem_cgroup *page_memcg_check(struct page *page) +static inline struct mem_cgroup *page_memcg_check(const struct page *page) { return NULL; } @@ -1109,12 +1109,12 @@ static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *ob return NULL; } -static inline bool folio_memcg_kmem(struct folio *folio) +static inline bool folio_memcg_kmem(const struct folio *folio) { return false; } -static inline bool PageMemcgKmem(struct page *page) +static inline bool PageMemcgKmem(const struct page *page) { return false; } @@ -1248,7 +1248,7 @@ static inline struct mem_cgroup *get_mem_cgroup_from_current(void) return NULL; } -static inline struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio) +static inline struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio) { return NULL; } @@ -1406,7 +1406,7 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg, { } -static inline void mod_memcg_page_state(struct page *page, +static inline void mod_memcg_page_state(const struct page *page, enum memcg_stat_item idx, int val) { } @@ -1471,7 +1471,7 @@ static inline void count_memcg_events(struct mem_cgroup *memcg, { } -static inline void count_memcg_folio_events(struct folio *folio, +static inline void count_memcg_folio_events(const struct folio *folio, enum vm_event_item idx, unsigned long nr) { } @@ -1767,7 +1767,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order); * needs to be used outside of the local scope. */ struct obj_cgroup *current_obj_cgroup(void); -struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio); +struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio); static inline struct obj_cgroup *get_obj_cgroup_from_current(void) { @@ -1870,7 +1870,7 @@ static inline void __memcg_kmem_uncharge_page(struct page *page, int order) { } -static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio) +static inline struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio) { return NULL; } @@ -1901,7 +1901,7 @@ static inline void count_objcg_events(struct obj_cgroup *objcg, { } -static inline ino_t page_cgroup_ino(struct page *page) +static inline ino_t page_cgroup_ino(const struct page *page) { return 0; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 791e536efaeb..72c0e9358cfc 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -348,7 +348,7 @@ EXPORT_SYMBOL(memcg_bpf_enabled_key); * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup * is returned. */ -struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio) +struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio) { struct mem_cgroup *memcg; @@ -373,7 +373,7 @@ struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio) * after page_cgroup_ino() returns, so it only should be used by callers that * do not care (such as procfs interfaces). */ -ino_t page_cgroup_ino(struct page *page) +ino_t page_cgroup_ino(const struct page *page) { struct mem_cgroup *memcg; unsigned long ino = 0; @@ -1256,7 +1256,7 @@ again: * * See folio_memcg() for folio->objcg/memcg binding rules. */ -struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio) +struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio) { struct mem_cgroup *memcg; @@ -3154,7 +3154,7 @@ from_memcg: return rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1); } -struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio) +struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio) { struct obj_cgroup *objcg; From da67c138b5fdc15722985684ccc6b8762a71d1da Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:06 -0400 Subject: [PATCH 454/539] mm: memcontrol: constify obj_cgroup_memcg() and friends obj_cgroup_memcg() and get_mem_cgroup_from_objcg() only read from the objcg. Constify them. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-2-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/memcontrol.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 5a5ca6814782..65cb45f4be32 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -396,7 +396,7 @@ enum objext_flags { * * The caller must ensure that the returned memcg won't be released. */ -static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg) { lockdep_assert_once(rcu_read_lock_held() || lockdep_is_held(&cgroup_mutex)); return objcg ? READ_ONCE(objcg->memcg) : NULL; @@ -505,7 +505,7 @@ static inline struct mem_cgroup *page_memcg_check(const struct page *page) return folio_memcg_check((const struct folio *)page); } -static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg) { struct mem_cgroup *memcg; @@ -1075,7 +1075,7 @@ void mem_cgroup_flush_workqueue(void); extern int mem_cgroup_init(void); #else /* CONFIG_MEMCG */ -static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg) { return NULL; } @@ -1104,7 +1104,7 @@ static inline struct mem_cgroup *page_memcg_check(const struct page *page) return NULL; } -static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg) { return NULL; } From e791754fe99fc48324c6a73cf42cc3a9a2d58a67 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:07 -0400 Subject: [PATCH 455/539] mm: memcontrol: constify the lruvec helpers The lruvec lookup helpers only read from the memcg, folio, or lruvec they are given. Constify them, along with lruvec_pgdat() and the folio argument of the folio_lruvec_relock_irq() helpers. mem_cgroup_lruvec() may update lruvec->pgdat for a newly onlined node, but that lives in the per-node structure, not the memcg. Similarly, lruvec_pgdat() still returns a non-const pgdat and does not use container_of_const(). Use container_of_const() in lruvec_memcg() and mem_cgroup_get_zone_lru_size(), so the const isn't silently cast away. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-3-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/memcontrol.h | 46 +++++++++++++++++++------------------- include/linux/mmzone.h | 2 +- mm/memcontrol.c | 6 ++--- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 65cb45f4be32..436934030d79 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -722,7 +722,7 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new); * @pgdat combination. This can be the node lruvec, if the memory * controller is disabled. */ -static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, +static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg, struct pglist_data *pgdat) { struct mem_cgroup_per_node *mz; @@ -763,7 +763,7 @@ out: * their binding is stable if the returned lruvec matches the one the caller has * locked. Useful for lock batching. */ -static inline struct lruvec *folio_lruvec(struct folio *folio) +static inline struct lruvec *folio_lruvec(const struct folio *folio) { struct mem_cgroup *memcg = folio_memcg(folio); @@ -779,9 +779,9 @@ struct mem_cgroup *get_mem_cgroup_from_current(void); struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio); -struct lruvec *folio_lruvec_lock(struct folio *folio); -struct lruvec *folio_lruvec_lock_irq(struct folio *folio); -struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, +struct lruvec *folio_lruvec_lock(const struct folio *folio); +struct lruvec *folio_lruvec_lock_irq(const struct folio *folio); +struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flags); static inline @@ -861,14 +861,14 @@ static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) return mem_cgroup_from_css(seq_css(m)); } -static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) +static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec) { - struct mem_cgroup_per_node *mz; + const struct mem_cgroup_per_node *mz; if (mem_cgroup_disabled()) return NULL; - mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); return mz->memcg; } @@ -919,13 +919,13 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages); static inline -unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, +unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, enum lru_list lru, int zone_idx) { long val; - struct mem_cgroup_per_node *mz; + const struct mem_cgroup_per_node *mz; - mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); val = READ_ONCE(mz->lru_zone_size[zone_idx][lru]); if (WARN_ON_ONCE(val < 0)) return 0; @@ -1215,13 +1215,13 @@ static inline void mem_cgroup_migrate(struct folio *old, struct folio *new) { } -static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, +static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg, struct pglist_data *pgdat) { return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec(struct folio *folio) +static inline struct lruvec *folio_lruvec(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); return &pgdat->__lruvec; @@ -1281,7 +1281,7 @@ static inline void mem_cgroup_put(struct mem_cgroup *memcg) { } -static inline struct lruvec *folio_lruvec_lock(struct folio *folio) +static inline struct lruvec *folio_lruvec_lock(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1290,7 +1290,7 @@ static inline struct lruvec *folio_lruvec_lock(struct folio *folio) return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) +static inline struct lruvec *folio_lruvec_lock_irq(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1299,7 +1299,7 @@ static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, +static inline struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flagsp) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1354,7 +1354,7 @@ static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) return NULL; } -static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) +static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec) { return NULL; } @@ -1365,7 +1365,7 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg) } static inline -unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, +unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, enum lru_list lru, int zone_idx) { return 0; @@ -1505,7 +1505,7 @@ static inline void mem_cgroup_flush_workqueue(void) { } static inline int mem_cgroup_init(void) { return 0; } #endif /* CONFIG_MEMCG */ -static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) +static inline struct lruvec *parent_lruvec(const struct lruvec *lruvec) { struct mem_cgroup *memcg; @@ -1568,15 +1568,15 @@ static inline void lruvec_unlock_irqrestore(struct lruvec *lruvec, unsigned long } /* Test requires a stable folio->memcg binding, see folio_memcg() */ -static inline bool folio_matches_lruvec(struct folio *folio, - struct lruvec *lruvec) +static inline bool folio_matches_lruvec(const struct folio *folio, + const struct lruvec *lruvec) { return lruvec_pgdat(lruvec) == folio_pgdat(folio) && lruvec_memcg(lruvec) == folio_memcg(folio); } /* Don't lock again iff page's lruvec locked */ -static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, +static inline struct lruvec *folio_lruvec_relock_irq(const struct folio *folio, struct lruvec *locked_lruvec) { if (locked_lruvec) { @@ -1590,7 +1590,7 @@ static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, } /* Don't lock again iff folio's lruvec locked */ -static inline void folio_lruvec_relock_irqsave(struct folio *folio, +static inline void folio_lruvec_relock_irqsave(const struct folio *folio, struct lruvec **lruvecp, unsigned long *flags) { if (*lruvecp) { diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 65de3bb13eb3..8e4e0bda3b58 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1638,7 +1638,7 @@ extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn extern void lruvec_init(struct lruvec *lruvec); -static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) +static inline struct pglist_data *lruvec_pgdat(const struct lruvec *lruvec) { #ifdef CONFIG_MEMCG return lruvec->pgdat; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 72c0e9358cfc..294827ef54a5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1477,7 +1477,7 @@ void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, * * Return: The lruvec this folio is on with its lock held and rcu read lock held. */ -struct lruvec *folio_lruvec_lock(struct folio *folio) +struct lruvec *folio_lruvec_lock(const struct folio *folio) { struct lruvec *lruvec; @@ -1505,7 +1505,7 @@ retry: * Return: The lruvec this folio is on with its lock held and interrupts * disabled and rcu read lock held. */ -struct lruvec *folio_lruvec_lock_irq(struct folio *folio) +struct lruvec *folio_lruvec_lock_irq(const struct folio *folio) { struct lruvec *lruvec; @@ -1534,7 +1534,7 @@ retry: * Return: The lruvec this folio is on with its lock held and interrupts * disabled and rcu read lock held. */ -struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, +struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flags) { struct lruvec *lruvec; From 4cb5edd32dfab3a8b2e490641d5d3523ca3b9aaa Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:08 -0400 Subject: [PATCH 456/539] mm/page_io: take a const folio in bio_associate_blkg_from_folio() bio_associate_blkg_from_folio() and its helpers only read from the folio. Constify their folio arguments. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-4-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- mm/page_io.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index 1da4ff484f09..5f7756e370f7 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -256,12 +256,13 @@ out_unlock: } #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) -static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio) +static struct cgroup_subsys_state *folio_memcg_blkg_css(const struct folio *folio) { return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys); } -static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +static bool folio_blkg_can_merge(const struct folio *folio, + const struct folio *prev_folio) { bool can_merge = true; @@ -277,7 +278,8 @@ static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) return can_merge; } -static void bio_associate_blkg_from_folio(struct bio *bio, struct folio *folio) +static void bio_associate_blkg_from_folio(struct bio *bio, + const struct folio *folio) { struct cgroup_subsys_state *css; @@ -294,11 +296,13 @@ static void bio_associate_blkg_from_folio(struct bio *bio, struct folio *folio) css_put(css); } #else -static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +static bool folio_blkg_can_merge(const struct folio *folio, + const struct folio *prev_folio) { return true; } -static void bio_associate_blkg_from_folio(struct bio *bio, struct folio *folio) +static void bio_associate_blkg_from_folio(struct bio *bio, + const struct folio *folio) { } #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ From ed1bb84579a4d8567c434d4eac132746f2a604bc Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:09 -0400 Subject: [PATCH 457/539] mm: memcontrol: constify the mem_cgroup accessors mem_cgroup_id(), parent_mem_cgroup(), mem_cgroup_is_root(), mem_cgroup_is_descendant(), memcg_kmem_id(), and the other memcg accessors only read from the memcg. Constify them, along with mem_cgroup_shrink_is_root()'s shrink_control. mem_cgroup_print_oom_context()'s task stays non-const, as task_cgroup() takes a non-const task. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-5-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/memcontrol.h | 41 +++++++++++++++++++------------------- mm/memcontrol.c | 3 ++- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 436934030d79..01f74413fc79 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -539,7 +539,7 @@ static inline bool PageMemcgKmem(const struct page *page) return folio_memcg_kmem(page_folio(page)); } -static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) +static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg) { return (memcg == root_mem_cgroup); } @@ -555,7 +555,7 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) * and do not honour sc->memcg can use this to early-return 0 in per-memcg * contexts. */ -static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc) +static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc) { return !sc->memcg || mem_cgroup_is_root(sc->memcg); } @@ -840,7 +840,7 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, int (*)(struct task_struct *, void *), void *arg); -static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg) +static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg) { if (mem_cgroup_disabled()) return 0; @@ -849,7 +849,7 @@ static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg) } struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id); -static inline u64 mem_cgroup_id(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg) { return memcg ? cgroup_id(memcg->css.cgroup) : 0; } @@ -878,13 +878,13 @@ static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec) * * Returns the parent memcg, or NULL if this is the root. */ -static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) +static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg) { return mem_cgroup_from_css(memcg->css.parent); } -static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, - struct mem_cgroup *root) +static inline bool mem_cgroup_is_descendant(const struct mem_cgroup *memcg, + const struct mem_cgroup *root) { if (root == memcg) return true; @@ -892,7 +892,7 @@ static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, } static inline bool mm_match_cgroup(struct mm_struct *mm, - struct mem_cgroup *memcg) + const struct mem_cgroup *memcg) { struct mem_cgroup *task_memcg; bool match = false; @@ -943,7 +943,7 @@ static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask) unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg); -void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, +void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, struct task_struct *p); void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg); @@ -1119,12 +1119,12 @@ static inline bool PageMemcgKmem(const struct page *page) return false; } -static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) +static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg) { return true; } -static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc) +static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc) { return true; } @@ -1227,13 +1227,13 @@ static inline struct lruvec *folio_lruvec(const struct folio *folio) return &pgdat->__lruvec; } -static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) +static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg) { return NULL; } static inline bool mm_match_cgroup(struct mm_struct *mm, - struct mem_cgroup *memcg) + const struct mem_cgroup *memcg) { return true; } @@ -1327,7 +1327,7 @@ static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, { } -static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg) +static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg) { return 0; } @@ -1339,7 +1339,7 @@ static inline struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id) return NULL; } -static inline u64 mem_cgroup_id(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg) { return 0; } @@ -1377,7 +1377,8 @@ static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) } static inline void -mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) +mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, + struct task_struct *p) { } @@ -1813,7 +1814,7 @@ static inline void memcg_kmem_uncharge_page(struct page *page, int order) * A helper for accessing memcg's kmem_id, used for getting * corresponding LRU lists. */ -static inline int memcg_kmem_id(struct mem_cgroup *memcg) +static inline int memcg_kmem_id(const struct mem_cgroup *memcg) { return memcg ? memcg->kmemcg_id : -1; } @@ -1885,7 +1886,7 @@ static inline bool memcg_kmem_online(void) return false; } -static inline int memcg_kmem_id(struct mem_cgroup *memcg) +static inline int memcg_kmem_id(const struct mem_cgroup *memcg) { return -1; } @@ -1962,7 +1963,7 @@ static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) #ifdef CONFIG_MEMCG_V1 bool mem_cgroup_oom_synchronize(bool wait); -static inline bool task_in_memcg_oom(struct task_struct *p) +static inline bool task_in_memcg_oom(const struct task_struct *p) { return p->memcg_in_oom; } @@ -1980,7 +1981,7 @@ static inline void mem_cgroup_exit_user_fault(void) } #else /* CONFIG_MEMCG_V1 */ -static inline bool task_in_memcg_oom(struct task_struct *p) +static inline bool task_in_memcg_oom(const struct task_struct *p) { return false; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 294827ef54a5..adc93d28dbe7 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1832,7 +1832,8 @@ static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is * enabled */ -void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) +void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, + struct task_struct *p) { rcu_read_lock(); From 97ea370b4a6f96a485036baf0f653217acf07b95 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:10 -0400 Subject: [PATCH 458/539] mm: page_counter: constify page_counter_read() and page_counter_margin() Both only read the counter. Constify them so that users can read counters from const memcgs. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-6-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Reviewed-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/page_counter.h | 4 ++-- mm/page_counter.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index 07b7cb12249c..2baf7a2b29b2 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -63,12 +63,12 @@ static inline void page_counter_init(struct page_counter *counter, counter->track_failcnt = false; } -static inline unsigned long page_counter_read(struct page_counter *counter) +static inline unsigned long page_counter_read(const struct page_counter *counter) { return atomic_long_read(&counter->usage); } -long page_counter_margin(struct page_counter *counter); +long page_counter_margin(const struct page_counter *counter); void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages); void page_counter_charge(struct page_counter *counter, unsigned long nr_pages); bool page_counter_try_charge(struct page_counter *counter, diff --git a/mm/page_counter.c b/mm/page_counter.c index 98322803941a..9167ffd1380c 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -54,7 +54,7 @@ static void propagate_protected_usage(struct page_counter *c, * Return: The minimum value of max minus usage across @counter and all of * its ancestors. The value may be negative during a concurrent charge. */ -long page_counter_margin(struct page_counter *counter) +long page_counter_margin(const struct page_counter *counter) { long margin = PAGE_COUNTER_MAX; From cb347711ed75ba817b822dfc1994b26f85336aee Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:11 -0400 Subject: [PATCH 459/539] mm: memcontrol: constify the reclaim protection helpers mem_cgroup_protection(), mem_cgroup_unprotected(), mem_cgroup_below_low(), and mem_cgroup_below_min() only read the protection state. Constify them. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-7-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/memcontrol.h | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 01f74413fc79..22067899eb6c 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -570,8 +570,8 @@ static inline bool mem_cgroup_disabled(void) return !cgroup_subsys_enabled(memory_cgrp_subsys); } -static inline void mem_cgroup_protection(struct mem_cgroup *root, - struct mem_cgroup *memcg, +static inline void mem_cgroup_protection(const struct mem_cgroup *root, + const struct mem_cgroup *memcg, unsigned long *min, unsigned long *low, unsigned long *usage) @@ -625,8 +625,8 @@ static inline void mem_cgroup_protection(struct mem_cgroup *root, void mem_cgroup_calculate_protection(struct mem_cgroup *root, struct mem_cgroup *memcg); -static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { /* * The root memcg doesn't account charges, and doesn't support @@ -637,8 +637,8 @@ static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, memcg == target; } -static inline bool mem_cgroup_below_low(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_low(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { if (mem_cgroup_unprotected(target, memcg)) return false; @@ -647,8 +647,8 @@ static inline bool mem_cgroup_below_low(struct mem_cgroup *target, page_counter_read(&memcg->memory); } -static inline bool mem_cgroup_below_min(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_min(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { if (mem_cgroup_unprotected(target, memcg)) return false; @@ -1149,8 +1149,8 @@ static inline void memcg_memory_event_mm(struct mm_struct *mm, { } -static inline void mem_cgroup_protection(struct mem_cgroup *root, - struct mem_cgroup *memcg, +static inline void mem_cgroup_protection(const struct mem_cgroup *root, + const struct mem_cgroup *memcg, unsigned long *min, unsigned long *low, unsigned long *usage) @@ -1163,19 +1163,19 @@ static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root, { } -static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return true; } -static inline bool mem_cgroup_below_low(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_low(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return false; } -static inline bool mem_cgroup_below_min(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_min(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return false; } From d5c32fb2c2435040ea9549fbfaad2625d98fb951 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:12 -0400 Subject: [PATCH 460/539] mm: memcontrol: constify the memcg and lruvec stat readers The memcg_page_state(), memcg_events(), and lruvec_page_state() families only read counters. Constify them. Use container_of_const() in the lruvec_page_state() family while at it, so the const isn't silently cast away. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-8-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/memcontrol.h | 23 ++++++++++++----------- mm/memcontrol-v1.h | 6 +++--- mm/memcontrol.c | 30 +++++++++++++++--------------- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 22067899eb6c..9beb065c0879 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -971,15 +971,16 @@ static inline void mod_memcg_page_state(const struct page *page, rcu_read_unlock(); } -unsigned long memcg_events(struct mem_cgroup *memcg, int event); -unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx); -unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item); +unsigned long memcg_events(const struct mem_cgroup *memcg, int event); +unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx); +unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item); bool memcg_stat_item_valid(int idx); bool memcg_vm_event_item_valid(enum vm_event_item idx); -unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx); -unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, +unsigned long lruvec_page_state(const struct lruvec *lruvec, + enum node_stat_item idx); +unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx); -unsigned long lruvec_page_state_local(struct lruvec *lruvec, +unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx); void mem_cgroup_flush_stats(struct mem_cgroup *memcg); @@ -1412,12 +1413,12 @@ static inline void mod_memcg_page_state(const struct page *page, { } -static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) +static inline unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx) { return 0; } -static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item) +static inline unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item) { return 0; } @@ -1432,19 +1433,19 @@ static inline bool memcg_vm_event_item_valid(enum vm_event_item idx) return false; } -static inline unsigned long lruvec_page_state(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state(lruvec_pgdat(lruvec), idx); } -static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state_monotonic(lruvec_pgdat(lruvec), idx); } -static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state(lruvec_pgdat(lruvec), idx); diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index 2cd37e1792d7..0952b2a783e5 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -37,9 +37,9 @@ static inline bool do_memsw_account(void) return !cgroup_subsys_on_dfl(memory_cgrp_subsys); } -unsigned long memcg_events_local(struct mem_cgroup *memcg, int event); -unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx); -unsigned long memcg_page_state_local_output(struct mem_cgroup *memcg, int item); +unsigned long memcg_events_local(const struct mem_cgroup *memcg, int event); +unsigned long memcg_page_state_local(const struct mem_cgroup *memcg, int idx); +unsigned long memcg_page_state_local_output(const struct mem_cgroup *memcg, int item); bool memcg1_alloc_events(struct mem_cgroup *memcg); void memcg1_free_events(struct mem_cgroup *memcg); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index adc93d28dbe7..1b0e511a1863 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -506,9 +506,9 @@ struct lruvec_stats { long state_pending[NR_MEMCG_NODE_STAT_ITEMS]; }; -unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) +unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx) { - struct mem_cgroup_per_node *pn; + const struct mem_cgroup_per_node *pn; long x; int i; @@ -519,7 +519,7 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0; - pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); x = READ_ONCE(pn->lruvec_stats->state[i]); #ifdef CONFIG_SMP if (x < 0) @@ -547,10 +547,10 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) * monotonically-incremented event counters are stored in * enum node_stat_item. */ -unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, +unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx) { - struct mem_cgroup_per_node *pn; + const struct mem_cgroup_per_node *pn; int i; if (mem_cgroup_disabled()) @@ -560,14 +560,14 @@ unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0; - pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); return (unsigned long)READ_ONCE(pn->lruvec_stats->state[i]); } -unsigned long lruvec_page_state_local(struct lruvec *lruvec, +unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx) { - struct mem_cgroup_per_node *pn; + const struct mem_cgroup_per_node *pn; long x; int i; @@ -578,7 +578,7 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec, if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0; - pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); x = READ_ONCE(pn->lruvec_stats->state_local[i]); #ifdef CONFIG_SMP if (x < 0) @@ -841,7 +841,7 @@ static void flush_memcg_stats_dwork(struct work_struct *w) queue_delayed_work(system_dfl_wq, &stats_flush_dwork, FLUSH_TIME); } -unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) +unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx) { long x; int i = memcg_stats_index(idx); @@ -964,7 +964,7 @@ void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, #ifdef CONFIG_MEMCG_V1 /* idx can be of type enum memcg_stat_item or node_stat_item. */ -unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) +unsigned long memcg_page_state_local(const struct mem_cgroup *memcg, int idx) { long x; int i = memcg_stats_index(idx); @@ -1127,7 +1127,7 @@ void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, put_cpu(); } -unsigned long memcg_events(struct mem_cgroup *memcg, int event) +unsigned long memcg_events(const struct mem_cgroup *memcg, int event) { int i = memcg_events_index(event); @@ -1146,7 +1146,7 @@ bool memcg_vm_event_item_valid(enum vm_event_item idx) } #ifdef CONFIG_MEMCG_V1 -unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) +unsigned long memcg_events_local(const struct mem_cgroup *memcg, int event) { int i = memcg_events_index(event); @@ -1729,14 +1729,14 @@ static int memcg_page_state_output_unit(int item) } } -unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item) +unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item) { return memcg_page_state(memcg, item) * memcg_page_state_output_unit(item); } #ifdef CONFIG_MEMCG_V1 -unsigned long memcg_page_state_local_output(struct mem_cgroup *memcg, int item) +unsigned long memcg_page_state_local_output(const struct mem_cgroup *memcg, int item) { return memcg_page_state_local(memcg, item) * memcg_page_state_output_unit(item); From 56c44e678eafa7e90eda54152c7e17bd9615b699 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:13 -0400 Subject: [PATCH 461/539] mm: memcontrol: constify the swap accounting helpers mem_cgroup_get_nr_swap_pages(), mem_cgroup_get_folio_swap_margin(), and mem_cgroup_swap_full() only read swap counters and limits. Constify them. Remove externs from function declarations while at it. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-9-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/swap.h | 12 ++++++------ mm/memcontrol.c | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 61005501888c..82f0bfec611f 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -527,9 +527,9 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_p __mem_cgroup_uncharge_swap(id, nr_pages); } -long mem_cgroup_get_folio_swap_margin(struct folio *folio); -extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg); -extern bool mem_cgroup_swap_full(struct folio *folio); +long mem_cgroup_get_folio_swap_margin(const struct folio *folio); +long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg); +bool mem_cgroup_swap_full(const struct folio *folio); #else static inline int mem_cgroup_try_charge_swap(struct folio *folio) { @@ -541,17 +541,17 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, { } -static inline long mem_cgroup_get_folio_swap_margin(struct folio *folio) +static inline long mem_cgroup_get_folio_swap_margin(const struct folio *folio) { return PAGE_COUNTER_MAX; } -static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) +static inline long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg) { return get_nr_swap_pages(); } -static inline bool mem_cgroup_swap_full(struct folio *folio) +static inline bool mem_cgroup_swap_full(const struct folio *folio) { return vm_swap_full(); } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1b0e511a1863..d954ffb43a43 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -6017,7 +6017,7 @@ void __mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages) rcu_read_unlock(); } -long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) +long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg) { long nr_swap_pages = get_nr_swap_pages(); @@ -6033,7 +6033,7 @@ long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) * * Return: Remaining chargeable pages in the folio's memcg hierarchy. */ -long mem_cgroup_get_folio_swap_margin(struct folio *folio) +long mem_cgroup_get_folio_swap_margin(const struct folio *folio) { struct mem_cgroup *memcg; long margin; @@ -6050,7 +6050,7 @@ long mem_cgroup_get_folio_swap_margin(struct folio *folio) return margin; } -bool mem_cgroup_swap_full(struct folio *folio) +bool mem_cgroup_swap_full(const struct folio *folio) { struct mem_cgroup *memcg; bool ret = false; From bc936f771a2d18bf2fa5090a5050f8d35b420abd Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:14 -0400 Subject: [PATCH 462/539] mm: memcontrol: constify mem_cgroup_swappiness() and mem_cgroup_get_max() Both only read swappiness and the memory limits. Constify them. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-10-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/memcontrol.h | 4 ++-- mm/memcontrol.c | 2 +- mm/swap.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 9beb065c0879..f11899085473 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -941,7 +941,7 @@ static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask) __mem_cgroup_handle_over_high(gfp_mask); } -unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg); +unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg); void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, struct task_struct *p); @@ -1372,7 +1372,7 @@ unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, return 0; } -static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) +static inline unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg) { return 0; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index d954ffb43a43..36101129679f 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1898,7 +1898,7 @@ void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) /* * Return the memory (and swap, if configured) limit for a memcg. */ -unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) +unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg) { unsigned long max = READ_ONCE(memcg->memory.max); diff --git a/mm/swap.h b/mm/swap.h index b3b54c28929a..1957960dc60d 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -82,7 +82,7 @@ enum swap_cluster_flags { extern int vm_swappiness; -static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) +static inline int mem_cgroup_swappiness(const struct mem_cgroup *memcg) { #ifdef CONFIG_MEMCG_V1 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && From de6be06f4a2dc88066e6227ec6864e7903985549 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:20:15 -0400 Subject: [PATCH 463/539] mm: memcontrol: constify the zswap and socket pressure helpers mem_cgroup_zswap_writeback_enabled() only reads the zswap_writeback flags, and mem_cgroup_get_socket_pressure() only reads the socket pressure timestamp. Constify them. Link: https://lore.kernel.org/20260915-folio_memcg-const-v3-11-c239a6010b58@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Muchun Song Acked-by: Shakeel Butt Acked-by: Lorenzo Stoakes (ARM) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie --- include/linux/memcontrol.h | 8 ++++---- mm/memcontrol.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index f11899085473..64c183be8cbf 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1684,7 +1684,7 @@ static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg) write_sequnlock_irqrestore(&memcg->socket_pressure_seqlock, flags); } -static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg) { unsigned int seq; u64 val; @@ -1702,7 +1702,7 @@ static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg) WRITE_ONCE(memcg->socket_pressure, jiffies + HZ); } -static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg) { return READ_ONCE(memcg->socket_pressure); } @@ -1937,7 +1937,7 @@ static inline void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, bool obj_cgroup_may_zswap(struct obj_cgroup *objcg); void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size); void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size); -bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg); +bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg); #else static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg) { @@ -1951,7 +1951,7 @@ static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size) { } -static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) +static inline bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg) { /* if zswap is disabled, do not block pages going to the swapping device */ return true; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 36101129679f..4d00748c8a5b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -6318,7 +6318,7 @@ void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size) rcu_read_unlock(); } -bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) +bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg) { /* if zswap is disabled, do not block pages going to the swapping device */ if (!zswap_is_enabled()) From 103142868af5a61bc0399ebe71a4dca50d48ff43 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 16 Sep 2026 05:51:22 -0700 Subject: [PATCH 464/539] mm: filemap: move lruvec accounting outside the xarray lock __filemap_add_folio() inserts a folio and updates mapping->nrpages while holding mapping->i_pages.xa_lock with interrupts disabled. The XArray insertion and nrpages update require the lock, but the lruvec statistic updates do not. With CONFIG_MEMCG, those calls also update per-CPU memcg and lruvec counters and notify cgroup rstat, extending the critical section. Move the lruvec accounting after a successful XArray insertion and after xas_unlock_irq(). The page-cache references pin the folio, while the folio lock keeps folio->mapping stable and prevents removal until accounting is complete. This moves one lruvec update for ordinary folios and a second for PMD-mappable folios out of the serialized section. In a 30-second system-wide perf lock contention -ab capture on a production host, the hottest caller-stack record attributed to __filemap_add_folio() had 20,867 contentions and 557.930 ms total wait. That was 14% of the 3.998 seconds of aggregate lock wait in the capture. Moving lruvec accuting outside of critical section should help optimize it. Link: https://lore.kernel.org/20260916125122.2696271-1-usama.arif@linux.dev Signed-off-by: Usama Arif Signed-off-by: Andrew Morton Reviewed-by: Shakeel Butt Acked-by: Muchun Song Reviewed-by: Vishal Moola (Fractile) Reviewed-by: Jan Kara Cc: David Hildenbrand Cc: Johannes Weiner Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Rik van Riel Cc: Roman Gushchin --- mm/filemap.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 00fd89cf6f55..4720bbfc1a66 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -918,14 +918,6 @@ noinline int __filemap_add_folio(struct address_space *mapping, mapping->nrpages += nr; - /* hugetlb pages do not participate in page cache accounting */ - if (!huge) { - lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); - if (folio_test_pmd_mappable(folio)) - lruvec_stat_mod_folio(folio, - NR_FILE_THPS, nr); - } - unlock: xas_unlock_irq(&xas); @@ -942,6 +934,13 @@ unlock: if (xas_error(&xas)) goto error; + /* hugetlb pages do not participate in page cache accounting */ + if (!huge) { + lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); + if (folio_test_pmd_mappable(folio)) + lruvec_stat_mod_folio(folio, NR_FILE_THPS, nr); + } + trace_mm_filemap_add_to_page_cache(folio); return 0; error: From 331f94007d79b3c02d3098eea469a92aea3efc2c Mon Sep 17 00:00:00 2001 From: Yuanhe Shu Date: Wed, 16 Sep 2026 19:25:45 +0800 Subject: [PATCH 465/539] mm/page_alloc: do not boost watermarks in kdump capture kernels A watermark boost is not confined to one watermark: wmark_pages() adds it to min, low and high alike, so every watermark check sees it, including should_reclaim_retry() and the last ditch ALLOC_WMARK_HIGH attempt in __alloc_pages_may_oom(). Once the boost exceeds the memory still free the allocator gives up and invokes the OOM killer, and a capture kernel that is still booting has nothing to kill: the boot panics and the vmcore is lost. Seen on an arm64 machine with 64K pages, CONFIG_PAGE_BLOCK_MAX_ORDER=10 (pageblock = 64M) and crashkernel=512M, running a distribution kernel based on 7.0.14. A high order UNMOVABLE allocation fell back to a MOVABLE pageblock while the capture kernel was still in do_initcalls(): Node 0 DMA free:68096kB boost:65536kB min:68160kB low:68800kB high:69440kB managed:479168kB Out of memory and no killable processes... Kernel panic - not syncing: System is deadlocked on memory The zone was not short of memory. Subtracting the boost gives min:2624kB low:3264kB high:3904kB, so the 68096kB still free sat 17 times above the high watermark and the allocator would not even have entered its slow path. The boost supplied 65536kB of the 68160kB min and by itself put the zone 64kB under water. It is that large because boost_watermark() clamps it with max(pageblock_nr_pages, max_boost); watermark_boost_factor alone would have allowed 5824kB. Commit 14f69140ff9c ("mm: limit boost_watermark on small zones") already tried to protect capture kernels, but it infers them from the zone size and skips the boost only below four pageblocks. arm64 64K pageblocks were 512M then, so the guard reached zones up to 2G; CONFIG_PAGE_BLOCK_MAX_ORDER can cap them at 64M, which shrinks the guard to zones under 256M and lets this 468M zone through. kdump is a property of the kernel, not of the zone, so test for it directly. A capture kernel exits within seconds and never uses the fragmentation avoidance the boost buys. Normal kernels are unaffected: the size based check still covers their genuinely tiny zones. Passing sysctl.vm.watermark_boost_factor=0 to the capture kernel does not cover this window: sysctl.* parameters are written through procfs by do_sysctl_args(), which runs after do_initcalls() where the panic above happened, and watermark_boost_factor has no early_param of its own. Link: https://lore.kernel.org/20260916112545.3707893-1-xiangzao@linux.alibaba.com Fixes: 1c30844d2dfe ("mm: reclaim small amounts of memory when an external fragmentation event occurs") Signed-off-by: Yuanhe Shu Signed-off-by: Andrew Morton Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Johannes Weiner Cc: Henry Willard Cc: Brendan Jackman Cc: David Hildenbrand Cc: Mel Gorman Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Zi Yan Cc: # v5.0 --- mm/page_alloc.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3265f7ff909a..ff7b9acf0144 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -2160,10 +2161,17 @@ static inline bool boost_watermark(struct zone *zone) if (!watermark_boost_factor) return false; + + /* + * A kdump capture kernel exits before a boost can pay off, while + * the raised watermark can exceed the memory left for the dump. + */ + if (is_kdump_kernel()) + return false; + /* * Don't bother in zones that are unlikely to produce results. - * On small machines, including kdump capture kernels running - * in a small area, boosting the watermark can cause an out of + * On small machines, boosting the watermark can cause an out of * memory situation immediately. */ if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) From dba177ad7829cb70de1054b49b6ec9999bb595c2 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 16 Sep 2026 12:31:52 +0800 Subject: [PATCH 466/539] mm: mincore: use per-vma lock during page table walk do_mincore() performs a read-only, per-VMA residency query, making it a good candidate for per-VMA locking. Convert it to acquire the per-VMA lock, thereby reducing contention on the per-MM mmap_lock. Link: https://lore.kernel.org/20260916043153.2631696-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Signed-off-by: Andrew Morton Reviewed-by: Pedro Falcato Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Cc: Jann Horn Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Zi Yan --- mm/mincore.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/mm/mincore.c b/mm/mincore.c index c086836bc4bc..0fe50f8a7e62 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -235,24 +235,22 @@ static const struct mm_walk_ops mincore_walk_ops = { .pmd_entry = mincore_pte_range, .pte_hole = mincore_unmapped_range, .hugetlb_entry = mincore_hugetlb, - .walk_lock = PGWALK_RDLOCK, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, }; /* * Do a chunk of "sys_mincore()". We've already checked - * all the arguments, we hold the mmap semaphore: we should + * all the arguments, we hold the VMA read lock: we should * just return the amount of info we're asked for. */ -static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *vec) +static long do_mincore(struct vm_area_struct *vma, unsigned long addr, + unsigned long pages, unsigned char *vec) { - struct vm_area_struct *vma; - unsigned long end; + unsigned long end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); int err; - vma = vma_lookup(current->mm, addr); - if (!vma) - return -ENOMEM; - end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); + vma_assert_locked(vma); + if (!can_do_mincore(vma)) { unsigned long pages = DIV_ROUND_UP(end - addr, PAGE_SIZE); memset(vec, 1, pages); @@ -319,13 +317,20 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, retval = 0; while (pages) { + struct vm_area_struct *vma; + + vma = vma_start_read_unlocked(current->mm, start); + if (!vma) { + retval = -ENOMEM; + break; + } + /* * Do at most PAGE_SIZE entries per iteration, due to * the temporary buffer size. */ - mmap_read_lock(current->mm); - retval = do_mincore(start, min(pages, PAGE_SIZE), tmp); - mmap_read_unlock(current->mm); + retval = do_mincore(vma, start, min(pages, PAGE_SIZE), tmp); + vma_end_read(vma); if (retval <= 0) break; From d8020965b2f841fc5d71488c4261864ee9cbbec2 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Tue, 15 Sep 2026 19:42:15 -0400 Subject: [PATCH 467/539] vmcore: convert mmap_vmcore_fault() to use folios Use a folio for the page cache page the s390 fault handler reads the old kernel's memory into. This removes four compound_head() calls and one of the last callers of find_or_create_page(). The vmcore mapping only has order-0 folios, so the logic is unchanged. Compile tested for s390. Link: https://lore.kernel.org/20260915-vmcore-fault-folio-v1-1-a0cb6278670f@columbia.edu Signed-off-by: Tal Zussman Signed-off-by: Andrew Morton Acked-by: Pratyush Yadav Cc: Baoquan He Cc: Dave Young Cc: Mike Rapoport Cc: Pasha Tatashin --- fs/proc/vmcore.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c index 44d15436439f..2dc3803aa11d 100644 --- a/fs/proc/vmcore.c +++ b/fs/proc/vmcore.c @@ -474,29 +474,30 @@ static vm_fault_t mmap_vmcore_fault(struct vm_fault *vmf) pgoff_t index = vmf->pgoff; struct iov_iter iter; struct kvec kvec; - struct page *page; + struct folio *folio; loff_t offset; int rc; - page = find_or_create_page(mapping, index, GFP_KERNEL); - if (!page) + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); + if (IS_ERR(folio)) return VM_FAULT_OOM; - if (!PageUptodate(page)) { - offset = (loff_t) index << PAGE_SHIFT; - kvec.iov_base = page_address(page); - kvec.iov_len = PAGE_SIZE; - iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, PAGE_SIZE); + if (!folio_test_uptodate(folio)) { + offset = folio_pos(folio); + kvec.iov_base = folio_address(folio); + kvec.iov_len = folio_size(folio); + iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, folio_size(folio)); rc = __read_vmcore(&iter, &offset); if (rc < 0) { - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); return vmf_error(rc); } - SetPageUptodate(page); + folio_mark_uptodate(folio); } - unlock_page(page); - vmf->page = page; + folio_unlock(folio); + vmf->page = folio_file_page(folio, index); return 0; #else return VM_FAULT_SIGBUS; From 4154a1cf2f70ec542c4e24355b43ddaff32c31d6 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Mon, 21 Sep 2026 17:13:02 +0200 Subject: [PATCH 468/539] mm: swap: move LRU insertion out of the swap cache allocator Patch series "mm: zswap: free cold writeback folios promptly", v6. When zswap writes an entry back, it allocates an order-0 swap cache folio, decompresses into it, and issues the write. The folio is cold by construction, yet today it is left on the LRU for page reclaim to find and free later. That wastes a reclaim scan and keeps cold memory resident longer than necessary. Rather than implement this in zswap, extend the existing dropbehind mechanism to swap cache folios and have zswap opt into it (Yosry). A PG_dropbehind folio is already dropped from its cache once writeback completes instead of being left for reclaim; for a swap cache folio that "drop" is removing it from the swap cache. Patch 1 - move LRU insertion out of the swap cache allocator into its callers, so zswap writeback can allocate off the LRU. Patch 2 - drop dropbehind swap cache folios on writeback completion. Patch 3 - zswap allocates its writeback folio off the LRU and marks it dropbehind, opting into the mechanism above. This patch (of 3): This is a preparatory patch. swap_cache_alloc_folio() adds the new folio to the LRU itself, which leaves its callers no way to act on the folio before it becomes visible to reclaim. Two users need exactly that: - moving the refault evaluation out of the swap cache folio allocation requires it to happen before folio_add_lru(): that consumes PG_active to file the folio on the inactive or the active list, and under MGLRU it also reads PG_workingset to pick the generation. Setting either flag afterwards does not move the folio; - zswap writeback dropbehind needs the buffer folio to stay off the LRU entirely, as the per-CPU LRU batch would hold a reference on it and keep remove_mapping() from freeing it once writeback completes. Defer the LRU insertion to the callers and rename the helper to __swap_cache_alloc_folio(): each caller adds the folio right after the allocation, so there is no functional change intended. Link: https://lore.kernel.org/20260921151306.625134-1-alex@ghiti.fr Link: https://lore.kernel.org/20260921151306.625134-2-alex@ghiti.fr Signed-off-by: Alexandre Ghiti Signed-off-by: Andrew Morton Suggested-by: Kairui Song Reviewed-by: Kairui Song Reviewed-by: Nhat Pham Reviewed-by: Kunwu Chan Acked-by: Usama Arif Reviewed-by: Barry Song Cc: Al Viro Cc: Axel Rasmussen Cc: Baoquan He Cc: Chengming Zhou Cc: Chis Li (Google) Cc: Christian Brauner (Amutable) Cc: "David Hildenbrand (arm)" Cc: Jan Kara Cc: Johannes Weiner Cc: Kemeng Shi Cc: Lorenzo Stoakes (ARM) Cc: Matthew Wilcox Cc: Michal Hocko Cc: Qi Zheng Cc: Shakeel Butt Cc: Tal Zussman Cc: Wei Xu Cc: Yosry Ahmed Cc: Youngjun Park Cc: Yuanchu Xie --- mm/swap.h | 6 +++--- mm/swap_state.c | 20 ++++++++++++-------- mm/swapfile.c | 2 +- mm/zswap.c | 5 +++-- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/mm/swap.h b/mm/swap.h index 1957960dc60d..d5bf21f517dc 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -312,9 +312,9 @@ bool swap_cache_has_folio(swp_entry_t entry); struct folio *swap_cache_get_folio(swp_entry_t entry); void *swap_cache_get_shadow(swp_entry_t entry); void swap_cache_del_folio(struct folio *folio); -struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, - unsigned long orders, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx); +struct folio *__swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, + unsigned long orders, struct vm_fault *vmf, + struct mempolicy *mpol, pgoff_t ilx); /* Below helpers require the caller to lock and pass in the swap cluster. */ void __swap_cache_add_folio(struct swap_cluster_info *ci, struct folio *folio, swp_entry_t entry); diff --git a/mm/swap_state.c b/mm/swap_state.c index cef44aadee61..87790369dd9a 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -497,13 +497,11 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages); lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages); - /* Caller will initiate read into locked new_folio */ - folio_add_lru(folio); return folio; } /** - * swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache. + * __swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache. * @targ_entry: swap entry indicating the target slot * @gfp: memory allocation flags * @orders: allocation orders, must be non zero @@ -515,13 +513,17 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, * doing IO (e.g. swap in or zswap writeback). The swap slot indicated by * @targ_entry must have a non-zero swap count (swapped out). * + * The returned folio is locked and is NOT on the LRU. The caller must either + * add it to the LRU with folio_add_lru() so page reclaim can find it, or free + * it directly once done; a folio left off the LRU is unreclaimable and leaks. + * * Context: Caller must protect the swap device with reference count or locks. * Return: Returns the folio if allocation succeeded and folio is in the swap * cache. Returns error code if failed due to race, OOM or invalid arguments. */ -struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, - unsigned long orders, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx) +struct folio *__swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, + unsigned long orders, struct vm_fault *vmf, + struct mempolicy *mpol, pgoff_t ilx) { int order, err; struct folio *ret; @@ -657,12 +659,13 @@ static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx, folio = swap_cache_get_folio(entry); if (folio) return folio; - folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx); + folio = __swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx); } while (PTR_ERR(folio) == -EEXIST); if (IS_ERR_OR_NULL(folio)) return NULL; + folio_add_lru(folio); swap_read_folio(ctx, folio); if (readahead) { folio_set_readahead(folio); @@ -698,12 +701,13 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, folio = swap_cache_get_folio(entry); if (folio) return folio; - folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); + folio = __swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); } while (PTR_ERR(folio) == -EEXIST); if (IS_ERR(folio)) return folio; + folio_add_lru(folio); swap_read_folio(&ctx, folio); swap_read_submit(&ctx); return folio; diff --git a/mm/swapfile.c b/mm/swapfile.c index 2cd0d0ba966c..16ceaa28db3a 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1887,7 +1887,7 @@ void folio_put_swap(struct folio *folio, struct page *page) * CPU1 CPU2 * do_swap_page() * ... swapoff+swapon - * swap_cache_alloc_folio() + * __swap_cache_alloc_folio() * // check swap_map * // verify PTE not changed * diff --git a/mm/zswap.c b/mm/zswap.c index 584dd3063769..c79cca61abf9 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1019,8 +1019,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry, return -ENOENT; mpol = get_task_policy(current); - folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol, - NO_INTERLEAVE_INDEX); + folio = __swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol, + NO_INTERLEAVE_INDEX); put_swap_device(si); /* @@ -1032,6 +1032,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, */ if (IS_ERR(folio)) return PTR_ERR(folio); + folio_add_lru(folio); /* * folio is locked, and the swapcache is now secured against From 7ccf5a3e5bdb9714041ab8199c2f8b1bb60e3b3c Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Mon, 21 Sep 2026 17:13:03 +0200 Subject: [PATCH 469/539] mm: swap: drop dropbehind swap cache folios on writeback completion A PG_dropbehind folio is dropped from its cache once writeback completes rather than left for reclaim to find later; this is implemented for file folios in folio_end_dropbehind(). Extend it to swap cache folios. The drop blocks on the folio lock, so it cannot run in interrupt context. Set BIO_COMPLETE_IN_TASK on the write, as the file dropbehind paths do, and drop the folio directly from folio_end_writeback(). It has to block rather than trylock: the folio is off the LRU, so skipping it would leave it in the swap cache with nothing able to reclaim it, and it cannot be put back while another thread holds its lock. Link: https://lore.kernel.org/20260921151306.625134-3-alex@ghiti.fr Signed-off-by: Alexandre Ghiti Signed-off-by: Andrew Morton Suggested-by: Yosry Ahmed Suggested-by: Johannes Weiner Suggested-by: Nhat Pham Reviewed-by: Nhat Pham Reviewed-by: Kunwu Chan Reviewed-by: Barry Song Cc: Al Viro Cc: Axel Rasmussen Cc: Baoquan He Cc: Chengming Zhou Cc: Chis Li Cc: Christian Brauner (Amutable) Cc: "David Hildenbrand (arm)" Cc: Jan Kara Cc: Kairui Song Cc: Kemeng Shi Cc: Lorenzo Stoakes (ARM) Cc: Matthew Wilcox Cc: Michal Hocko Cc: Qi Zheng Cc: Shakeel Butt Cc: Tal Zussman Cc: Usama Arif Cc: Wei Xu Cc: Youngjun Park Cc: Yuanchu Xie --- include/linux/swap.h | 6 ++++++ mm/filemap.c | 19 +++++++++++++++++ mm/page_io.c | 9 ++++++++ mm/swap_state.c | 42 +++++++++++++++++++++++++++++++++++++ mm/vmscan.c | 49 +++++++++++++++++++++++++++++++++++--------- 5 files changed, 115 insertions(+), 10 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 82f0bfec611f..cb434cccd653 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -336,6 +336,9 @@ static inline bool lru_cache_disabled(void) extern unsigned long shrink_all_memory(unsigned long nr_pages); long remove_mapping(struct address_space *mapping, struct folio *folio); +long remove_mapping_set_shadow(struct address_space *mapping, + struct folio *folio, + struct mem_cgroup *target_memcg); #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) extern int reclaim_register_node(struct node *node); @@ -421,6 +424,8 @@ void swap_put_entries_direct(swp_entry_t entry, int nr); */ bool folio_free_swap(struct folio *folio); +void swap_writeback_dropbehind_folio(struct folio *folio); + /* Allocate / free (hibernation) exclusive entries */ swp_entry_t swap_alloc_hibernation_slot(int type); void swap_free_hibernation_slot(swp_entry_t entry); @@ -431,6 +436,7 @@ static inline void put_swap_device(struct swap_info_struct *si) } #else /* CONFIG_SWAP */ +static inline void swap_writeback_dropbehind_folio(struct folio *folio) {} static inline struct swap_info_struct *get_swap_device(swp_entry_t entry) { return NULL; diff --git a/mm/filemap.c b/mm/filemap.c index 4720bbfc1a66..b74bc1e5015c 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1685,6 +1685,8 @@ EXPORT_SYMBOL_GPL(folio_end_writeback_no_dropbehind); */ void folio_end_writeback(struct folio *folio) { + bool swap_dropbehind; + VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio); /* @@ -1694,7 +1696,24 @@ void folio_end_writeback(struct folio *folio) * reused before the folio_wake_bit(). */ folio_get(folio); + + /* + * Sample this before folio_end_writeback_no_dropbehind() clears + * PG_writeback: until then a racing swapin cannot remove the folio from + * the swap cache. Afterwards it can, and the drop below then finds a + * non-swapcache folio and puts it back on the LRU instead. The + * reference taken above keeps the folio alive across that window. + */ + swap_dropbehind = folio_test_swapcache(folio) && + folio_test_dropbehind(folio); + folio_end_writeback_no_dropbehind(folio); + + if (swap_dropbehind) { + swap_writeback_dropbehind_folio(folio); + return; + } + folio_end_dropbehind(folio); folio_put(folio); } diff --git a/mm/page_io.c b/mm/page_io.c index 5f7756e370f7..0808808f0309 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -608,6 +608,15 @@ static void swap_bdev_submit_write(struct swap_io_ctx *ctx) submit_bio_wait(bio); end_swap_bio_write(bio); } else { + int p; + + for (p = 0; p < sio->nr_bvecs; p++) { + if (folio_test_dropbehind(bvec_folio(&sio->bvecs[p]))) { + bio_set_flag(bio, BIO_COMPLETE_IN_TASK); + break; + } + } + bio->bi_end_io = end_swap_bio_write; submit_bio(bio); } diff --git a/mm/swap_state.c b/mm/swap_state.c index 87790369dd9a..2475ba29126d 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -551,6 +551,48 @@ struct folio *__swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, return ret; } +/** + * swap_writeback_dropbehind_folio - drop a dropbehind swap cache folio + * @folio: the off-LRU folio whose writeback has completed + * + * Context: task context, with the reference taken by folio_end_writeback() + * donated to us. + */ +void swap_writeback_dropbehind_folio(struct folio *folio) +{ + struct mem_cgroup *memcg; + + folio_lock(folio); + + /* The folio was allocated off the LRU and nothing re-adds it here. */ + VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); + + rcu_read_lock(); + memcg = folio_memcg(folio); + if (!mem_cgroup_tryget(memcg)) + memcg = NULL; + rcu_read_unlock(); + + /* + * Gate remove_mapping_set_shadow() on folio_test_swapcache(): a racing + * swapin may have freed the swap slot (folio_free_swap()) and dropped the + * folio from the cache, and it must not run on a non-swapcache folio (it + * would trip __remove_mapping()'s mapping == folio_mapping() check). + */ + if (!folio_test_swapcache(folio) || folio_test_writeback(folio) || + !remove_mapping_set_shadow(swap_address_space(folio->swap), folio, + memcg)) { + /* Raced: the folio is now owned by the swapin; put it back. */ + folio_clear_dropbehind(folio); + folio_add_lru(folio); + } + + mem_cgroup_put(memcg); + + folio_unlock(folio); + folio_put(folio); +} + /* * If we are the only user, then try to free up the swap cache. * diff --git a/mm/vmscan.c b/mm/vmscan.c index e09da1ebec0e..01a1a47417fb 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -854,6 +854,22 @@ cannot_free: return 0; } +static long __remove_mapping_unfreeze(struct address_space *mapping, + struct folio *folio, bool reclaimed, + struct mem_cgroup *target_memcg) +{ + if (__remove_mapping(mapping, folio, reclaimed, target_memcg)) { + /* + * Unfreezing the refcount with 1 effectively + * drops the pagecache ref for us without requiring another + * atomic operation. + */ + folio_ref_unfreeze(folio, 1); + return folio_nr_pages(folio); + } + return 0; +} + /** * remove_mapping() - Attempt to remove a folio from its mapping. * @mapping: The address space. @@ -868,16 +884,29 @@ cannot_free: */ long remove_mapping(struct address_space *mapping, struct folio *folio) { - if (__remove_mapping(mapping, folio, false, NULL)) { - /* - * Unfreezing the refcount with 1 effectively - * drops the pagecache ref for us without requiring another - * atomic operation. - */ - folio_ref_unfreeze(folio, 1); - return folio_nr_pages(folio); - } - return 0; + return __remove_mapping_unfreeze(mapping, folio, false, NULL); +} + +/** + * remove_mapping_set_shadow() - Remove a folio and record an eviction shadow. + * @mapping: The address space. + * @folio: The folio to remove. + * @target_memcg: The memcg to charge the eviction shadow to; the caller must + * keep it alive across the call. + * + * Like remove_mapping(), but stores a workingset eviction shadow the way page + * reclaim does, so that a later refault can be detected and the folio + * re-activated. + * Return: The number of pages removed from the mapping. 0 if the folio + * could not be removed. + * Context: The caller should have a single refcount on the folio and + * hold its lock. + */ +long remove_mapping_set_shadow(struct address_space *mapping, + struct folio *folio, + struct mem_cgroup *target_memcg) +{ + return __remove_mapping_unfreeze(mapping, folio, true, target_memcg); } /** From 4fce3f3c696d2af64546361cd2aa293597dbbce4 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Mon, 21 Sep 2026 17:13:04 +0200 Subject: [PATCH 470/539] mm: zswap: drop cold writeback folios via swap dropbehind zswap writeback decompresses an entry into a fresh swap cache folio and writes it back. The folio is cold by construction, yet it is left on the LRU for reclaim to find and free later, wasting a reclaim scan and keeping cold memory resident longer than necessary. Allocate the folio off the LRU and mark it PG_dropbehind so the swap dropbehind path frees it from the swap cache once writeback completes. __swap_cache_alloc_folio() evaluates a refault on the new folio, and workingset_refault() sets PG_active when it looks recent. Until now folio_add_lru() consumed that flag and __page_cache_release() cleared it once the folio left the LRU. This folio never reaches the LRU, so nothing would clear PG_active and the folio would be freed with a PAGE_FLAGS_CHECK_AT_FREE flag set, tripping bad_page() under CONFIG_DEBUG_VM. Clear it after allocation. That is a workaround: the refault should not be evaluated on a writeback buffer at all. A fix for that is on the mailing list [1]. Link: https://lore.kernel.org/20260921151306.625134-4-alex@ghiti.fr Link: https://lore.kernel.org/linux-mm/20260911092012.92399-1-alex@ghiti.fr/ [1] Signed-off-by: Alexandre Ghiti Signed-off-by: Andrew Morton Suggested-by: Johannes Weiner Suggested-by: Nhat Pham Reviewed-by: Nhat Pham Reviewed-by: Kunwu Chan Cc: Al Viro Cc: Axel Rasmussen Cc: Baoquan He Cc: Barry Song Cc: Chengming Zhou Cc: Chis Li Cc: Christian Brauner (Amutable) Cc: "David Hildenbrand (arm)" Cc: Jan Kara Cc: Kairui Song Cc: Kemeng Shi Cc: Lorenzo Stoakes (ARM) Cc: Matthew Wilcox Cc: Michal Hocko Cc: Qi Zheng Cc: Shakeel Butt Cc: Tal Zussman Cc: Usama Arif Cc: Wei Xu Cc: Yosry Ahmed Cc: Youngjun Park Cc: Yuanchu Xie --- mm/zswap.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index c79cca61abf9..ae19e301fced 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1032,7 +1032,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry, */ if (IS_ERR(folio)) return PTR_ERR(folio); - folio_add_lru(folio); + + folio_clear_active(folio); /* * folio is locked, and the swapcache is now secured against @@ -1046,12 +1047,12 @@ static int zswap_writeback_entry(struct zswap_entry *entry, tree = swap_zswap_tree(swpentry); if (entry != xa_load(tree, offset)) { ret = -ENOMEM; - goto out; + goto err; } if (!zswap_decompress(entry, folio)) { ret = -EIO; - goto out; + goto err; } xa_erase(tree, offset); @@ -1065,18 +1066,30 @@ static int zswap_writeback_entry(struct zswap_entry *entry, /* folio is up to date */ folio_mark_uptodate(folio); - /* move it to the tail of the inactive list after end_writeback */ - folio_set_reclaim(folio); + folio_set_dropbehind(folio); + + /* + * Drop our reference before starting writeback so the swap cache holds + * the only one: the drop in folio_end_writeback() needs that for + * remove_mapping_set_shadow() to succeed, otherwise the folio is + * handed back to reclaim instead. + * + * Nothing can free the folio in the meantime: we hold the folio lock + * until writeback starts, PG_writeback then blocks swap cache removal, + * and folio_end_writeback() takes its own reference before clearing + * PG_writeback and donates it to the drop. + */ + folio_put(folio); /* start writeback */ __swap_writeout(&ctx, folio); swap_write_submit(&ctx); -out: - if (ret) { - swap_cache_del_folio(folio); - folio_unlock(folio); - } + return 0; + +err: + swap_cache_del_folio(folio); + folio_unlock(folio); folio_put(folio); return ret; } From 4c7d26f11f81cb6c9ea2caaa367bb1a0f7bf4b3a Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:21:59 -0700 Subject: [PATCH 471/539] mm/damon/api: remove NR_DAMOS_FILTER_TYPES Patch series "mm/damon: improve readability, clarity and test coverage". Yet another batch of miscellaneous DAMON minor improvements. Mostly focused on readability and clarity of code and document, and unit/self test coverage. No user-visible behavioral change is intended. This patch (of 10): Nobody uses NR_DAMOS_FILTER_TYPES. Remove it. Link: https://lore.kernel.org/20260917142210.90829-1-sj@kernel.org Link: https://lore.kernel.org/20260917142210.90829-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- include/linux/damon.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index bbb190b47401..836353c4ab9a 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -400,7 +400,6 @@ struct damos_stat { * @DAMOS_FILTER_TYPE_ADDR: Address range. * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. * @DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: probe_hits weighted sum range. - * @NR_DAMOS_FILTER_TYPES: Number of filter types. * * All types except &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET and * &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM are handled by the underlying &struct @@ -422,7 +421,6 @@ enum damos_filter_type { DAMOS_FILTER_TYPE_ADDR, DAMOS_FILTER_TYPE_TARGET, DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, - NR_DAMOS_FILTER_TYPES, }; /** From 9c6e6207aa4f45a8d94e59b78c94877f6bdc5850 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:00 -0700 Subject: [PATCH 472/539] mm/damon/core: use abs_diff() in damon_feed_loop_next_input() damon_feed_loop_next_input() is open-coding absolute diff calculation instead of the dedicated helper, abs_diff(), for no good reason. Use the dedicated helper. Link: https://lore.kernel.org/20260917142210.90829-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index e1b49c3d72b8..e49bbf7c07e8 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2926,10 +2926,7 @@ static unsigned long damon_feed_loop_next_input(unsigned long last_input, if (score >= goal * 2) return min_input; - if (over_achieving) - score_goal_diff = score - goal; - else - score_goal_diff = goal - score; + score_goal_diff = abs_diff(score, goal); if (last_input < ULONG_MAX / score_goal_diff) compensation = last_input * score_goal_diff / goal; From f5a038a92b254c2d10df80ed66992311686f2929 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:01 -0700 Subject: [PATCH 473/539] mm/damon/core: use mult_frac() in damon_feed_loop_next_input() damon_feed_loop_next_input() does its best effort overflow protection. score_goal_diff is always smaller than goal (10,000). Hence the calculation can be replaced to use mult_frac() without concerning the overflow. Use mult_frac(). Link: https://lore.kernel.org/20260917142210.90829-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index e49bbf7c07e8..f3ccfc9ad0f0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2927,11 +2927,7 @@ static unsigned long damon_feed_loop_next_input(unsigned long last_input, return min_input; score_goal_diff = abs_diff(score, goal); - - if (last_input < ULONG_MAX / score_goal_diff) - compensation = last_input * score_goal_diff / goal; - else - compensation = last_input / goal * score_goal_diff; + compensation = mult_frac(last_input, score_goal_diff, goal); if (over_achieving) return max(last_input - compensation, min_input); From 1ead8a1db06612aec260038dc418f6afd21a6027 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:02 -0700 Subject: [PATCH 474/539] mm/damon/core: set damon_ctx->walk_control_obsolete in damon_new_ctx() damos_walk() should be called for a damon_ctx context that has successfully started at least once. That's because damon_ctx->walk_control_obsolete is initialized when kdamond starts. If the rule is violated, an indefinite wait can happen. There is no existing violation of the rule. damon_call() had a similar rule, and it turned out keeping the rule is not easy for damon_call()'s case. Hence, commit 8023b5f47e09 ("mm/damon/core: set ctx->call_controls_obsolete in damon_new_ctx()") added the initialization in damon_new_ctx() and removed the rule. Keeping the rule for damos_walk() is relatively easier. But having slightly different rules for similar functions could be confusing. Sashiko, for example, repeatedly asked questions about this. Do the initialization of walk_control_obsolete in damon_new_ctx() for consistency. Link: https://lore.kernel.org/20260917142210.90829-5-sj@kernel.org Link: https://lore.kernel.org/20260915011614.102342-1-sj@kernel.org [1] Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index f3ccfc9ad0f0..327277ba3658 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -939,6 +939,7 @@ struct damon_ctx *damon_new_ctx(void) INIT_LIST_HEAD(&ctx->schemes); ctx->call_controls_obsolete = true; + ctx->walk_control_obsolete = true; prandom_seed_state(&ctx->rnd_state, get_random_u64()); return ctx; @@ -2308,10 +2309,6 @@ canceled: * passed at least one &damos->apply_interval_us, kdamond marks the request as * completed so that damos_walk() can wakeup and return. * - * Note that this function should be called only after damon_start() with the - * @ctx has succeeded. Otherwise, this function could fall into an indefinite - * wait. - * * Return: 0 on success, negative error code otherwise. */ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) From 0ba2507de806764577d356829d4d979215d22fb1 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:03 -0700 Subject: [PATCH 475/539] mm/damon/core: document damon_call()/damon_start() race hang issue Let's suppose damon_start() and damon_call() are executed in parallel for the same DAMON context. Then, damon_call() could show ctx->damon_calls_obsolete set while ctx->kdamond is unset. If damon_start() sets ctx->kdamond before damon_call() starts the cancelling, damon_call() can indefinitely hang. No DAMON API caller does such parallel execution of damon_start() and damon_call(), so the issue doesn't exist. But who knows what will happen in future. Add a clarification comment for caution. Link: https://lore.kernel.org/20260917142210.90829-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 327277ba3658..add1b7afb957 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2258,6 +2258,9 @@ int damon_kdamond_pid(struct damon_ctx *ctx) * * When this function is failed, the @ctx is guaranteed to be stopped. * + * This function should not be called in parallel to damon_start() for the + * @ctx. In the case, this function could indefinitely hang. + * * Return: 0 on success, negative error code otherwise. */ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) From 8de5dfa5d8d0ec82f83f796d0ab2bcba3cd767e4 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:04 -0700 Subject: [PATCH 476/539] mm/damon/paddr: remove pa parameter from damon_pa_filter_pass() damon_pa_filter_pass() receives the 'pa' parameter, but doesn't use it. Remove the parameter from the function signature. Link: https://lore.kernel.org/20260917142210.90829-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/paddr.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5abfabaa339e..2cfdc356b415 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -163,8 +163,7 @@ static bool damon_pa_filter_match(struct damon_filter *filter, return matched == filter->matching; } -static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, - struct damon_probe *p) +static bool damon_pa_filter_pass(struct folio *folio, struct damon_probe *p) { struct damon_filter *f; bool pass = true; @@ -200,7 +199,7 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, ctx->addr_unit); folio = damon_get_folio(PHYS_PFN(pa)); damon_for_each_probe(p, ctx) { - if (damon_pa_filter_pass(pa, folio, p)) + if (damon_pa_filter_pass(folio, p)) r->probe_hits[i]++; i++; } From 266909694598fb905de7fba589e432900a27e687 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:05 -0700 Subject: [PATCH 477/539] mm/damon/tests/core-kunit: test eligible_mem_bp commitment There was a DAMOS quota goal commit bug [1] that doesn't update the nid field for DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP metric goal. Add a kunit test case for confirming nid commitment. Link: https://lore.kernel.org/20260917142210.90829-8-sj@kernel.org Link: https://lore.kkernel.org/20260827045035.94611-1-sj@kernel.org [1] Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/tests/core-kunit.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 5da84caf4124..1f19fefdd98c 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -835,6 +835,9 @@ static void damos_test_commit_quota_goal_for(struct kunit *test, KUNIT_EXPECT_EQ(test, dst->nid, src->nid); KUNIT_EXPECT_EQ(test, dst->memcg_id, src->memcg_id); break; + case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP: + KUNIT_EXPECT_EQ(test, dst->nid, src->nid); + break; default: break; } @@ -898,6 +901,13 @@ static void damos_test_commit_quota_goal(struct kunit *test) .current_value = 345, .last_psi_total = 567, }); + damos_test_commit_quota_goal_for(test, &dst, + &(struct damos_quota_goal){ + .metric = DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP, + .target_value = 12, + .current_value = 345, + .nid = 6, + }); } static void damos_test_commit_quota_goals_for(struct kunit *test, From 85368c3750ac6bd70a4b93b8dceab5b67fb5c749 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:06 -0700 Subject: [PATCH 478/539] mm/damon/tests/core-kunit: add probe_hits_wsum damos filter commit test DAMOS filter commit kunit test lacks test cases for probe_hits_wsum filter type. Add test cases for probe_hits_wsum type DAMOS filter commit. Link: https://lore.kernel.org/20260917142210.90829-9-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- mm/damon/tests/core-kunit.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 1f19fefdd98c..5ff0436c5844 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1166,6 +1166,10 @@ static void damos_test_commit_filter_for(struct kunit *test, KUNIT_EXPECT_EQ(test, dst->sz_range.min, src->sz_range.min); KUNIT_EXPECT_EQ(test, dst->sz_range.max, src->sz_range.max); break; + case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: + KUNIT_EXPECT_EQ(test, dst->range_min, src->range_min); + KUNIT_EXPECT_EQ(test, dst->range_max, src->range_max); + break; default: break; } @@ -1239,6 +1243,22 @@ static void damos_test_commit_filter(struct kunit *test) .allow = true, .target_idx = 6, }, false); + damos_test_commit_filter_for(test, &dst, + &(struct damos_filter){ + .type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, + .matching = false, + .allow = true, + .range_min = 12, + .range_max = 34, + }, false); + damos_test_commit_filter_for(test, &dst, + &(struct damos_filter){ + .type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, + .matching = false, + .allow = true, + .range_min = 34, + .range_max = 12, + }, true); } static void damos_test_help_initailize_scheme(struct damos *scheme) From 844eaab7a11bc6491193e683f78c2b69f9705cec Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:07 -0700 Subject: [PATCH 479/539] selftests/damon/sysfs_memcg_path_leak: fail only for real DAMON leak The selftest can fail for any leak if it happens while the test is running. Remove the false positive test failures by further checking if the expected leaking function is called out on the report. Link: https://lore.kernel.org/20260917142210.90829-10-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Randy Dunlap Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- tools/testing/selftests/damon/sysfs_memcg_path_leak.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/testing/selftests/damon/sysfs_memcg_path_leak.sh b/tools/testing/selftests/damon/sysfs_memcg_path_leak.sh index 33a7ff43ed6c..34c37129c49f 100755 --- a/tools/testing/selftests/damon/sysfs_memcg_path_leak.sh +++ b/tools/testing/selftests/damon/sysfs_memcg_path_leak.sh @@ -41,5 +41,12 @@ if [ "$kmemleak_report" = "" ] then exit 0 fi +if ! echo "$kmemleak_report" | grep "memcg_path_store" --quiet +then + echo "[WARN] memleak found; apparently not from DAMON, though" + echo "$kmemleak_report" + exit 0 +fi + echo "$kmemleak_report" exit 1 From e1012e4f37607fe451f0208fa5c2727923a8f966 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Thu, 17 Sep 2026 07:22:08 -0700 Subject: [PATCH 480/539] Docs/mm/damon/design: clarify bp is basis point DAMON design document uses "bp" for "basis point" in multiple places. Because it is not clearly mentioned, it is difficult to understand what "bp" stands for. Add the clarification. Link: https://lore.kernel.org/20260917142210.90829-11-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reported-by: Randy Dunlap Closes: https://lore.kernel.org/107dc6ba-697e-4b25-ba3e-8ce2499cac9a@infradead.org Acked-by: Randy Dunlap Acked-by: Zenghui Yu (Huawei) Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Gow Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka --- Documentation/mm/damon/design.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 0a86792f90a1..e82390e77a70 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -445,7 +445,7 @@ users to set the aimed amount of access events to observe via DAMON within given time interval. The target can be specified by the user as a ratio of DAMON-observed access events to the theoretical maximum amount of the events (``access_bp``) that measured within a given number of aggregations -(``aggrs``). +(``aggrs``). The ratio is in basis point (bp or 1/10,000). The DAMON-observed access events are calculated in byte granularity based on DAMON :ref:`region assumption `. For @@ -717,7 +717,8 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to in microseconds that measured from last quota reset to next quota reset. DAMOS does the measurement on its own, so only ``target_value`` need to be set by users at the initial time. In other words, DAMOS does self-feedback. -- ``node_mem_used_bp``: Specific NUMA node's used memory ratio in bp (1/10,000). +- ``node_mem_used_bp``: Specific NUMA node's used memory ratio in basis point + (bp or 1/10,000). - ``node_mem_free_bp``: Specific NUMA node's free memory ratio in bp (1/10,000). - ``node_memcg_used_bp``: Specific cgroup's node used memory ratio for a specific NUMA node, in bp (1/10,000). From 947ab1dd1c2d91c0d390e0008b6bfdc9a9d977b6 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 17 Sep 2026 06:47:35 -0700 Subject: [PATCH 481/539] Documentation: kmemleak: describe the metadata pool, not the early log Patch series "kmemleak: fix stale documentation and raise the verbose default". The first two patches fix statements in Documentation/dev-tools/kmemleak.rst that do not match mm/kmemleak.c. The third requires one more consecutive unreferenced scan before a CONFIG_DEBUG_KMEMLEAK_VERBOSE kernel reports a leak on the console, to avoid the last false positives I am seeing when running CONFIG_DEBUG_KMEMLEAK_VERBOSE on a daily basis. This patch (of 3): commit c5665868183f ("mm: kmemleak: use the memory pool for early allocations") removed the early log buffer in favour of a static pool of kmemleak_object structures, but the documentation still describes the old mechanism. Fix the documentation by describing what the pool actually is, matching the Kconfig help text. Link: https://lore.kernel.org/20260917142210.90829-1-sj@kernel.org Link: https://lore.kernel.org/20260917-b4-kmemleak-doc-v1-1-84fde6d1f749@debian.org Fixes: c5665868183f ("mm: kmemleak: use the memory pool for early allocations") Signed-off-by: Breno Leitao Signed-off-by: Andrew Morton Reviewed-by: Catalin Marinas Cc: Jonathan Corbet Cc: Randy Dunlap --- Documentation/dev-tools/kmemleak.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst index d1b690b17169..8dad7647742d 100644 --- a/Documentation/dev-tools/kmemleak.rst +++ b/Documentation/dev-tools/kmemleak.rst @@ -66,9 +66,14 @@ Memory scanning parameters can be modified at run-time by writing to the Kmemleak can also be disabled at boot-time by passing ``kmemleak=off`` on the kernel command line. -Memory may be allocated or freed before kmemleak is initialised and -these actions are stored in an early log buffer. The size of this buffer -is configured via the CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE option. +Memory may be allocated or freed before kmemleak is initialised, so a +static pool of metadata objects is used to track those allocations. Once +kmemleak is fully initialised the pool becomes an emergency reserve, used +whenever a metadata object cannot be allocated from the slab. The number +of objects in the pool is configured via the +CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE option. Exhausting it at run time +prints "Cannot allocate a kmemleak_object structure" and disables +kmemleak. If CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF are enabled, the kmemleak is disabled by default. Passing ``kmemleak=on`` on the kernel command From b523fc8e6bfae8e8bb05c39e259c4ce4a084e439 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 17 Sep 2026 06:47:36 -0700 Subject: [PATCH 482/539] Documentation: kmemleak: fix stale statements about scanning Three statements in the "false positives/negatives" and "Limitations" sections have never matched the code: - task stack scanning is on by default (kmemleak_stack_scan = 1), as the parameter list earlier in the same document already states; - MSECS_MIN_AGE has been 5000, not 1000, since the initial commit; - scanning is done by a periodic kthread. Reading the debugfs file only lists what the last scan found; kmemleak_open() calls seq_open() and never scans. Link: https://lore.kernel.org/20260917-b4-kmemleak-doc-v1-2-84fde6d1f749@debian.org Fixes: 04f70336c80c ("kmemleak: Add documentation on the memory leak detector") Fixes: e0a2a1601bec ("kmemleak: Enable task stacks scanning by default") Signed-off-by: Breno Leitao Signed-off-by: Andrew Morton Reviewed-by: Catalin Marinas Cc: Jonathan Corbet Cc: Randy Dunlap --- Documentation/dev-tools/kmemleak.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst index 8dad7647742d..b5fe7e671d0f 100644 --- a/Documentation/dev-tools/kmemleak.rst +++ b/Documentation/dev-tools/kmemleak.rst @@ -190,7 +190,8 @@ reported by kmemleak because values found during the memory scanning point to such objects. To reduce the number of false negatives, kmemleak provides the kmemleak_ignore, kmemleak_scan_area, kmemleak_no_scan and kmemleak_erase functions (see above). The task stacks also increase the -amount of false negatives and their scanning is not enabled by default. +amount of false negatives and their scanning is enabled by default; it +can be turned off with ``stack=off``. The false positives are objects wrongly reported as being memory leaks (orphan). For objects known not to be leaks, kmemleak provides the @@ -200,7 +201,7 @@ longer be scanned. Some of the reported leaks are only transient, especially on SMP systems, because of pointers temporarily stored in CPU registers or -stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 1000) representing +stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 5000) representing the minimum age of an object to be reported as a memory leak. The ``min_unref_scans`` module parameter requires an object to be seen @@ -217,8 +218,10 @@ Limitations and Drawbacks ------------------------- The main drawback is the reduced performance of memory allocation and -freeing. To avoid other penalties, the memory scanning is only performed -when the /sys/kernel/debug/kmemleak file is read. Anyway, this tool is +freeing. To avoid other penalties, the memory scanning is performed by a +periodic thread rather than on every allocation. Reading the +/sys/kernel/debug/kmemleak file only lists the objects found by the last +scan; writing ``scan`` to it triggers a new one. Anyway, this tool is intended for debugging purposes where the performance might not be the most important requirement. From 9596a3ef02fc3f5771f573e142138ef00a867603 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 17 Sep 2026 06:47:37 -0700 Subject: [PATCH 483/539] mm: kmemleak: raise min_unref_scans to 3 for verbose auto-scan CONFIG_DEBUG_KMEMLEAK_VERBOSE sends every report to the console, so a transient false positive there is broadcast to whatever collects the kernel log rather than sitting in the debugfs file until someone looks. That asymmetry justifies being more conservative than the general case. Require one more consecutive unreferenced scan before reporting. The only cost is that a genuine leak is reported one scan interval later (600s by default); the value stays writable at run time through the module parameter. Kernels without CONFIG_DEBUG_KMEMLEAK_VERBOSE keep reporting on the first unreferenced scan. I've been running constant upstream kernel with CONFIG_DEBUG_KMEMLEAK_VERBOSE set, and I am still seeing some rare false positive, that goes away with min_unref_scans=3, so, making it the default based on my heuristic. Link: https://lore.kernel.org/20260917-b4-kmemleak-doc-v1-3-84fde6d1f749@debian.org Signed-off-by: Breno Leitao Signed-off-by: Andrew Morton Reviewed-by: Catalin Marinas Cc: Jonathan Corbet Cc: Randy Dunlap --- Documentation/dev-tools/kmemleak.rst | 2 +- mm/kmemleak.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst index b5fe7e671d0f..c0d329372342 100644 --- a/Documentation/dev-tools/kmemleak.rst +++ b/Documentation/dev-tools/kmemleak.rst @@ -206,7 +206,7 @@ the minimum age of an object to be reported as a memory leak. The ``min_unref_scans`` module parameter requires an object to be seen unreferenced in that many consecutive scans before it is reported. It -defaults to 2 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled, where the +defaults to 3 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled, where the periodic scan thread confirms a leak on its own, and to 1 otherwise. A value of 1 preserves the historical behaviour; higher values filter the transient false positives described above, at the cost of delaying genuine diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 8fa409a4f9fb..5d0daea93c47 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -238,7 +238,7 @@ static struct task_struct *scan_thread; static unsigned long jiffies_min_age; /* consecutive scans an object must stay unreferenced before reporting */ static unsigned int min_unref_scans = - IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 2 : 1; + IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 3 : 1; module_param(min_unref_scans, uint, 0644); static unsigned long jiffies_last_scan; /* delay between automatic memory scannings */ From 9490353d923fe72a4b57b796025acacae709610b Mon Sep 17 00:00:00 2001 From: Lisa Wang Date: Thu, 17 Sep 2026 20:43:47 +0000 Subject: [PATCH 484/539] mm: memory_failure: clarify the MF_DELAYED definition Patch series "mm: Fix MF_DELAYED handling on memory failure", v6. This series addresses an issue in the memory failure handling path where MF_DELAYED is incorrectly treated as an error. This issue was discovered while testing memory failure handling for guest_memfd. The proposed solution involves - 1. Clarifying the definition of MF_DELAYED to mean that memory failure handling is only partially completed, and that the metadata for the memory that failed (as in struct page/folio) is still referenced. 2. Updating shmems handling to align with the clarified definition. 3. Updating how the result of .error_remove_folio() is interpreted. This patch (of 5): This patch clarifies the definition of MF_DELAYED to represent cases where a folio's removal is initiated but not immediately completed (e.g., due to remaining metadata references). Link: https://lore.kernel.org/20260917-memory-failure-mf-delayed-fix-v6-0-4b00856b5364@google.com Link: https://lore.kernel.org/20260917-memory-failure-mf-delayed-fix-v6-1-4b00856b5364@google.com Signed-off-by: Lisa Wang Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Miaohe Lin Reviewed-by: Ackerley Tng Cc: Andi Kleen Cc: Baolin Wang Cc: Dave Hansen Cc: David Rientjes Cc: Fuad Tabba Cc: Hidehiro Kawai Cc: Hugh Dickins Cc: Isaku Yamahata Cc: Jiaqi Yan Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michael Roth Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Paolo Bonzini Cc: Rik van Riel Cc: Sean Christopherson Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Annapurve Cc: Vlastimil Babka Cc: Xiaoyao Li Cc: Yu Zhang --- mm/memory-failure.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index a8b03e2920ba..057cb9537db2 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -847,24 +847,25 @@ static int kill_accessing_process(struct task_struct *p, unsigned long pfn, } /* - * MF_IGNORED - The m-f() handler marks the page as PG_hwpoisoned'ed. + * MF_IGNORED - The m-f() handler marks the page as PG_hwpoison'ed. * But it could not do more to isolate the page from being accessed again, * nor does it kill the process. This is extremely rare and one of the * potential causes is that the page state has been changed due to * underlying race condition. This is the most severe outcomes. * - * MF_FAILED - The m-f() handler marks the page as PG_hwpoisoned'ed. + * MF_FAILED - The m-f() handler marks the page as PG_hwpoison'ed. * It should have killed the process, but it can't isolate the page, * due to conditions such as extra pin, unmap failure, etc. Accessing * the page again may trigger another MCE and the process will be killed * by the m-f() handler immediately. * - * MF_DELAYED - The m-f() handler marks the page as PG_hwpoisoned'ed. - * The page is unmapped, and is removed from the LRU or file mapping. - * An attempt to access the page again will trigger page fault and the - * PF handler will kill the process. + * MF_DELAYED - The m-f() handler marks the page as PG_hwpoison'ed. + * It means the page was unmapped and partially isolated (e.g. removed from + * file mapping or the LRU) but full cleanup is deferred (e.g. the metadata + * for the memory, as in struct page/folio, is still referenced). Any + * further access to the page will result in the process being killed. * - * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoisoned'ed. + * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoison'ed. * The page has been completely isolated, that is, unmapped, taken out of * the buddy system, or hole-punched out of the file mapping. */ From 9d442b019b0f15de3ef6d9bb1257c13f29b49954 Mon Sep 17 00:00:00 2001 From: Lisa Wang Date: Thu, 17 Sep 2026 20:43:48 +0000 Subject: [PATCH 485/539] mm: memory_failure: Allow truncate_error_folio to return MF_DELAYED The .error_remove_folio a_ops is used by different filesystems to handle folio truncation upon discovery of a memory failure in the memory associated with the given folio. Currently, MF_DELAYED is treated as an error, causing "Failed to punch page" to be written to the console. MF_DELAYED is then relayed to the caller of truncate_error_folio() as MF_FAILED. This further causes memory_failure() to return -EBUSY, which then always causes a SIGBUS. This is also implies that regardless of whether the thread's memory corruption kill policy is PR_MCE_KILL_EARLY or PR_MCE_KILL_LATE, a memory failure with MF_DELAYED will always cause a SIGBUS. Update truncate_error_folio() to return MF_DELAYED to the caller if the .error_remove_folio() callback reports MF_DELAYED. Link: https://lore.kernel.org/20260917-memory-failure-mf-delayed-fix-v6-2-4b00856b5364@google.com Fixes: 6a46079cf57a ("HWPOISON: The high level memory error handler in the VM v7") Fixes: a7800aa80ea4 ("KVM: Add KVM_CREATE_GUEST_MEMFD ioctl() for guest-specific backing memory") Signed-off-by: Lisa Wang Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Miaohe Lin Reviewed-by: Ackerley Tng Cc: Andi Kleen Cc: Baolin Wang Cc: Dave Hansen Cc: David Rientjes Cc: Fuad Tabba Cc: Hidehiro Kawai Cc: Hugh Dickins Cc: Isaku Yamahata Cc: Jiaqi Yan Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michael Roth Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Paolo Bonzini Cc: Rik van Riel Cc: Sean Christopherson Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Annapurve Cc: Vlastimil Babka Cc: Xiaoyao Li Cc: Yu Zhang --- mm/memory-failure.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 057cb9537db2..83ab0fce22a6 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -939,10 +939,12 @@ static int truncate_error_folio(struct folio *folio, unsigned long pfn, if (mapping->a_ops->error_remove_folio) { int err = mapping->a_ops->error_remove_folio(mapping, folio); - if (err != 0) + if (err == MF_DELAYED) + ret = err; + else if (err != 0) pr_info("%#lx: Failed to punch page: %d\n", pfn, err); else if (!filemap_release_folio(folio, GFP_NOIO)) - pr_info("%#lx: failed to release buffers\n", pfn); + pr_info("%#lx: Failed to release buffers\n", pfn); else ret = MF_RECOVERED; } else { From b9181977de32c393bd15a39170e09dc0ea523e02 Mon Sep 17 00:00:00 2001 From: Lisa Wang Date: Thu, 17 Sep 2026 20:43:49 +0000 Subject: [PATCH 486/539] mm: shmem: Update shmem handler to the MF_DELAYED definition To align with the definition of MF_DELAYED, update shmem_error_remove_folio() to return MF_DELAYED. shmem handles memory failures but defers the actual file truncation. The function's return value should therefore be MF_DELAYED to accurately reflect the state. Currently, this logical error does not cause a bug, because: - For shmem folios, folio->private is not set. - As a result, filemap_release_folio() is a no-op and returns true. - This, in turn, causes truncate_error_folio() to incorrectly return MF_RECOVERED. - The caller then treats MF_RECOVERED as a success condition, masking the issue. The previous patch relays MF_DELAYED to the caller of truncate_error_folio() before any logging, so returning MF_DELAYED from shmem_error_remove_folio() will retain the original behavior of not adding any logs. The return value of truncate_error_folio() is consumed in action_result(), which treats MF_DELAYED the same way as MF_RECOVERED, hence action_result() also returns the same thing after this change. Link: https://lore.kernel.org/20260917-memory-failure-mf-delayed-fix-v6-3-4b00856b5364@google.com Signed-off-by: Lisa Wang Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Miaohe Lin Reviewed-by: Ackerley Tng Cc: Andi Kleen Cc: Baolin Wang Cc: Dave Hansen Cc: David Rientjes Cc: Fuad Tabba Cc: Hidehiro Kawai Cc: Hugh Dickins Cc: Isaku Yamahata Cc: Jiaqi Yan Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michael Roth Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Paolo Bonzini Cc: Rik van Riel Cc: Sean Christopherson Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Annapurve Cc: Vlastimil Babka Cc: Xiaoyao Li Cc: Yu Zhang --- mm/shmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/shmem.c b/mm/shmem.c index 05bc7c3aa525..f33dbf5af2cb 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -5363,7 +5363,7 @@ static void __init shmem_destroy_inodecache(void) static int shmem_error_remove_folio(struct address_space *mapping, struct folio *folio) { - return 0; + return MF_DELAYED; } static const struct address_space_operations shmem_aops = { From fd57828da9c280877825f0075066d81583520a3e Mon Sep 17 00:00:00 2001 From: Lisa Wang Date: Thu, 17 Sep 2026 20:43:50 +0000 Subject: [PATCH 487/539] mm: memory_failure: Generalize extra_pins handling to all MF_DELAYED cases Generalize extra_pins handling to all MF_DELAYED cases not only shmem_mapping. If MF_DELAYED is returned, the filemap continues to hold refcounts on the folio. Hence, take that into account when checking for extra refcounts. As clarified in an earlier patch, a return value of MF_DELAYED implies that the page still has elevated refcounts. Hence, set extra_pins to true if the return value is MF_DELAYED. This is aligned with the implementation in me_swapcache_dirty(), where, if a folio is still in the swap cache, ret is set to MF_DELAYED and extra_pins is set to true. Link: https://lore.kernel.org/20260917-memory-failure-mf-delayed-fix-v6-4-4b00856b5364@google.com Signed-off-by: Lisa Wang Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Acked-by: Miaohe Lin Reviewed-by: Ackerley Tng Cc: Andi Kleen Cc: Baolin Wang Cc: Dave Hansen Cc: David Rientjes Cc: Fuad Tabba Cc: Hidehiro Kawai Cc: Hugh Dickins Cc: Isaku Yamahata Cc: Jiaqi Yan Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michael Roth Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Paolo Bonzini Cc: Rik van Riel Cc: Sean Christopherson Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Annapurve Cc: Vlastimil Babka Cc: Xiaoyao Li Cc: Yu Zhang --- mm/memory-failure.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 83ab0fce22a6..d237f556b3a0 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1039,18 +1039,14 @@ static int me_pagecache_clean(struct page_state *ps, struct page *p) goto out; } - /* - * The shmem page is kept in page cache instead of truncating - * so is expected to have an extra refcount after error-handling. - */ - extra_pins = shmem_mapping(mapping); - /* * Truncation is a bit tricky. Enable it per file system for now. * * Open: to take i_rwsem or not for this? Right now we don't. */ ret = truncate_error_folio(folio, page_to_pfn(p), mapping); + + extra_pins = ret == MF_DELAYED; if (has_extra_refcount(ps, p, extra_pins)) ret = MF_FAILED; From 69567ac40ee3b29d95cccc721bb08aa211cc3292 Mon Sep 17 00:00:00 2001 From: Lisa Wang Date: Thu, 17 Sep 2026 20:43:51 +0000 Subject: [PATCH 488/539] mm: selftests: Add shmem into memory failure test Add a shmem memory failure selftest to test the shmem memory failure is correct after modifying shmem return value. Specifically, test the expected behavior under various scenarios combining page dirtiness (dirty vs clean) and failure types (hard vs soft): + Dirty + Hard: Trigger a SIGBUS on injection, and trigger another SIGBUS when reading the page again. + Dirty + Soft: No SIGBUS is triggered, and the original value can be read successfully. + Clean + Hard: No SIGBUS is triggered on injection, but trigger a SIGBUS when trying to read the page again. + Clean + Soft: No SIGBUS is triggered, and the page can be read successfully. [akpm@linux-foundation.org: fix comment typo, per Lisa] Link: https://lore.kernel.org/20260917-memory-failure-mf-delayed-fix-v6-5-4b00856b5364@google.com Signed-off-by: Lisa Wang Signed-off-by: Andrew Morton Acked-by: Miaohe Lin Cc: Ackerley Tng Cc: Andi Kleen Cc: Baolin Wang Cc: Dave Hansen Cc: David Hildenbrand (Arm) Cc: David Rientjes Cc: Fuad Tabba Cc: Hidehiro Kawai Cc: Hugh Dickins Cc: Isaku Yamahata Cc: Jiaqi Yan Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michael Roth Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Paolo Bonzini Cc: Rik van Riel Cc: Sean Christopherson Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Annapurve Cc: Vlastimil Babka Cc: Xiaoyao Li Cc: Yu Zhang --- tools/testing/selftests/mm/memory-failure.c | 114 +++++++++++++++++++- 1 file changed, 111 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/mm/memory-failure.c b/tools/testing/selftests/mm/memory-failure.c index f3cb578b1609..135a2e8069ba 100644 --- a/tools/testing/selftests/mm/memory-failure.c +++ b/tools/testing/selftests/mm/memory-failure.c @@ -29,9 +29,14 @@ enum result_type { MADV_HARD_ANON, MADV_HARD_CLEAN_PAGECACHE, MADV_HARD_DIRTY_PAGECACHE, + MADV_HARD_CLEAN_SHMEM, + MADV_HARD_DIRTY_SHMEM, MADV_SOFT_ANON, MADV_SOFT_CLEAN_PAGECACHE, MADV_SOFT_DIRTY_PAGECACHE, + MADV_SOFT_CLEAN_SHMEM, + MADV_SOFT_DIRTY_SHMEM, + READ_ERROR, }; static jmp_buf signal_jmp_buf; @@ -157,17 +162,22 @@ static void check(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure case MADV_HARD_CLEAN_PAGECACHE: case MADV_SOFT_CLEAN_PAGECACHE: case MADV_SOFT_DIRTY_PAGECACHE: - /* It is not expected to receive a SIGBUS signal. */ - ASSERT_EQ(setjmp, 0); - + case MADV_SOFT_DIRTY_SHMEM: /* The page content should remain unchanged. */ ASSERT_TRUE(check_memory(vaddr, self->page_size)); + /* FALLTHROUGH */ + case MADV_HARD_CLEAN_SHMEM: + case MADV_SOFT_CLEAN_SHMEM: + /* It is not expected to receive a SIGBUS signal. */ + ASSERT_EQ(setjmp, 0); /* The backing pfn of addr should have changed. */ ASSERT_NE(pagemap_get_pfn(self->pagemap_fd, vaddr), self->pfn); break; case MADV_HARD_ANON: case MADV_HARD_DIRTY_PAGECACHE: + case MADV_HARD_DIRTY_SHMEM: + case READ_ERROR: /* The SIGBUS signal should have been received. */ ASSERT_EQ(setjmp, 1); @@ -263,6 +273,20 @@ static int prepare_file(const char *fname, unsigned long size) return fd; } +static int prepare_shmem(const char *fname, unsigned long size) +{ + int fd; + + fd = memfd_create(fname, 0); + if (fd < 0) + return -1; + if (ftruncate(fd, size) < 0) { + close(fd); + return -1; + } + return fd; +} + /* Borrowed from mm/gup_longterm.c. */ static int get_fs_type(int fd) { @@ -365,4 +389,88 @@ TEST_F(memory_failure, dirty_pagecache) ASSERT_EQ(close(fd), 0); } +TEST_F(memory_failure, dirty_shmem) +{ + int fd; + char *addr; + int ret; + + fd = prepare_shmem("shmem-file", self->page_size); + if (fd < 0) + SKIP(return, "failed to open test shmem-file.\n"); + + addr = mmap(0, self->page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + close(fd); + SKIP(return, "mmap failed, not enough memory.\n"); + } + memset(addr, 0xce, self->page_size); + + prepare(_metadata, self, addr); + + ret = sigsetjmp(signal_jmp_buf, 1); + if (!ret && !self->injection_attempted) { + self->injection_attempted = true; + ASSERT_EQ(variant->inject(self, addr), 0); + } + + if (variant->type == MADV_HARD) { + check(_metadata, self, addr, MADV_HARD_DIRTY_SHMEM, ret); + ret = sigsetjmp(signal_jmp_buf, 1); + if (ret == 0) + FORCE_READ(*addr); + check(_metadata, self, addr, READ_ERROR, ret); + } else { + check(_metadata, self, addr, MADV_SOFT_DIRTY_SHMEM, ret); + } + + ASSERT_EQ(munmap(addr, self->page_size), 0); + + ASSERT_EQ(close(fd), 0); +} + +TEST_F(memory_failure, clean_shmem) +{ + int fd; + char *addr; + int ret; + + fd = prepare_shmem("shmem-file", self->page_size); + if (fd < 0) + SKIP(return, "failed to open test shmem-file.\n"); + + addr = mmap(0, self->page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + close(fd); + SKIP(return, "mmap failed, not enough memory.\n"); + } + FORCE_READ(*addr); + + prepare(_metadata, self, addr); + + ret = sigsetjmp(signal_jmp_buf, 1); + if (!ret && !self->injection_attempted) { + self->injection_attempted = true; + ASSERT_EQ(variant->inject(self, addr), 0); + } + + if (variant->type == MADV_HARD) { + check(_metadata, self, addr, MADV_HARD_CLEAN_SHMEM, ret); + ret = sigsetjmp(signal_jmp_buf, 1); + if (ret == 0) + FORCE_READ(*addr); + check(_metadata, self, addr, READ_ERROR, ret); + } else { + /* Test the address accessability without check_memory(). */ + FORCE_READ(*addr); + check(_metadata, self, addr, MADV_SOFT_CLEAN_SHMEM, ret); + } + + ASSERT_EQ(munmap(addr, self->page_size), 0); + + ASSERT_EQ(close(fd), 0); +} + TEST_HARNESS_MAIN From fe0ba1909c59660f078560f5130e7af7c6004f12 Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Fri, 18 Sep 2026 10:56:10 +0800 Subject: [PATCH 489/539] mm/hugetlb_cgroup: move per-node usage on cross node migration Patch series "mm/hugetlb_cgroup: move the per-node usage along with the folio", v2. The per-node usage reported by hugetlb..numa_stat is accounted against folio_nid() in __hugetlb_cgroup_commit_charge() and __hugetlb_cgroup_uncharge_folio(), so it is only correct while a folio stays charged on the same node and in the same hugetlb_cgroup. Two paths move a folio which stays charged, and neither moves the usage with it. hugetlb_cgroup_migrate() only moves the hugetlb_cgroup pointers of a folio migrated to another node, and hugetlb_cgroup_move_parent() only moves the page_counter charges and the hugetlb_cgroup pointer of the folios of a dying cgroup. In both cases the node (or cgroup) which was charged keeps a usage which never goes away, while the node (or cgroup) which ends up uncharging the folio underflows as soon as the folio is freed. Patch 1/2 moves the usage along with the folio on cross node migration, patch 2/2 does the same for the folios a dying cgroup reparents. This patch (of 2): hugetlb..numa_stat uses folio_nid() to account usage in __hugetlb_cgroup_commit_charge() and __hugetlb_cgroup_uncharge_folio(). hugetlb_cgroup_migrate() only moves hugetlb_cgroup pointers, leaving per-node usage behind on the source node during cross-node migration. When the migrated folio gets uncharged, we subtract usage from the destination node counter. This creates stale usage on the source node and unsigned long counter underflow on the destination node. The hugetlb..numa_stat interface exposes these incorrect per-node usage values to userspace. Add a hugetlb_cgroup_move_usage() helper which moves the usage from the old node to the new node, and call it from hugetlb_cgroup_migrate(). Link: https://lore.kernel.org/20260918-for-hugetlb-charge-v2-0-2b6d8c2bdc36@kylinos.cn Link: https://lore.kernel.org/20260918-for-hugetlb-charge-v2-1-2b6d8c2bdc36@kylinos.cn Fixes: f47761999052 ("hugetlb: add hugetlb.*.numa_stat file") Signed-off-by: Hongfu Li Signed-off-by: Andrew Morton Acked-by: Muchun Song Cc: Colin Ian King Cc: David Hildenbrand Cc: Kees Cook Cc: Mina Almasry Cc: Oscar Salvador Cc: Shakeel Butt Cc: --- mm/hugetlb_cgroup.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index ecb6e0b7819a..7cf7c18119b4 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -179,6 +179,34 @@ static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css) hugetlb_cgroup_free(hugetlb_cgroup_from_css(css)); } +static void hugetlb_cgroup_move_usage(struct hugetlb_cgroup *from, + struct hugetlb_cgroup *to, + struct folio *from_folio, + struct folio *to_folio) +{ + int idx = hstate_index(folio_hstate(from_folio)); + unsigned long nr_pages = folio_nr_pages(from_folio); + int from_nid = folio_nid(from_folio); + int to_nid = folio_nid(to_folio); + unsigned long usage; + + lockdep_assert_held(&hugetlb_lock); + + if (!from || !to) + return; + + if (from == to && from_nid == to_nid) + return; + + usage = from->nodeinfo[from_nid]->usage[idx]; + if (WARN_ON_ONCE(usage < nr_pages)) + return; + WRITE_ONCE(from->nodeinfo[from_nid]->usage[idx], usage - nr_pages); + + usage = to->nodeinfo[to_nid]->usage[idx]; + WRITE_ONCE(to->nodeinfo[to_nid]->usage[idx], usage + nr_pages); +} + /* * Should be called with hugetlb_lock held. * Since we are holding hugetlb_lock, pages cannot get moved from @@ -906,6 +934,9 @@ void hugetlb_cgroup_migrate(struct folio *old_folio, struct folio *new_folio) /* move the h_cg details to new cgroup */ set_hugetlb_cgroup(new_folio, h_cg); set_hugetlb_cgroup_rsvd(new_folio, h_cg_rsvd); + + hugetlb_cgroup_move_usage(h_cg, h_cg, old_folio, new_folio); + list_move(&new_folio->lru, &h->hugepage_activelist); spin_unlock_irq(&hugetlb_lock); } From 23fac055c21176f66ed453394b533bdfc00f7fba Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Fri, 18 Sep 2026 10:56:11 +0800 Subject: [PATCH 490/539] mm/hugetlb_cgroup: move per-node usage on cgroup reparenting hugetlb_cgroup_css_offline() hands the folios of a dying cgroup over to its parent with hugetlb_cgroup_move_parent(), which moves the page_counter charges and the hugetlb_cgroup pointer of the folio but not its per-node usage. The parent's hugetlb..numa_stat is short by that usage while they are charged, and underflows once they are freed, exposing incorrect per-node usage values to userspace. Move the per-node usage to the parent as well. The folios keep their node here, so only the cgroup which holds the usage changes. Link: https://lore.kernel.org/20260918-for-hugetlb-charge-v2-2-2b6d8c2bdc36@kylinos.cn Fixes: f47761999052 ("hugetlb: add hugetlb.*.numa_stat file") Signed-off-by: Hongfu Li Signed-off-by: Andrew Morton Acked-by: Muchun Song Cc: Colin Ian King Cc: David Hildenbrand Cc: Kees Cook Cc: Mina Almasry Cc: Oscar Salvador Cc: Shakeel Butt Cc: --- mm/hugetlb_cgroup.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index 7cf7c18119b4..3fb41311e4c7 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -241,6 +241,8 @@ static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg, /* Take the pages off the local counter */ page_counter_cancel(counter, nr_pages); + hugetlb_cgroup_move_usage(h_cg, parent, folio, folio); + set_hugetlb_cgroup(folio, parent); out: return; From 87ee17d24a8963b46826743e0638323d72c62384 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 18 Sep 2026 08:33:12 -0700 Subject: [PATCH 491/539] proc/task_mmu: remove unnecessary helpers Patch series "read proc/pid/smaps_rollup under per-vma lock", v5. proc/pid/smaps_rollup can be read using the combination of RCU and VMA read locks, similar to proc/pid/{maps|smaps|numa_maps}. RCU is required to safely traverse the VMA tree and VMA lock stabilizes the VMA being processed and the pagetable walk. Note that we have to keep the logic to drop mmap_lock on contention because even when using per-VMA locks we might have to fall back to holding the mmap_lock. The first 5 patches are cleanups making later change simpler. The main change is in patch 6. Patch 7 extends existing proc-maps-race tearing test to verify smaps_rollup content. This patch (of 7): When per-vma locks were behind a config option, a number of helper functions were needed to simplify the locking code. Now that these locks are universally available, we can do a little cleanup. Remove lock_vma_range(), unlock_vma_range(), query_vma_setup(), query_vma_teardown() helpers. No functional change intended. Link: https://lore.kernel.org/20260918153318.758387-1-surenb@google.com Link: https://lore.kernel.org/20260918153318.758387-2-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Liam R. Howlett (Oracle) Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: Usama Arif Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Matthew Wilcox (Oracle) Cc: "Paul E . McKenney" Cc: Pedro Falcato Cc: Vlastimil Babka --- fs/proc/task_mmu.c | 67 ++++++++++++---------------------------------- 1 file changed, 17 insertions(+), 50 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 78e00c6d4e2d..6253d6f738e4 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -160,25 +160,6 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } -static inline bool lock_vma_range(struct seq_file *m, - struct proc_maps_locking_ctx *lock_ctx) -{ - rcu_read_lock(); - reset_lock_ctx(lock_ctx); - - return true; -} - -static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) -{ - if (lock_ctx->mmap_locked) { - unlock_ctx_mm(lock_ctx); - } else { - unlock_ctx_vma(lock_ctx); - rcu_read_unlock(); - } -} - static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, loff_t last_pos) { @@ -286,13 +267,8 @@ static void *m_start(struct seq_file *m, loff_t *ppos) return NULL; } - if (!lock_vma_range(m, lock_ctx)) { - mmput(mm); - put_task_struct(priv->task); - priv->task = NULL; - return ERR_PTR(-EINTR); - } - + rcu_read_lock(); + reset_lock_ctx(lock_ctx); /* * Reset current position if last_addr was set before * and it's not a sentinel. @@ -325,7 +301,12 @@ static void m_stop(struct seq_file *m, void *v) return; release_task_mempolicy(priv); - unlock_vma_range(&priv->lock_ctx); + if (priv->lock_ctx.mmap_locked) { + unlock_ctx_mm(&priv->lock_ctx); + } else { + unlock_ctx_vma(&priv->lock_ctx); + rcu_read_unlock(); + } mmput(mm); put_task_struct(priv->task); priv->task = NULL; @@ -518,21 +499,6 @@ static int pid_maps_open(struct inode *inode, struct file *file) PROCMAP_QUERY_VMA_FLAGS \ ) -static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) -{ - reset_lock_ctx(lock_ctx); - - return 0; -} - -static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) -{ - if (lock_ctx->mmap_locked) - unlock_ctx_mm(lock_ctx); - else - unlock_ctx_vma(lock_ctx); -} - static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, unsigned long addr) { @@ -653,12 +619,7 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) if (!mm || !mmget_not_zero(mm)) return -ESRCH; - err = query_vma_setup(&lock_ctx); - if (err) { - mmput(mm); - return err; - } - + reset_lock_ctx(&lock_ctx); vma = query_matching_vma(&lock_ctx, karg.query_addr, karg.query_flags); if (IS_ERR(vma)) { err = PTR_ERR(vma); @@ -732,7 +693,10 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) vm_file = get_file(vma->vm_file); /* unlock vma or mmap_lock, and put mm_struct before copying data to user */ - query_vma_teardown(&lock_ctx); + if (lock_ctx.mmap_locked) + unlock_ctx_mm(&lock_ctx); + else + unlock_ctx_vma(&lock_ctx); mmput(mm); if (karg.build_id_size) { @@ -773,7 +737,10 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) return 0; out: - query_vma_teardown(&lock_ctx); + if (lock_ctx.mmap_locked) + unlock_ctx_mm(&lock_ctx); + else + unlock_ctx_vma(&lock_ctx); mmput(mm); out_file: if (vm_file) From e2dfa1f31b03b81e3d43dfcab32cc2445305ef32 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 18 Sep 2026 08:33:13 -0700 Subject: [PATCH 492/539] proc/task_mmu: remove unnecessary inlines in function definitions It was pointed out in the previous reviews of this code that many functions are specified as inline, which is unnecessary as the compiler can make that decision by itself. Cleanup these definitions. No change in the resulting binary file size with gcc v15.2.0. No functional change intended. [akpm@linux-foundation.org: fix CONFIG_NUMA=y && CONFIG_PROC_PAGE_MONITOR=n] Link: https://lore.kernel.org/20260918153318.758387-3-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Liam R. Howlett (Oracle) Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: Usama Arif Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Matthew Wilcox (Oracle) Cc: "Paul E . McKenney" Cc: Pedro Falcato Cc: Vlastimil Babka --- fs/proc/task_mmu.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 6253d6f738e4..ca8213d7e53e 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -130,7 +130,8 @@ static void release_task_mempolicy(struct proc_maps_private *priv) } #endif -static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +#ifdef CONFIG_PROC_PAGE_MONITOR +static int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) { int ret = mmap_read_lock_killable(lock_ctx->mm); @@ -139,8 +140,9 @@ static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) return ret; } +#endif -static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +static void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) { mmap_read_unlock(lock_ctx->mm); lock_ctx->mmap_locked = false; @@ -177,8 +179,8 @@ static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, return vma; } -static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, - loff_t pos) +static bool fallback_to_mmap_lock(struct proc_maps_private *priv, + loff_t pos) { struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; @@ -194,7 +196,8 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return true; } -static inline void drop_rcu(struct proc_maps_private *priv) +#if defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA) +static void drop_rcu(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) return; @@ -202,7 +205,7 @@ static inline void drop_rcu(struct proc_maps_private *priv) rcu_read_unlock(); } -static inline void reacquire_rcu(struct proc_maps_private *priv) +static void reacquire_rcu(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) return; @@ -211,6 +214,7 @@ static inline void reacquire_rcu(struct proc_maps_private *priv) /* Reinitialize the iterator. */ vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end); } +#endif static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) { @@ -1230,7 +1234,7 @@ static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, }; -static inline const struct mm_walk_ops * +static const struct mm_walk_ops * get_smaps_walk_ops(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) @@ -1238,7 +1242,7 @@ get_smaps_walk_ops(struct proc_maps_private *priv) return &smaps_walk_vma_lock_ops; } -static inline const struct mm_walk_ops * +static const struct mm_walk_ops * get_smaps_shmem_walk_ops(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) @@ -1572,7 +1576,7 @@ struct clear_refs_private { enum clear_refs_types type; }; -static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) +static bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { struct folio *folio; @@ -1588,8 +1592,8 @@ static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, return folio_maybe_dma_pinned(folio); } -static inline void clear_soft_dirty(struct vm_area_struct *vma, - unsigned long addr, pte_t *pte) +static void clear_soft_dirty(struct vm_area_struct *vma, unsigned long addr, + pte_t *pte) { if (!pgtable_supports_soft_dirty()) return; @@ -1620,7 +1624,7 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma, } #if defined(CONFIG_TRANSPARENT_HUGEPAGE) -static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, +static void clear_soft_dirty_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { pmd_t old, pmd = *pmdp; @@ -1646,7 +1650,7 @@ static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, } } #else -static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, +static void clear_soft_dirty_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { } @@ -1848,7 +1852,7 @@ struct pagemapread { #define PM_END_OF_BUFFER 1 -static inline pagemap_entry_t make_pme(u64 frame, u64 flags) +static pagemap_entry_t make_pme(u64 frame, u64 flags) { return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags }; } @@ -3390,7 +3394,7 @@ static const struct mm_walk_ops show_numa_vma_lock_ops = { .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, }; -static inline const struct mm_walk_ops * +static const struct mm_walk_ops * get_show_numa_ops(struct proc_maps_private *priv) { if (priv->lock_ctx.mmap_locked) From 66757cdd0baa87ab0f9c3d5ebd702a6760048c96 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 18 Sep 2026 08:33:14 -0700 Subject: [PATCH 493/539] proc/task_mmu: clarify shmem mapping walk conditions in smap_gather_stats() smap_gather_stats() optimizes stats gathering by skipping the walk for shmem mappings in certain conditions. Update the comment to clarify these conditions and use vma_is_cow_mapping() for CoW identification instead of open-coding it. No functional change intended. Link: https://lore.kernel.org/20260918153318.758387-4-surenb@google.com Suggested-by: David Hildenbrand (Arm) Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett (Oracle) Cc: Matthew Wilcox (Oracle) Cc: "Paul E . McKenney" Cc: Pedro Falcato Cc: Usama Arif Cc: Vlastimil Babka --- fs/proc/task_mmu.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index ca8213d7e53e..d98bbd7cd38f 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1274,23 +1274,18 @@ static void smap_gather_stats(struct proc_maps_private *priv, if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { /* - * For shared or readonly shmem mappings we know that all - * swapped out pages belong to the shmem object, and we can - * obtain the swap value much more efficiently. For private - * writable mappings, we might have COW pages that are - * not affected by the parent swapped out pages of the shmem - * object, so we have to distinguish them during the page walk. - * Unless we know that the shmem object (or the part mapped by - * our VMA) has no swapped out pages at all. + * CoW mappings might map anon folios that do not belong to + * shmem. Perform a less efficient page table walk in this + * situation, unless we know that the shmem object (or the + * part mapped by our VMA) has no swapped out pages at all. */ - unsigned long shmem_swapped = shmem_swap_usage(vma); + const unsigned long shmem_swapped = shmem_swap_usage(vma); + const bool is_cow = vma_is_cow_mapping(vma); - if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) || - !(vma->vm_flags & VM_WRITE))) { - mss->swap += shmem_swapped; - } else { + if (start || (shmem_swapped && is_cow)) ops = get_smaps_shmem_walk_ops(priv); - } + else + mss->swap += shmem_swapped; } if (!start) From a2432e2cfea7b2230db0d9d42b40397c804e68a8 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 18 Sep 2026 08:33:15 -0700 Subject: [PATCH 494/539] proc/task_mmu: remove special-casing of smap_gather_stats() start parameter smap_gather_stats() interprets its start parameter to mean vma->vm_start when it's set to 0. Eliminate this special interpretation and provide two separate functions for a partial and complete VMA walk. Since smap_gather_stats() operates within a single VMA, we can replace walk_page_vma()/walk_page_range() calls with walk_page_range_vma() which is simpler and also can be called while holding per-VMA lock. No functional change intended. Link: https://lore.kernel.org/20260918153318.758387-5-surenb@google.com Suggested-by: Lorenzo Stoakes Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett (Oracle) Cc: Matthew Wilcox (Oracle) Cc: "Paul E . McKenney" Cc: Pedro Falcato Cc: Usama Arif Cc: Vlastimil Babka --- fs/proc/task_mmu.c | 52 +++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index d98bbd7cd38f..06fa97241e9b 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1250,20 +1250,26 @@ get_smaps_shmem_walk_ops(struct proc_maps_private *priv) return &smaps_shmem_walk_vma_lock_ops; } -/* - * Gather mem stats from @vma with the indicated beginning - * address @start, and keep them in @mss. +/** + * smap_gather_stats_range() - Gather mem stats from a portion of the @vma. + * @priv: proc maps private state. + * @vma: The VMA to gather stats for. + * @mss: The accumulated stats. + * @start: The address from which to start. * - * Use vm_start of @vma as the beginning address if @start is 0. + * This gathers stats for the portion of the VMA starting at the @start + * address. */ -static void smap_gather_stats(struct proc_maps_private *priv, - struct vm_area_struct *vma, - struct mem_size_stats *mss, unsigned long start) +static void smap_gather_stats_range(struct proc_maps_private *priv, + struct vm_area_struct *vma, + struct mem_size_stats *mss, + unsigned long start) { const struct mm_walk_ops *ops = get_smaps_walk_ops(priv); + const bool is_partial = start > vma->vm_start; /* Invalid start */ - if (start >= vma->vm_end) + if (start < vma->vm_start || start >= vma->vm_end) return; if (vma == get_gate_vma(priv->lock_ctx.mm)) @@ -1282,20 +1288,31 @@ static void smap_gather_stats(struct proc_maps_private *priv, const unsigned long shmem_swapped = shmem_swap_usage(vma); const bool is_cow = vma_is_cow_mapping(vma); - if (start || (shmem_swapped && is_cow)) + if (is_partial || (shmem_swapped && is_cow)) ops = get_smaps_shmem_walk_ops(priv); else mss->swap += shmem_swapped; } - if (!start) - walk_page_vma(vma, ops, mss); - else - walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss); + walk_page_range_vma(vma, start, vma->vm_end, ops, mss); reacquire_rcu(priv); } +/** + * smap_gather_stats() - Gather mem stats from the entire @vma. + * @priv: proc maps private state. + * @vma: The VMA to gather stats for. + * @mss: The accumulated stats. + * + * This gathers stats for the whole of the VMA. + */ +static void smap_gather_stats(struct proc_maps_private *priv, + struct vm_area_struct *vma, struct mem_size_stats *mss) +{ + smap_gather_stats_range(priv, vma, mss, vma->vm_start); +} + #define SEQ_PUT_DEC(str, val) \ seq_put_decimal_ull_width(m, str, (val) >> 10, 8) @@ -1346,7 +1363,7 @@ static int show_smap(struct seq_file *m, void *v) struct vm_area_struct *vma = v; struct mem_size_stats mss = {}; - smap_gather_stats(priv, vma, &mss, 0); + smap_gather_stats(priv, vma, &mss); show_map_vma(m, vma); @@ -1399,7 +1416,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) vma_start = vma->vm_start; do { - smap_gather_stats(priv, vma, &mss, 0); + smap_gather_stats(priv, vma, &mss); last_vma_end = vma->vm_end; /* @@ -1458,14 +1475,15 @@ static int show_smaps_rollup(struct seq_file *m, void *v) /* Case 1 and 2 above */ if (vma->vm_start >= last_vma_end) { - smap_gather_stats(priv, vma, &mss, 0); + smap_gather_stats(priv, vma, &mss); last_vma_end = vma->vm_end; continue; } /* Case 4 above */ if (vma->vm_end > last_vma_end) { - smap_gather_stats(priv, vma, &mss, last_vma_end); + smap_gather_stats_range(priv, vma, &mss, + last_vma_end); last_vma_end = vma->vm_end; } } From 60d21d883f071e69f363b6186201762c07aff548 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 18 Sep 2026 08:33:16 -0700 Subject: [PATCH 495/539] proc/task_mmu: change proc_get_vma() to stop returning gate VMA at the end proc_get_vma() returning gate VMA at the end is desirable for the its current m_start/m_next callers, as they need to report a gate VMA at the end of the address space. This behavior is very specific to these callers and makes proc_get_vma() hard to use for other purposes. Move this usage-specific behavior into the callers themselves so that proc_get_vma() returns either a valid VMA, an error or a NULL when no more VMAs are available. This makes it more generic, simpler and usable in the later patches. Link: https://lore.kernel.org/20260918153318.758387-6-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett (Oracle) Cc: Matthew Wilcox (Oracle) Cc: "Paul E . McKenney" Cc: Pedro Falcato Cc: Usama Arif Cc: Vlastimil Babka --- fs/proc/task_mmu.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 06fa97241e9b..d075b715a80c 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -240,9 +240,6 @@ retry: * found the extended vma with the same vm_start. */ *ppos = vma->vm_end; - } else { - *ppos = SENTINEL_VMA_GATE; - vma = get_gate_vma(priv->lock_ctx.mm); } return vma; @@ -252,6 +249,7 @@ static void *m_start(struct seq_file *m, loff_t *ppos) { struct proc_maps_private *priv = m->private; struct proc_maps_locking_ctx *lock_ctx; + struct vm_area_struct *vma; loff_t last_addr = *ppos; struct mm_struct *mm; @@ -281,19 +279,39 @@ static void *m_start(struct seq_file *m, loff_t *ppos) *ppos = last_addr = priv->last_pos; vma_iter_init(&priv->iter, mm, (unsigned long)last_addr); hold_task_mempolicy(priv); + /* + * If seq_file had to flush its collected data right after m_next() set + * position to SENTINEL_VMA_GATE, m_start() will get that sentinel and + * should return gate_vma without calling proc_get_vma(). + */ if (last_addr == SENTINEL_VMA_GATE) return get_gate_vma(mm); - return proc_get_vma(m, ppos); + vma = proc_get_vma(m, ppos); + if (vma) + return vma; + + /* Return gate VMA at the end */ + *ppos = SENTINEL_VMA_GATE; + return get_gate_vma(mm); } static void *m_next(struct seq_file *m, void *v, loff_t *ppos) { + struct proc_maps_private *priv = m->private; + struct vm_area_struct *vma; + if (*ppos == SENTINEL_VMA_GATE) { *ppos = SENTINEL_VMA_END; return NULL; } - return proc_get_vma(m, ppos); + vma = proc_get_vma(m, ppos); + if (vma) + return vma; + + /* Return gate VMA at the end */ + *ppos = SENTINEL_VMA_GATE; + return get_gate_vma(priv->lock_ctx.mm); } static void m_stop(struct seq_file *m, void *v) From 0e985559d72a5af5389762ce16b7209e328af4f5 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 18 Sep 2026 08:33:17 -0700 Subject: [PATCH 496/539] proc/task_mmu: read proc/pid/smaps_rollup under per-vma lock proc/pid/smaps_rollup can be read using the combination of RCU and VMA read locks, similar to proc/pid/{maps|smaps|numa_maps}. RCU is required to safely traverse the VMA tree and VMA lock stabilizes the VMA being processed and the pagetable walk. Note that we have to keep the logic to drop mmap_lock on contention because even when using per-VMA locks we might have to fall back to holding the mmap_lock. Running Paul's contention benchmark [1] shows considerable improvement both in median and in the worst case latencies: Execution command: run-proc-vs-map.sh --nsamples 20 --rawdata -- \ --busyduration 2 --procfile smaps_rollup Baseline: Median Minimum Maximum 0.174 0.161 2.553 0.174 0.164 2.663 0.174 0.165 2.664 0.174 0.166 2.679 0.174 0.167 2.691 0.174 0.168 2.704 0.174 0.169 2.729 0.174 0.172 2.741 0.174 0.174 2.745 0.174 0.174 2.755 0.174 0.175 2.790 0.174 0.177 2.809 0.174 0.179 3.096 0.174 0.183 3.144 0.174 0.184 3.158 0.174 0.185 3.175 0.174 0.185 4.568 0.174 0.198 4.821 0.174 0.214 5.143 0.174 0.251 5.220 Patched: Median Minimum Maximum 0.007 0.007 1.952 0.007 0.007 1.955 0.007 0.007 1.955 0.007 0.007 1.955 0.007 0.007 1.957 0.007 0.007 1.969 0.007 0.007 2.065 0.007 0.007 2.075 0.007 0.007 2.146 0.007 0.007 2.195 0.007 0.007 2.223 0.007 0.007 2.259 0.007 0.007 2.488 0.007 0.007 2.562 0.007 0.007 2.599 0.007 0.007 2.697 0.007 0.007 3.030 0.007 0.007 3.075 0.007 0.007 3.145 0.007 0.007 3.225 Remove now unused lock_ctx_mm() and move unlock_ctx_vma() next to unlock_ctx_mm() as they are logically related. Remove a long comment about 4 cases that we handle when dropping the mmap lock in the middle of VMA walk due to contention. The first 3 cases explained there are handled naturally and only case 4 needs to be handled in a special way, which is done in smap_gather_stats() by gathering stats from the portion of the VMA that has not yet been processed. For posterity, moving this comment here: After dropping the lock, there are four cases to consider. See the following example for explanation. +------+------+-----------+ | VMA1 | VMA2 | VMA3 | +------+------+-----------+ | | | | 4k 8k 16k 400k Suppose we drop the lock after reading VMA2 due to contention, then we get: last_vma_end = 16k 1) VMA2 is freed, but VMA3 exists: vma_next(vmi) will return VMA3. In this case, just continue from VMA3. 2) VMA2 still exists: vma_next(vmi) will return VMA3. In this case, just continue from VMA3. 3) No more VMAs can be found: vma_next(vmi) will return NULL. No more things to do, just break. 4) (last_vma_end - 1) is the middle of a vma (VMA'): vma_next(vmi) will return VMA' whose range contains last_vma_end. Iterate VMA' from last_vma_end. Link: https://lore.kernel.org/20260918153318.758387-7-surenb@google.com Link: https://github.com/paulmckrcu/proc-mmap_sem-test [1] Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Reviewed-by: Lorenzo Stoakes (ARM) Cc: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett (Oracle) Cc: Matthew Wilcox (Oracle) Cc: "Paul E . McKenney" Cc: Pedro Falcato Cc: Usama Arif Cc: Vlastimil Babka --- fs/proc/task_mmu.c | 159 ++++++++++++++++++--------------------------- 1 file changed, 63 insertions(+), 96 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index d075b715a80c..ee9fc2fd2b2f 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -130,30 +130,12 @@ static void release_task_mempolicy(struct proc_maps_private *priv) } #endif -#ifdef CONFIG_PROC_PAGE_MONITOR -static int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) -{ - int ret = mmap_read_lock_killable(lock_ctx->mm); - - if (!ret) - lock_ctx->mmap_locked = true; - - return ret; -} -#endif - static void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) { mmap_read_unlock(lock_ctx->mm); lock_ctx->mmap_locked = false; } -static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) -{ - lock_ctx->locked_vma = NULL; - lock_ctx->mmap_locked = false; -} - static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) { if (lock_ctx->locked_vma) { @@ -162,6 +144,12 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } +static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) +{ + lock_ctx->locked_vma = NULL; + lock_ctx->mmap_locked = false; +} + static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, loff_t last_pos) { @@ -1406,12 +1394,14 @@ static int show_smap(struct seq_file *m, void *v) static int show_smaps_rollup(struct seq_file *m, void *v) { struct proc_maps_private *priv = m->private; + struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; + struct mm_struct *mm = lock_ctx->mm; struct mem_size_stats mss = {}; - struct mm_struct *mm = priv->lock_ctx.mm; + unsigned long last_vma_end = 0; + unsigned long vma_start = 0; struct vm_area_struct *vma; - unsigned long vma_start = 0, last_vma_end = 0; + loff_t pos = 0; int ret = 0; - VMA_ITERATOR(vmi, mm, 0); priv->task = get_proc_task(priv->inode); if (!priv->task) @@ -1422,90 +1412,63 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_task; } - ret = lock_ctx_mm(&priv->lock_ctx); - if (ret) - goto out_put_mm; - hold_task_mempolicy(priv); - vma = vma_next(&vmi); + rcu_read_lock(); + reset_lock_ctx(lock_ctx); + vma_iter_init(&priv->iter, mm, 0); + vma = proc_get_vma(m, &pos); if (unlikely(!vma)) goto empty_set; - vma_start = vma->vm_start; - do { - smap_gather_stats(priv, vma, &mss); + if (!IS_ERR(vma)) + vma_start = vma->vm_start; + + while (vma) { + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto out_unlock; + } + + if (vma->vm_start < last_vma_end) { + /* + * After retaking the lock, already reported VMA grew + * or got merged with the next one and we found it + * again. Gather stats for the remaining portion by + * starting at last_vma_end. + */ + smap_gather_stats_range(priv, vma, &mss, last_vma_end); + } else { + /* Found next unreported VMA, start from its beginning */ + smap_gather_stats(priv, vma, &mss); + } last_vma_end = vma->vm_end; /* - * Release mmap_lock temporarily if someone wants to - * access it for write request. + * If the VMA lock is not taken, we hold the often contended + * mmap lock. This can happen if we had to fall back to the + * mmap lock. + * + * To relieve pressure, check if it is indeed contended, then + * temporarily release it. */ - if (mmap_lock_is_contended(mm)) { - vma_iter_invalidate(&vmi); - unlock_ctx_mm(&priv->lock_ctx); - ret = lock_ctx_mm(&priv->lock_ctx); - if (ret) { - release_task_mempolicy(priv); - goto out_put_mm; - } - + if (lock_ctx->mmap_locked && + mmap_lock_is_contended(lock_ctx->mm)) { + unlock_ctx_mm(lock_ctx); /* - * After dropping the lock, there are four cases to - * consider. See the following example for explanation. - * - * +------+------+-----------+ - * | VMA1 | VMA2 | VMA3 | - * +------+------+-----------+ - * | | | | - * 4k 8k 16k 400k - * - * Suppose we drop the lock after reading VMA2 due to - * contention, then we get: - * - * last_vma_end = 16k - * - * 1) VMA2 is freed, but VMA3 exists: - * - * vma_next(vmi) will return VMA3. - * In this case, just continue from VMA3. - * - * 2) VMA2 still exists: - * - * vma_next(vmi) will return VMA3. - * In this case, just continue from VMA3. - * - * 3) No more VMAs can be found: - * - * vma_next(vmi) will return NULL. - * No more things to do, just break. - * - * 4) (last_vma_end - 1) is the middle of a vma (VMA'): - * - * vma_next(vmi) will return VMA' whose range - * contains last_vma_end. - * Iterate VMA' from last_vma_end. + * Even though we previously fell back to mmap lock, + * we try taking VMA lock for the next VMA, since it + * might not be under modification. In the worst case + * we will fall back to mmap lock again. */ - vma = vma_next(&vmi); - /* Case 3 above */ - if (!vma) - break; - - /* Case 1 and 2 above */ - if (vma->vm_start >= last_vma_end) { - smap_gather_stats(priv, vma, &mss); - last_vma_end = vma->vm_end; - continue; - } - - /* Case 4 above */ - if (vma->vm_end > last_vma_end) { - smap_gather_stats_range(priv, vma, &mss, - last_vma_end); - last_vma_end = vma->vm_end; - } + rcu_read_lock(); + reset_lock_ctx(lock_ctx); + /* Resume from the last position. */ + pos = last_vma_end; + vma_iter_init(&priv->iter, mm, pos); } - } for_each_vma(vmi, vma); + vma = proc_get_vma(m, &pos); + } empty_set: show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); @@ -1514,10 +1477,14 @@ empty_set: __show_smap(m, &mss, true); +out_unlock: + if (lock_ctx->mmap_locked) { + unlock_ctx_mm(lock_ctx); + } else { + unlock_ctx_vma(lock_ctx); + rcu_read_unlock(); + } release_task_mempolicy(priv); - unlock_ctx_mm(&priv->lock_ctx); - -out_put_mm: mmput(mm); out_put_task: put_task_struct(priv->task); From ca5480fb53fa73cb5816b696b103826daedab1ca Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 18 Sep 2026 08:33:18 -0700 Subject: [PATCH 497/539] selftests/proc: add /proc/pid/smaps_rollup tearing tests During tearing tests, smaps_rollup Pss* metrics should stay constant. Extend /proc/pid/smaps tearing tests to also check for smaps_rollup consistency. Link: https://lore.kernel.org/20260918153318.758387-8-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton Acked-by: Lorenzo Stoakes (ARM) Cc: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett (Oracle) Cc: Matthew Wilcox (Oracle) Cc: "Paul E . McKenney" Cc: Pedro Falcato Cc: Usama Arif Cc: Vlastimil Babka --- tools/testing/selftests/proc/proc-maps-race.c | 186 +++++++++++++++++- 1 file changed, 181 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/proc/proc-maps-race.c b/tools/testing/selftests/proc/proc-maps-race.c index 415eccb70468..bf4c5073f6fc 100644 --- a/tools/testing/selftests/proc/proc-maps-race.c +++ b/tools/testing/selftests/proc/proc-maps-race.c @@ -80,6 +80,61 @@ enum maps_file { struct vma_modifier_info; +enum smaps_rollup_stat { + Rss, + Pss, + Pss_Dirty, + Pss_Anon, + Pss_File, + Pss_Shmem, + Shared_Clean, + Shared_Dirty, + Private_Clean, + Private_Dirty, + Referenced, + Anonymous, + KSM, + LazyFree, + AnonHugePages, + ShmemPmdMapped, + FilePmdMapped, + Shared_Hugetlb, + Private_Hugetlb, + Swap, + SwapPss, + Locked, + RollupFieldCount +}; + +static const char *smaps_rollup_stat_names[RollupFieldCount] = { + "Rss", + "Pss", + "Pss_Dirty", + "Pss_Anon", + "Pss_File", + "Pss_Shmem", + "Shared_Clean", + "Shared_Dirty", + "Private_Clean", + "Private_Dirty", + "Referenced", + "Anonymous", + "KSM", + "LazyFree", + "AnonHugePages", + "ShmemPmdMapped", + "FilePmdMapped", + "Shared_Hugetlb", + "Private_Hugetlb", + "Swap", + "SwapPss", + "Locked", +}; + +struct smaps_rollup_stats { + unsigned long values[RollupFieldCount]; +}; + FIXTURE(proc_maps_race) { struct vma_modifier_info *mod_info; @@ -91,6 +146,7 @@ FIXTURE(proc_maps_race) enum maps_file maps_file; int shared_mem_size; int skip_pages; + int rollup_fd; int page_size; int vma_count; bool verbose; @@ -132,12 +188,12 @@ struct vma_modifier_info { void *child_mapped_addr[]; }; -static bool read_page(FIXTURE_DATA(proc_maps_race) *self, +static bool read_page(FIXTURE_DATA(proc_maps_race) *self, int fd, struct page_content *page) { ssize_t bytes_read; - bytes_read = read(self->maps_fd, page->data, self->page_size); + bytes_read = read(fd, page->data, self->page_size); if (bytes_read <= 0) return false; @@ -175,7 +231,7 @@ static int locate_containing_page(FIXTURE_DATA(proc_maps_race) *self, char *curr_pos; char *end_pos; - if (!read_page(self, &self->page1)) + if (!read_page(self, self->maps_fd, &self->page1)) return -1; curr_pos = self->page1.data; @@ -205,10 +261,11 @@ static bool read_two_pages(FIXTURE_DATA(proc_maps_race) *self) return false; for (int i = 0; i < self->skip_pages; i++) - if (!read_page(self, &self->page1)) + if (!read_page(self, self->maps_fd, &self->page1)) return false; - return read_page(self, &self->page1) && read_page(self, &self->page2); + return read_page(self, self->maps_fd, &self->page1) && + read_page(self, self->maps_fd, &self->page2); } static void copy_line(const char *line_start, const char *line_end, @@ -317,6 +374,61 @@ static bool read_boundary_lines(FIXTURE_DATA(proc_maps_race) *self, &first_line->end_addr) == 2; } +static bool parse_smaps_rollup(FIXTURE_DATA(proc_maps_race) *self, + struct smaps_rollup_stats *stats) +{ + unsigned int dev_maj, dev_min, inode; + unsigned long start, end, offs; + unsigned long value; + char name[32], perm[5]; + char *curr_pos; + char *end_pos; + char *line_end; + + if (lseek(self->rollup_fd, 0, SEEK_SET) < 0) + return false; + + if (!read_page(self, self->rollup_fd, &self->page1)) + return false; + + curr_pos = self->page1.data; + end_pos = self->page1.data + self->page1.size; + + line_end = strchr(curr_pos, '\n'); + if (!line_end) + return false; + + if (sscanf(curr_pos, "%lx-%lx %4s %lx %u:%u %u %31s", + &start, &end, perm, &offs, &dev_maj, &dev_min, &inode, name) != 8) + return false; + + if (strcmp(name, "[rollup]")) + return false; + + for (int stat = 0; stat < ARRAY_SIZE(smaps_rollup_stat_names); stat++) { + int len; + + curr_pos = line_end + 1; + if (curr_pos >= end_pos) + return false; + + line_end = strchr(curr_pos, '\n'); + if (!line_end) + return false; + + if (sscanf(curr_pos, "%31s %lu kB", name, &value) != 2) + return false; + + len = strlen(name); + if (name[len - 1] != ':' || strncmp(name, smaps_rollup_stat_names[stat], len - 1)) + return false; + + stats->values[stat] = value; + } + + return true; +} + /* Thread synchronization routines */ static void wait_for_state(struct vma_modifier_info *mod_info, enum test_state state) { @@ -397,6 +509,40 @@ static bool print_boundaries_on(bool condition, const char *title, return condition; } +static void print_smaps_rollup_stats(const char *title, FIXTURE_DATA(proc_maps_race) *self, + struct smaps_rollup_stats *stats) +{ + printf("%s", title); + for (int stat = 0; stat < ARRAY_SIZE(smaps_rollup_stat_names); stat++) + printf("%64s %lu kB\n", smaps_rollup_stat_names[stat], stats->values[stat]); +} + +static bool cmp_smaps_rollup_stat(struct smaps_rollup_stats *s1, + struct smaps_rollup_stats *s2, enum smaps_rollup_stat stat) +{ + return s1->values[stat] == s2->values[stat]; +} + +static bool compare_smaps_rollup(FIXTURE_DATA(proc_maps_race) *self, + struct smaps_rollup_stats *expected, + struct smaps_rollup_stats *actual) +{ + /* + * Clean/dirty metrics might change but Pss-related ones + * should stay constant. + */ + if (cmp_smaps_rollup_stat(expected, actual, Pss) && + cmp_smaps_rollup_stat(expected, actual, Pss_Anon) && + cmp_smaps_rollup_stat(expected, actual, Pss_File) && + cmp_smaps_rollup_stat(expected, actual, Pss_Shmem)) + return true; + + print_smaps_rollup_stats("Expected stats:", self, expected); + print_smaps_rollup_stats("Actual stats:", self, actual); + + return false; +} + static void report_test_start(const char *name, bool verbose) { if (verbose) @@ -572,6 +718,7 @@ FIXTURE_SETUP(proc_maps_race) unsigned long first_map_addr; unsigned long last_map_addr; unsigned long duration_sec; + char rollup_fname[32]; char fname[32]; self->page_size = (unsigned long)sysconf(_SC_PAGESIZE); @@ -649,6 +796,9 @@ FIXTURE_SETUP(proc_maps_race) break; case SMAPS: sprintf(fname, "/proc/%d/smaps", self->pid); + sprintf(rollup_fname, "/proc/%d/smaps_rollup", self->pid); + self->rollup_fd = open(rollup_fname, O_RDONLY); + ASSERT_NE(self->rollup_fd, -1); break; default: ksft_exit_fail(); @@ -711,6 +861,8 @@ FIXTURE_TEARDOWN(proc_maps_race) for (int i = 0; i < self->vma_count; i++) munmap(self->mod_info->child_mapped_addr[i], self->page_size); close(self->maps_fd); + if (self->maps_file == SMAPS) + close(self->rollup_fd); waitpid(self->pid, &status, 0); munmap(self->mod_info, self->shared_mem_size); } @@ -723,6 +875,7 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) struct line_content split_first_line; struct line_content restored_last_line; struct line_content restored_first_line; + struct smaps_rollup_stats orig_stats; wait_for_state(mod_info, SETUP_READY); @@ -736,6 +889,8 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) report_test_start("Tearing from split", self->verbose); ASSERT_TRUE(capture_mod_pattern(self, &split_last_line, &split_first_line, &restored_last_line, &restored_first_line)); + if (self->maps_file == SMAPS) + ASSERT_TRUE(parse_smaps_rollup(self, &orig_stats)); /* Now start concurrent modifications for self->duration_sec */ signal_state(mod_info, TEST_READY); @@ -799,6 +954,11 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) vma_end == self->last_line.end_addr) || (vma_start == split_first_line.start_addr && vma_end == split_first_line.end_addr)); + } else { + struct smaps_rollup_stats stats; + + ASSERT_TRUE(parse_smaps_rollup(self, &stats)); + ASSERT_TRUE(compare_smaps_rollup(self, &orig_stats, &stats)); } clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); @@ -817,6 +977,7 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize) struct line_content shrunk_first_line; struct line_content restored_last_line; struct line_content restored_first_line; + struct smaps_rollup_stats orig_stats; wait_for_state(mod_info, SETUP_READY); @@ -830,6 +991,8 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize) report_test_start("Tearing from resize", self->verbose); ASSERT_TRUE(capture_mod_pattern(self, &shrunk_last_line, &shrunk_first_line, &restored_last_line, &restored_first_line)); + if (self->maps_file == SMAPS) + ASSERT_TRUE(parse_smaps_rollup(self, &orig_stats)); /* Now start concurrent modifications for self->duration_sec */ signal_state(mod_info, TEST_READY); @@ -880,6 +1043,11 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize) ASSERT_TRUE(vma_start == self->last_line.start_addr && (vma_end - vma_start == self->page_size * 3 || vma_end - vma_start == self->page_size)); + } else { + struct smaps_rollup_stats stats; + + ASSERT_TRUE(parse_smaps_rollup(self, &stats)); + ASSERT_TRUE(compare_smaps_rollup(self, &orig_stats, &stats)); } clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); @@ -898,6 +1066,7 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap) struct line_content remapped_first_line; struct line_content restored_last_line; struct line_content restored_first_line; + struct smaps_rollup_stats orig_stats; wait_for_state(mod_info, SETUP_READY); @@ -911,6 +1080,8 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap) report_test_start("Tearing from remap", self->verbose); ASSERT_TRUE(capture_mod_pattern(self, &remapped_last_line, &remapped_first_line, &restored_last_line, &restored_first_line)); + if (self->maps_file == SMAPS) + ASSERT_TRUE(parse_smaps_rollup(self, &orig_stats)); /* Now start concurrent modifications for self->duration_sec */ signal_state(mod_info, TEST_READY); @@ -963,6 +1134,11 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap) vma_end - vma_start == self->page_size * 3) || (vma_start == self->last_line.start_addr + self->page_size && vma_end - vma_start == self->page_size)); + } else { + struct smaps_rollup_stats stats; + + ASSERT_TRUE(parse_smaps_rollup(self, &stats)); + ASSERT_TRUE(compare_smaps_rollup(self, &orig_stats, &stats)); } clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); From e4ea16e00288f9f925104ea11dba3a364dd2d1c4 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Sat, 19 Sep 2026 01:56:40 +0900 Subject: [PATCH 498/539] mm/swapops: remove unused is_hwpoison_entry() Since commit 93976a20345b ("mm: eliminate further swapops predicates"), is_hwpoison_entry() has no callers. So remove it. No functional change. Link: https://lore.kernel.org/20260918165642.1014988-1-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Signed-off-by: Andrew Morton Reviewed-by: SJ Park Reviewed-by: Zenghui Yu (Huawei) Reviewed-by: Barry Song Acked-by: Chris Li Cc: Baoquan He Cc: Kairui Song Cc: Kemeng Shi Cc: Nhat Pham Cc: Youngjun Park --- include/linux/swapops.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/include/linux/swapops.h b/include/linux/swapops.h index e7d0d529f3e0..603f9e3f909c 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -261,11 +261,6 @@ static inline swp_entry_t make_hwpoison_entry(struct page *page) return swp_entry(SWP_HWPOISON, page_to_pfn(page)); } -static inline int is_hwpoison_entry(swp_entry_t entry) -{ - return swp_type(entry) == SWP_HWPOISON; -} - #else static inline swp_entry_t make_hwpoison_entry(struct page *page) @@ -273,10 +268,6 @@ static inline swp_entry_t make_hwpoison_entry(struct page *page) return swp_entry(0, 0); } -static inline int is_hwpoison_entry(swp_entry_t swp) -{ - return 0; -} #endif typedef unsigned long pte_marker; From 23d0a7444fca5cf3d21a3e366ade0219c095fada Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Fri, 18 Sep 2026 16:52:29 +0530 Subject: [PATCH 499/539] selftests/mm: make file helpers return errors Patch series "selftests/mm: separate GUP microbenchmarking from functional testing", v11. gup_test.c currently serves two separate purposes: benchmarking (GUP_FAST_BENCHMARK, PIN_FAST_BENCHMARK and PIN_LONGTERM_BENCHMARK) and functional testing (GUP_BASIC_TEST, PIN_BASIC_TEST and DUMP_USER_PAGES_TEST). Keeping both in one program makes the functional tests harder to run and report individually, while run_vmtests.sh has to invoke the program repeatedly with different options. Separate these roles into tools/mm/gup_bench for benchmarking and tools/testing/selftests/mm/gup for functional testing. Move the shared file and hugepage helpers to tools/lib/mm/ so both programs can use them without duplicating the implementation. Patch 1 makes read_file(), write_file(), read_num(), write_num() and write_num_ignore_einval() return errors to their callers instead of exiting. It also makes read_num() reject negative and malformed values and updates the existing callers to handle failures. Patch 2 moves these file helpers from vm_util.c to tools/lib/mm/. It keeps them available to the mm selftests through vm_util.h and adjusts the selftests build accordingly. Patch 3 moves hugepage_settings.[ch] from selftests/mm to tools/lib/mm/. It also removes its kselftest dependency while preserving TAP-compatible diagnostics for selftest users. Patch 4 moves the existing gup_test implementation from selftests/mm to tools/mm as gup_bench. This keeps the code movement separate from the subsequent changes and makes it easier to review. Patch 5 removes the functional test modes and kselftest dependency from gup_bench. When run without arguments, it performs one GUP_FAST benchmark using the existing defaults instead of running the whole matrix. Other benchmark configurations can be selected through command-line options. Patch 6 adds a new harness-based GUP selftest. It covers THP, non-THP and HugeTLB mappings across private/shared and read/write variants. For each variant, it tests get_user_pages(), get_user_pages_fast(), pin_user_pages(), pin_user_pages_fast() and long-term pinning modes using four batch sizes. The HugeTLB variants share a one-time setup of two hugeTLB pages. This patch (of 6): Change read_file(), write_file(), read_num(), write_num() and write_num_ignore_einval() in vm_util.c to report failures to callers instead of exiting from the helper. Make read_file() return a negative errno on failure and 0 on success, so callers can distinguish a successful read from an I/O error. Also make read_num() reject negative and malformed values. Keep write_num_ignore_einval() silent for -EINVAL while returning other errors to its caller. Update callers to print diagnostics and fail wherever required. Modify a comment which implies write_num() uses ksft_exit_fail_msg(). Also add a helper print_file_access_error() in hugepage_settings.c to print TAP-compatible errors without a kselftest dependency. This prepares the helpers to be moved to tools/lib/mm without a kselftest dependency. [sarthak.sharma@arm.com: convert hugetlb_nr_resv_pages()] Link: https://lore.kernel.org/937939c3-ae9a-4148-a601-0f8876216423@arm.com Link: https://lore.kernel.org/20260918112234.195857-1-sarthak.sharma@arm.com Link: https://lore.kernel.org/20260918112234.195857-2-sarthak.sharma@arm.com Signed-off-by: Sarthak Sharma Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Tested-by: Muhammad Usama Anjum Cc: Anshuman Khandual Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Jason Gunthorpe Cc: John Hubbard Cc: Jonathan Corbet Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Mark Brown Cc: Michal Hocko Cc: Nico Pache Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Zi Yan --- .../testing/selftests/mm/hugepage_settings.c | 108 ++++++++++++--- .../selftests/mm/hugetlb-soft-offline.c | 16 ++- tools/testing/selftests/mm/khugepaged.c | 14 +- .../selftests/mm/split_huge_page_test.c | 5 +- tools/testing/selftests/mm/vm_util.c | 124 ++++++++++++------ tools/testing/selftests/mm/vm_util.h | 8 +- 6 files changed, 202 insertions(+), 73 deletions(-) diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/testing/selftests/mm/hugepage_settings.c index 584054736ce9..6f3abd357385 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "vm_util.h" #include "hugepage_settings.h" @@ -48,6 +49,11 @@ static const char * const shmem_enabled_strings[] = { NULL }; +static void print_file_access_error(const char *path, int ret) +{ + printf("# %s: %s (%d)\n", path, strerror(-ret), -ret); +} + int thp_read_string(const char *name, const char * const strings[]) { char path[PATH_MAX]; @@ -61,8 +67,9 @@ int thp_read_string(const char *name, const char * const strings[]) exit(EXIT_FAILURE); } - if (!read_file(path, buf, sizeof(buf))) { - perror(path); + ret = read_file(path, buf, sizeof(buf)); + if (ret) { + print_file_access_error(path, ret); exit(EXIT_FAILURE); } @@ -103,12 +110,17 @@ void thp_write_string(const char *name, const char *val) printf("%s: Pathname is too long\n", __func__); exit(EXIT_FAILURE); } - write_file(path, val, strlen(val) + 1); + ret = write_file(path, val, strlen(val) + 1); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } } unsigned long thp_read_num(const char *name) { char path[PATH_MAX]; + unsigned long num; int ret; ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); @@ -116,7 +128,13 @@ unsigned long thp_read_num(const char *name) printf("%s: Pathname is too long\n", __func__); exit(EXIT_FAILURE); } - return read_num(path); + ret = read_num(path, &num); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + + return num; } void thp_write_num(const char *name, unsigned long num) @@ -129,7 +147,11 @@ void thp_write_num(const char *name, unsigned long num) printf("%s: Pathname is too long\n", __func__); exit(EXIT_FAILURE); } - write_num(path, num); + ret = write_num(path, num); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } } void thp_read_settings(struct thp_settings *settings) @@ -157,8 +179,15 @@ void thp_read_settings(struct thp_settings *settings) .max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"), .pages_to_scan = thp_read_num("khugepaged/pages_to_scan"), }; - if (dev_queue_read_ahead_path[0]) - settings->read_ahead_kb = read_num(dev_queue_read_ahead_path); + if (dev_queue_read_ahead_path[0]) { + int ret = read_num(dev_queue_read_ahead_path, + &settings->read_ahead_kb); + + if (ret) { + print_file_access_error(dev_queue_read_ahead_path, ret); + exit(EXIT_FAILURE); + } + } for (i = 0; i < NR_ORDERS; i++) { if (!((1 << i) & orders)) { @@ -208,8 +237,15 @@ void thp_write_settings(struct thp_settings *settings) thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); - if (dev_queue_read_ahead_path[0]) - write_num(dev_queue_read_ahead_path, settings->read_ahead_kb); + if (dev_queue_read_ahead_path[0]) { + int ret = write_num(dev_queue_read_ahead_path, + settings->read_ahead_kb); + + if (ret) { + print_file_access_error(dev_queue_read_ahead_path, ret); + exit(EXIT_FAILURE); + } + } for (i = 0; i < NR_ORDERS; i++) { if (!((1 << i) & orders)) @@ -307,8 +343,15 @@ static unsigned long __thp_supported_orders(bool is_shmem) } ret = read_file(path, buf, sizeof(buf)); - if (ret) - orders |= 1UL << i; + if (ret) { + if (ret != -ENOENT) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + continue; + } + + orders |= 1UL << i; } return orders; @@ -382,8 +425,7 @@ int detect_hugetlb_page_sizes(unsigned long sizes[], int max) if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) continue; sizes[count++] = kb * 1024; - ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", - kb); + printf("# [INFO] detected hugetlb page size: %zu KiB\n", kb); } closedir(dir); return count; @@ -425,37 +467,66 @@ static void hugetlb_sysfs_path(char *buf, size_t buflen, unsigned long hugetlb_nr_pages(unsigned long size) { char path[PATH_MAX]; + unsigned long nr; + int ret; hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - return read_num(path); + ret = read_num(path, &nr); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + + return nr; } void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) { char path[PATH_MAX]; + int ret; hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - write_num_ignore_einval(path, nr); + ret = write_num_ignore_einval(path, nr); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } } unsigned long hugetlb_free_pages(unsigned long size) { char path[PATH_MAX]; + unsigned long nr; + int ret; hugetlb_sysfs_path(path, sizeof(path), size, "free_hugepages"); - return read_num(path); + ret = read_num(path, &nr); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + + return nr; } unsigned long hugetlb_nr_resv_pages(unsigned long size) { char path[PATH_MAX]; + unsigned long nr; + int ret; hugetlb_sysfs_path(path, sizeof(path), size, "resv_hugepages"); - return read_num(path); + ret = read_num(path, &nr); + if (ret) { + print_file_access_error(path, ret); + exit(EXIT_FAILURE); + } + + return nr; } static bool __hugetlb_setup(unsigned long size, unsigned long nr) @@ -511,7 +582,8 @@ unsigned long hugetlb_setup(unsigned long nr, unsigned long sizes[], return 0; if (nr_enabled > max) { - ksft_print_msg("detected %d huge page sizes, will only test %d\n", nr_enabled, max); + printf("# detected %d huge page sizes, will only test %d\n", + nr_enabled, max); nr_enabled = max; } diff --git a/tools/testing/selftests/mm/hugetlb-soft-offline.c b/tools/testing/selftests/mm/hugetlb-soft-offline.c index 4af9d3db7b5b..ffc85b958c69 100644 --- a/tools/testing/selftests/mm/hugetlb-soft-offline.c +++ b/tools/testing/selftests/mm/hugetlb-soft-offline.c @@ -85,8 +85,7 @@ static unsigned long orig_enable_soft_offline = -1UL; /* * Runs from an atexit handler, so it must not call anything that - * exits on failure: write_num() would re-enter exit() through - * ksft_exit_fail_msg(). + * exits on failure. */ static void restore_enable_soft_offline(void) { @@ -152,7 +151,10 @@ static void test_soft_offline_common(int enable_soft_offline) hugepagesize_kb = file_stat.f_bsize / 1024; ksft_print_msg("Hugepagesize is %ldkB\n", hugepagesize_kb); - write_num(ENABLE_SOFT_OFFLINE_PATH, enable_soft_offline); + ret = write_num(ENABLE_SOFT_OFFLINE_PATH, enable_soft_offline); + if (ret) + ksft_exit_fail_msg("Failed to write to %s: %s\n", + ENABLE_SOFT_OFFLINE_PATH, strerror(-ret)); nr_hugepages_before = hugetlb_nr_default_pages(); @@ -189,6 +191,8 @@ static void test_soft_offline_common(int enable_soft_offline) int main(int argc, char **argv) { + int ret; + ksft_print_header(); if (!hugetlb_setup_default(8)) @@ -196,7 +200,11 @@ int main(int argc, char **argv) ksft_set_plan(2); - orig_enable_soft_offline = read_num(ENABLE_SOFT_OFFLINE_PATH); + ret = read_num(ENABLE_SOFT_OFFLINE_PATH, &orig_enable_soft_offline); + if (ret) + ksft_exit_fail_msg("Failed to read %s: %s\n", + ENABLE_SOFT_OFFLINE_PATH, strerror(-ret)); + atexit(restore_enable_soft_offline); test_soft_offline_common(1); diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index f82673f5f6b4..6daa22f6da2f 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -122,6 +122,7 @@ static void get_finfo(const char *dir) char buf[1 << 10]; char path[PATH_MAX]; char *str, *end; + int ret; finfo.dir = dir; if (stat(finfo.dir, &path_stat)) @@ -142,8 +143,9 @@ static void get_finfo(const char *dir) major(path_stat.st_dev), minor(path_stat.st_dev)) >= sizeof(path)) ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); - if (!read_file(path, buf, sizeof(buf))) - ksft_exit_fail_perror("read_file(uevent)"); + ret = read_file(path, buf, sizeof(buf)); + if (ret) + ksft_exit_fail_msg("read_file(%s): %s\n", path, strerror(-ret)); if (strstr(buf, "DEVTYPE=disk")) { /* Found it */ if (snprintf(finfo.dev_queue_read_ahead_path, @@ -324,7 +326,7 @@ static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) { const int open_opt = setup == FILE_SETUP_READ_ONLY_FS ? O_RDONLY : O_RDWR; const int mmap_prot = setup == FILE_SETUP_READ_ONLY_FS ? PROT_READ : (PROT_READ | PROT_WRITE); - int fd; + int fd, ret; void *p; unsigned long size; @@ -362,7 +364,11 @@ static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) ksft_exit_fail_perror("mmap()"); /* Drop page cache */ - write_file("/proc/sys/vm/drop_caches", "3", 2); + ret = write_file("/proc/sys/vm/drop_caches", "3", 2); + if (ret) + ksft_exit_fail_msg("write_file(drop_caches): %s\n", + strerror(-ret)); + success("OK"); return p; } diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index c01d227d7fd6..a30927514b4f 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -145,7 +145,10 @@ static void write_debugfs(const char *fmt, ...) if (ret >= INPUT_MAX) ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); - write_file(SPLIT_DEBUGFS, input, ret + 1); + ret = write_file(SPLIT_DEBUGFS, input, ret + 1); + if (ret) + ksft_exit_fail_msg("write_file(%s): %s\n", SPLIT_DEBUGFS, + strerror(-ret)); } static char *allocate_zero_filled_hugepage(size_t len) diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 4821a3563036..d04c904e76c3 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -887,109 +887,149 @@ int unpoison_memory(unsigned long pfn) int read_file(const char *path, char *buf, size_t buflen) { - int fd; + int fd, err; ssize_t numread; fd = open(path, O_RDONLY); if (fd == -1) - return 0; + return -errno; numread = read(fd, buf, buflen - 1); if (numread < 1) { + err = numread ? errno : ENODATA; close(fd); - return 0; + return -err; } buf[numread] = '\0'; close(fd); - return (unsigned int) numread; + return 0; } -static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval) +int write_file(const char *path, const char *buf, size_t buflen) { int fd, saved_errno; ssize_t numwritten; if (buflen < 2) - ksft_exit_fail_msg("Incorrect buffer len: %zu\n", buflen); + return -EINVAL; fd = open(path, O_WRONLY); if (fd == -1) - ksft_exit_fail_msg("%s open failed: %s\n", path, strerror(errno)); + return -errno; numwritten = write(fd, buf, buflen - 1); saved_errno = errno; close(fd); - errno = saved_errno; - if (numwritten < 0) { - if (ignore_einval && errno == EINVAL) - return; - ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1), - buf, strerror(errno)); - } - if (numwritten != buflen - 1) - ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", - path, (int)(buflen - 1), buf, buflen - 1, numwritten); + + if (numwritten < 0) + return -saved_errno; + + if (numwritten != (ssize_t)(buflen - 1)) + return -EIO; + + return 0; } -void write_file(const char *path, const char *buf, size_t buflen) -{ - __write_file(path, buf, buflen, /* ignore_einval = */ false); -} - -unsigned long read_num(const char *path) +int read_num(const char *path, unsigned long *num) { + unsigned long val; + int ret; char buf[21]; + char *end; - if (!read_file(path, buf, sizeof(buf))) - ksft_exit_fail_perror("read_file()"); + if (!num) + return -EINVAL; - return strtoul(buf, NULL, 10); + ret = read_file(path, buf, sizeof(buf)); + if (ret) + return ret; + + /* Reject signs and leading whitespace that are accepted by strtoul() */ + if (buf[0] < '0' || buf[0] > '9') + return -EINVAL; + + errno = 0; + val = strtoul(buf, &end, 10); + if (errno) + return -errno; + + /* Only allow a newline after the number */ + if (*end == '\n') + end++; + + if (*end != '\0') + return -EINVAL; + + *num = val; + return 0; } -static void __write_num(const char *path, unsigned long num, bool ignore_einval) +int write_num(const char *path, unsigned long num) { char buf[21]; sprintf(buf, "%lu", num); - __write_file(path, buf, strlen(buf) + 1, ignore_einval); + return write_file(path, buf, strlen(buf) + 1); } -void write_num(const char *path, unsigned long num) +int write_num_ignore_einval(const char *path, unsigned long num) { - return __write_num(path, num, /* ignore_einval = */ false); -} + int ret; -void write_num_ignore_einval(const char *path, unsigned long num) -{ - return __write_num(path, num, /* ignore_einval = */ true); + ret = write_num(path, num); + return ret == -EINVAL ? 0 : ret; } static unsigned long shmall, shmmax; void __shm_limits_restore(void) { - if (shmmax) - write_num("/proc/sys/kernel/shmmax", shmmax); - if (shmall) - write_num("/proc/sys/kernel/shmall", shmall); + int ret; + + if (shmmax) { + ret = write_num("/proc/sys/kernel/shmmax", shmmax); + if (ret < 0) + ksft_exit_fail_msg("Failed to restore shmmax: %s\n", + strerror(-ret)); + } + if (shmall) { + ret = write_num("/proc/sys/kernel/shmall", shmall); + if (ret < 0) + ksft_exit_fail_msg("Failed to restore shmall: %s\n", + strerror(-ret)); + } } void shm_limits_prepare(unsigned long length) { unsigned long nr = length / psize(); unsigned long val; + int ret; + + ret = read_num("/proc/sys/kernel/shmmax", &val); + if (ret < 0) + ksft_exit_fail_msg("Failed to read /proc/sys/kernel/shmmax: %s\n", + strerror(-ret)); - val = read_num("/proc/sys/kernel/shmmax"); if (val < length) { - write_num("/proc/sys/kernel/shmmax", length); + ret = write_num("/proc/sys/kernel/shmmax", length); + if (ret < 0) + ksft_exit_fail_msg("Failed to write %lu to /proc/sys/kernel/shmmax: %s\n", + length, strerror(-ret)); shmmax = val; } - val = read_num("/proc/sys/kernel/shmall"); + ret = read_num("/proc/sys/kernel/shmall", &val); + if (ret < 0) + ksft_exit_fail_msg("Failed to read /proc/sys/kernel/shmall: %s\n", + strerror(-ret)); if (val < nr) { - write_num("/proc/sys/kernel/shmall", nr); + ret = write_num("/proc/sys/kernel/shmall", nr); + if (ret < 0) + ksft_exit_fail_msg("Failed to write %lu to /proc/sys/kernel/shmall: %s\n", + nr, strerror(-ret)); shmall = val; } } diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 9a49af88702e..62f6f5b42649 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -166,11 +166,11 @@ int unpoison_memory(unsigned long pfn); #define PAGEMAP_PRESENT(ent) (((ent) & (1ull << 63)) != 0) #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1)) -void write_file(const char *path, const char *buf, size_t buflen); +int write_file(const char *path, const char *buf, size_t buflen); int read_file(const char *path, char *buf, size_t buflen); -unsigned long read_num(const char *path); -void write_num(const char *path, unsigned long num); -void write_num_ignore_einval(const char *path, unsigned long num); +int read_num(const char *path, unsigned long *num); +int write_num(const char *path, unsigned long num); +int write_num_ignore_einval(const char *path, unsigned long num); void shm_limits_prepare(unsigned long length); void __shm_limits_restore(void); From 1d59dd1383de0061c08117a00969c563aa75bd5e Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Fri, 18 Sep 2026 16:52:30 +0530 Subject: [PATCH 500/539] tools/lib/mm: add shared file helpers Move read_file(), write_file(), read_num(), write_num() and write_num_ignore_einval() out of tools/testing/selftests/mm/vm_util.c into a new shared helper under tools/lib/mm/. These helpers are used by mm selftests today and will also be needed by shared hugepage helpers in subsequent patches. Move them to a generic location so they can be reused outside selftests as well. Keep the helpers exposed to mm selftests through vm_util.h by including the new shared header there, and link the new helper into the selftests/mm build. Update the explicit x86 protection_keys 32-bit and 64-bit build rules to preserve prerequisite paths, now that file_utils.c is built from tools/lib/mm. Add tools/lib/mm/ to the MEMORY MANAGEMENT - MISC entry in MAINTAINERS. Link: https://lore.kernel.org/20260918112234.195857-3-sarthak.sharma@arm.com Signed-off-by: Sarthak Sharma Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Tested-by: Muhammad Usama Anjum Cc: Anshuman Khandual Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Jason Gunthorpe Cc: John Hubbard Cc: Jonathan Corbet Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Mark Brown Cc: Michal Hocko Cc: Nico Pache Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Zi Yan --- MAINTAINERS | 1 + tools/lib/mm/file_utils.c | 106 +++++++++++++++++++++++++++ tools/lib/mm/file_utils.h | 13 ++++ tools/testing/selftests/mm/Makefile | 11 +-- tools/testing/selftests/mm/vm_util.c | 97 ------------------------ tools/testing/selftests/mm/vm_util.h | 7 +- 6 files changed, 127 insertions(+), 108 deletions(-) create mode 100644 tools/lib/mm/file_utils.c create mode 100644 tools/lib/mm/file_utils.h diff --git a/MAINTAINERS b/MAINTAINERS index edad4bf21735..a302a955fcd2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17290,6 +17290,7 @@ F: mm/mapping_dirty_helpers.c F: mm/page_idle.c F: mm/pgalloc-track.h F: mm/process_vm_access.c +F: tools/lib/mm/ F: tools/testing/selftests/mm/ MEMORY MANAGEMENT - NUMA MEMBLOCKS AND NUMA EMULATION diff --git a/tools/lib/mm/file_utils.c b/tools/lib/mm/file_utils.c new file mode 100644 index 000000000000..9b2237e9823e --- /dev/null +++ b/tools/lib/mm/file_utils.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +#include "file_utils.h" + +int read_file(const char *path, char *buf, size_t buflen) +{ + int fd, err; + ssize_t numread; + + fd = open(path, O_RDONLY); + if (fd == -1) + return -errno; + + numread = read(fd, buf, buflen - 1); + if (numread < 1) { + err = numread ? errno : ENODATA; + close(fd); + return -err; + } + + buf[numread] = '\0'; + close(fd); + + return 0; +} + +int write_file(const char *path, const char *buf, size_t buflen) +{ + int fd, saved_errno; + ssize_t numwritten; + + if (buflen < 2) + return -EINVAL; + + fd = open(path, O_WRONLY); + if (fd == -1) + return -errno; + + numwritten = write(fd, buf, buflen - 1); + saved_errno = errno; + close(fd); + + if (numwritten < 0) + return -saved_errno; + + if (numwritten != (ssize_t)(buflen - 1)) + return -EIO; + + return 0; +} + +int read_num(const char *path, unsigned long *num) +{ + unsigned long val; + int ret; + char buf[21]; + char *end; + + if (!num) + return -EINVAL; + + ret = read_file(path, buf, sizeof(buf)); + if (ret) + return ret; + + /* Reject signs and leading whitespace that are accepted by strtoul() */ + if (buf[0] < '0' || buf[0] > '9') + return -EINVAL; + + errno = 0; + val = strtoul(buf, &end, 10); + if (errno) + return -errno; + + /* Only allow a newline after the number */ + if (*end == '\n') + end++; + + if (*end != '\0') + return -EINVAL; + + *num = val; + return 0; +} + +int write_num(const char *path, unsigned long num) +{ + char buf[21]; + + sprintf(buf, "%lu", num); + return write_file(path, buf, strlen(buf) + 1); +} + +int write_num_ignore_einval(const char *path, unsigned long num) +{ + int ret; + + ret = write_num(path, num); + return ret == -EINVAL ? 0 : ret; +} diff --git a/tools/lib/mm/file_utils.h b/tools/lib/mm/file_utils.h new file mode 100644 index 000000000000..50daa82c2b2b --- /dev/null +++ b/tools/lib/mm/file_utils.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __MM_FILE_UTILS_H__ +#define __MM_FILE_UTILS_H__ + +#include + +int read_file(const char *path, char *buf, size_t buflen); +int write_file(const char *path, const char *buf, size_t buflen); +int read_num(const char *path, unsigned long *num); +int write_num(const char *path, unsigned long num); +int write_num_ignore_einval(const char *path, unsigned long num); + +#endif diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index d3e9bd67904a..c36a9a7089bb 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -37,7 +37,8 @@ endif # LDLIBS. MAKEFLAGS += --no-builtin-rules -CFLAGS = -Wall -O2 -I $(top_srcdir) $(EXTRA_CFLAGS) $(KHDR_INCLUDES) $(TOOLS_INCLUDES) +CFLAGS = -Wall -O2 -I $(top_srcdir) -I $(top_srcdir)/tools/lib +CFLAGS += $(EXTRA_CFLAGS) $(KHDR_INCLUDES) $(TOOLS_INCLUDES) CFLAGS += -Wunreachable-code LDLIBS = -lrt -lpthread -lm @@ -184,8 +185,8 @@ TEST_FILES += write_hugetlb_memory.sh include ../lib.mk -$(TEST_GEN_PROGS): vm_util.c hugepage_settings.c -$(TEST_GEN_FILES): vm_util.c hugepage_settings.c +$(TEST_GEN_PROGS): vm_util.c hugepage_settings.c $(top_srcdir)/tools/lib/mm/file_utils.c +$(TEST_GEN_FILES): vm_util.c hugepage_settings.c $(top_srcdir)/tools/lib/mm/file_utils.c $(OUTPUT)/uffd-stress: uffd-common.c $(OUTPUT)/uffd-unit-tests: uffd-common.c @@ -214,7 +215,7 @@ $(BINARIES_32): CFLAGS += -m32 -mxsave $(BINARIES_32): LDLIBS += -lrt -ldl -lm $(BINARIES_32): $(OUTPUT)/%_32: %.c $(call msg,CC,,$@) - $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@ + $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $^ $(LDLIBS) -o $@ $(foreach t,$(VMTARGETS),$(eval $(call gen-target-rule-32,$(t)))) endif @@ -223,7 +224,7 @@ $(BINARIES_64): CFLAGS += -m64 -mxsave $(BINARIES_64): LDLIBS += -lrt -ldl $(BINARIES_64): $(OUTPUT)/%_64: %.c $(call msg,CC,,$@) - $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@ + $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $^ $(LDLIBS) -o $@ $(foreach t,$(VMTARGETS),$(eval $(call gen-target-rule-64,$(t)))) endif diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index d04c904e76c3..f8916b2abc2e 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -885,103 +885,6 @@ int unpoison_memory(unsigned long pfn) return ret > 0 ? 0 : -errno; } -int read_file(const char *path, char *buf, size_t buflen) -{ - int fd, err; - ssize_t numread; - - fd = open(path, O_RDONLY); - if (fd == -1) - return -errno; - - numread = read(fd, buf, buflen - 1); - if (numread < 1) { - err = numread ? errno : ENODATA; - close(fd); - return -err; - } - - buf[numread] = '\0'; - close(fd); - - return 0; -} - -int write_file(const char *path, const char *buf, size_t buflen) -{ - int fd, saved_errno; - ssize_t numwritten; - - if (buflen < 2) - return -EINVAL; - - fd = open(path, O_WRONLY); - if (fd == -1) - return -errno; - - numwritten = write(fd, buf, buflen - 1); - saved_errno = errno; - close(fd); - - if (numwritten < 0) - return -saved_errno; - - if (numwritten != (ssize_t)(buflen - 1)) - return -EIO; - - return 0; -} - -int read_num(const char *path, unsigned long *num) -{ - unsigned long val; - int ret; - char buf[21]; - char *end; - - if (!num) - return -EINVAL; - - ret = read_file(path, buf, sizeof(buf)); - if (ret) - return ret; - - /* Reject signs and leading whitespace that are accepted by strtoul() */ - if (buf[0] < '0' || buf[0] > '9') - return -EINVAL; - - errno = 0; - val = strtoul(buf, &end, 10); - if (errno) - return -errno; - - /* Only allow a newline after the number */ - if (*end == '\n') - end++; - - if (*end != '\0') - return -EINVAL; - - *num = val; - return 0; -} - -int write_num(const char *path, unsigned long num) -{ - char buf[21]; - - sprintf(buf, "%lu", num); - return write_file(path, buf, strlen(buf) + 1); -} - -int write_num_ignore_einval(const char *path, unsigned long num) -{ - int ret; - - ret = write_num(path, num); - return ret == -EINVAL ? 0 : ret; -} - static unsigned long shmall, shmmax; void __shm_limits_restore(void) diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 62f6f5b42649..fe0475f2bdf2 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -8,6 +8,7 @@ #include /* _SC_PAGESIZE */ #include "kselftest.h" #include +#include #define BIT_ULL(nr) (1ULL << (nr)) #define PM_SOFT_DIRTY BIT_ULL(55) @@ -166,12 +167,6 @@ int unpoison_memory(unsigned long pfn); #define PAGEMAP_PRESENT(ent) (((ent) & (1ull << 63)) != 0) #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1)) -int write_file(const char *path, const char *buf, size_t buflen); -int read_file(const char *path, char *buf, size_t buflen); -int read_num(const char *path, unsigned long *num); -int write_num(const char *path, unsigned long num); -int write_num_ignore_einval(const char *path, unsigned long num); - void shm_limits_prepare(unsigned long length); void __shm_limits_restore(void); From 32449ec2cc57bd5dc827d4e94ad9f0456e0db935 Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Fri, 18 Sep 2026 16:52:31 +0530 Subject: [PATCH 501/539] tools/lib/mm: move hugepage_settings out of selftests Move hugepage_settings.[ch] from tools/testing/selftests/mm/ to tools/lib/mm/ so the THP and HugeTLB helpers can be shared more easily between selftests and other tools. Keep the helpers exposed to mm selftests through vm_util.h where possible, and use direct includes for files that do not include vm_util.h. Adjust the selftests/mm build to compile the moved implementation from its new location. Remove the remaining kselftest dependency by including file_utils.h directly and using EXIT_FAILURE in the signal handler. Link: https://lore.kernel.org/20260918112234.195857-4-sarthak.sharma@arm.com Signed-off-by: Sarthak Sharma Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Tested-by: Muhammad Usama Anjum Cc: Anshuman Khandual Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Jason Gunthorpe Cc: John Hubbard Cc: Jonathan Corbet Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Mark Brown Cc: Michal Hocko Cc: Nico Pache Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Zi Yan --- .../{testing/selftests => lib}/mm/hugepage_settings.c | 11 +++++++++-- .../{testing/selftests => lib}/mm/hugepage_settings.h | 0 tools/testing/selftests/mm/Makefile | 6 ++++-- tools/testing/selftests/mm/compaction_test.c | 2 +- tools/testing/selftests/mm/cow.c | 1 - tools/testing/selftests/mm/folio_split_race_test.c | 1 - tools/testing/selftests/mm/guard-regions.c | 1 - tools/testing/selftests/mm/gup_longterm.c | 1 - tools/testing/selftests/mm/gup_test.c | 1 - tools/testing/selftests/mm/hmm-tests.c | 6 +++--- tools/testing/selftests/mm/hugetlb-madvise.c | 1 - tools/testing/selftests/mm/hugetlb-mmap.c | 1 - tools/testing/selftests/mm/hugetlb-mremap.c | 1 - tools/testing/selftests/mm/hugetlb-shm.c | 1 - tools/testing/selftests/mm/hugetlb-soft-offline.c | 2 +- tools/testing/selftests/mm/hugetlb_dio.c | 1 - tools/testing/selftests/mm/hugetlb_fault_after_madv.c | 1 - tools/testing/selftests/mm/hugetlb_madv_vs_map.c | 1 - tools/testing/selftests/mm/khugepaged.c | 1 - tools/testing/selftests/mm/ksm_tests.c | 1 - tools/testing/selftests/mm/migration.c | 2 +- tools/testing/selftests/mm/pagemap_ioctl.c | 1 - tools/testing/selftests/mm/prctl_thp_disable.c | 1 - tools/testing/selftests/mm/protection_keys.c | 2 +- tools/testing/selftests/mm/soft-dirty.c | 1 - tools/testing/selftests/mm/split_huge_page_test.c | 1 - tools/testing/selftests/mm/thuge-gen.c | 1 - tools/testing/selftests/mm/transhuge-stress.c | 1 - tools/testing/selftests/mm/uffd-common.h | 1 - tools/testing/selftests/mm/uffd-wp-mremap.c | 2 +- tools/testing/selftests/mm/va_high_addr_switch.c | 1 - tools/testing/selftests/mm/vm_util.h | 1 + 32 files changed, 22 insertions(+), 34 deletions(-) rename tools/{testing/selftests => lib}/mm/hugepage_settings.c (99%) rename tools/{testing/selftests => lib}/mm/hugepage_settings.h (100%) diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/lib/mm/hugepage_settings.c similarity index 99% rename from tools/testing/selftests/mm/hugepage_settings.c rename to tools/lib/mm/hugepage_settings.c index 6f3abd357385..656442c8d395 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/lib/mm/hugepage_settings.c @@ -10,11 +10,16 @@ #include #include -#include "vm_util.h" +#include "file_utils.h" #include "hugepage_settings.h" #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" #define MAX_SETTINGS_DEPTH 4 + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#endif + static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH]; static int settings_index; static struct thp_settings saved_settings; @@ -655,8 +660,10 @@ static void hugepage_restore_settings_atexit(void) static void hugepage_restore_settings_sighandler(int sig) { + (void)sig; + /* exit() will invoke the hugepage_restore_settings_atexit handler. */ - exit(KSFT_FAIL); + exit(EXIT_FAILURE); } void hugepage_save_settings(bool thp, bool hugetlb) diff --git a/tools/testing/selftests/mm/hugepage_settings.h b/tools/lib/mm/hugepage_settings.h similarity index 100% rename from tools/testing/selftests/mm/hugepage_settings.h rename to tools/lib/mm/hugepage_settings.h diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index c36a9a7089bb..67882e52d4ff 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -185,8 +185,10 @@ TEST_FILES += write_hugetlb_memory.sh include ../lib.mk -$(TEST_GEN_PROGS): vm_util.c hugepage_settings.c $(top_srcdir)/tools/lib/mm/file_utils.c -$(TEST_GEN_FILES): vm_util.c hugepage_settings.c $(top_srcdir)/tools/lib/mm/file_utils.c +$(TEST_GEN_PROGS): vm_util.c $(top_srcdir)/tools/lib/mm/hugepage_settings.c \ + $(top_srcdir)/tools/lib/mm/file_utils.c +$(TEST_GEN_FILES): vm_util.c $(top_srcdir)/tools/lib/mm/hugepage_settings.c \ + $(top_srcdir)/tools/lib/mm/file_utils.c $(OUTPUT)/uffd-stress: uffd-common.c $(OUTPUT)/uffd-unit-tests: uffd-common.c diff --git a/tools/testing/selftests/mm/compaction_test.c b/tools/testing/selftests/mm/compaction_test.c index 30d4ace7155a..b3f5377119cb 100644 --- a/tools/testing/selftests/mm/compaction_test.c +++ b/tools/testing/selftests/mm/compaction_test.c @@ -15,9 +15,9 @@ #include #include #include +#include #include "kselftest.h" -#include "hugepage_settings.h" #define MAP_SIZE_MB 100 #define MAP_SIZE (MAP_SIZE_MB * 1024 * 1024) diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 8aa5249d9bef..3264a828575b 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -29,7 +29,6 @@ #include "../../../../mm/gup_test.h" #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" static size_t pagesize; static int pagemap_fd; diff --git a/tools/testing/selftests/mm/folio_split_race_test.c b/tools/testing/selftests/mm/folio_split_race_test.c index 1960635a953e..e4660bf89b62 100644 --- a/tools/testing/selftests/mm/folio_split_race_test.c +++ b/tools/testing/selftests/mm/folio_split_race_test.c @@ -25,7 +25,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" uint64_t page_size; uint64_t pmd_pagesize; diff --git a/tools/testing/selftests/mm/guard-regions.c b/tools/testing/selftests/mm/guard-regions.c index b724d62d2b75..f7d53ea3c252 100644 --- a/tools/testing/selftests/mm/guard-regions.c +++ b/tools/testing/selftests/mm/guard-regions.c @@ -21,7 +21,6 @@ #include #include #include "vm_util.h" -#include "hugepage_settings.h" #include "../pidfd/pidfd.h" diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index 510de93be681..c9d8b4491263 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -29,7 +29,6 @@ #include "../../../../mm/gup_test.h" #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" static size_t pagesize; static int nr_hugetlbsizes; diff --git a/tools/testing/selftests/mm/gup_test.c b/tools/testing/selftests/mm/gup_test.c index 3f841a96f870..5f44761dbec0 100644 --- a/tools/testing/selftests/mm/gup_test.c +++ b/tools/testing/selftests/mm/gup_test.c @@ -14,7 +14,6 @@ #include #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" #define MB (1UL << 20) diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index e2642eca0d02..fa1a651963fd 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -10,9 +10,6 @@ * bugs. */ -#include "kselftest_harness.h" -#include "hugepage_settings.h" - #include #include #include @@ -33,6 +30,9 @@ #include #include #include +#include + +#include "kselftest_harness.h" /* * This is a private UAPI to the kernel test module so it isn't exported diff --git a/tools/testing/selftests/mm/hugetlb-madvise.c b/tools/testing/selftests/mm/hugetlb-madvise.c index 555b4b3d1430..57cf790ca478 100644 --- a/tools/testing/selftests/mm/hugetlb-madvise.c +++ b/tools/testing/selftests/mm/hugetlb-madvise.c @@ -14,7 +14,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #define MIN_FREE_PAGES 20 #define NR_HUGE_PAGES 10 /* common number of pages to map/allocate */ diff --git a/tools/testing/selftests/mm/hugetlb-mmap.c b/tools/testing/selftests/mm/hugetlb-mmap.c index 2edbc992e6bc..8853271fc825 100644 --- a/tools/testing/selftests/mm/hugetlb-mmap.c +++ b/tools/testing/selftests/mm/hugetlb-mmap.c @@ -18,7 +18,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #define LENGTH (256UL*1024*1024) #define PROTECTION (PROT_READ | PROT_WRITE) diff --git a/tools/testing/selftests/mm/hugetlb-mremap.c b/tools/testing/selftests/mm/hugetlb-mremap.c index ed3d92e862d8..9b724af66e93 100644 --- a/tools/testing/selftests/mm/hugetlb-mremap.c +++ b/tools/testing/selftests/mm/hugetlb-mremap.c @@ -26,7 +26,6 @@ #include #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" #define DEFAULT_LENGTH_MB 10UL #define MB_TO_BYTES(x) (x * 1024 * 1024) diff --git a/tools/testing/selftests/mm/hugetlb-shm.c b/tools/testing/selftests/mm/hugetlb-shm.c index 3ff7f062b7eb..f4514da49e1d 100644 --- a/tools/testing/selftests/mm/hugetlb-shm.c +++ b/tools/testing/selftests/mm/hugetlb-shm.c @@ -29,7 +29,6 @@ #include #include "vm_util.h" -#include "hugepage_settings.h" #define LENGTH (256UL*1024*1024) diff --git a/tools/testing/selftests/mm/hugetlb-soft-offline.c b/tools/testing/selftests/mm/hugetlb-soft-offline.c index ffc85b958c69..d9565219378a 100644 --- a/tools/testing/selftests/mm/hugetlb-soft-offline.c +++ b/tools/testing/selftests/mm/hugetlb-soft-offline.c @@ -22,10 +22,10 @@ #include #include #include +#include #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" #ifndef MADV_SOFT_OFFLINE #define MADV_SOFT_OFFLINE 101 diff --git a/tools/testing/selftests/mm/hugetlb_dio.c b/tools/testing/selftests/mm/hugetlb_dio.c index fb4600570e13..9495974eccbe 100644 --- a/tools/testing/selftests/mm/hugetlb_dio.c +++ b/tools/testing/selftests/mm/hugetlb_dio.c @@ -20,7 +20,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #ifndef STATX_DIOALIGN #define STATX_DIOALIGN 0x00002000U diff --git a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c index 2dc158054f66..56c5a8533e9d 100644 --- a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c +++ b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c @@ -10,7 +10,6 @@ #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #define INLOOP_ITER 100 diff --git a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c index 0f15eff1da04..1d111f42dd59 100644 --- a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c +++ b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c @@ -14,7 +14,6 @@ #include #include "vm_util.h" -#include "hugepage_settings.h" #define INLOOP_ITER 100 diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 6daa22f6da2f..525108cace54 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -22,7 +22,6 @@ #include "linux/magic.h" #include "vm_util.h" -#include "hugepage_settings.h" #define BASE_ADDR ((void *)(1UL << 30)) static unsigned long hpage_pmd_size; diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index 5fd7792a0d47..6711f4c61371 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -15,7 +15,6 @@ #include "kselftest.h" #include #include "vm_util.h" -#include "hugepage_settings.h" #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" #define KSM_FP(s) (KSM_SYSFS_PATH s) diff --git a/tools/testing/selftests/mm/migration.c b/tools/testing/selftests/mm/migration.c index f19d53c69576..a35e2b57e05b 100644 --- a/tools/testing/selftests/mm/migration.c +++ b/tools/testing/selftests/mm/migration.c @@ -5,7 +5,6 @@ */ #include "kselftest_harness.h" -#include "hugepage_settings.h" #include #include @@ -16,6 +15,7 @@ #include #include #include + #include "vm_util.h" #define TWOMEG (2<<20) diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index d9a4fb782ecf..03898b4f6cda 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -24,7 +24,6 @@ #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ PAGE_IS_FILE | PAGE_IS_PRESENT | \ diff --git a/tools/testing/selftests/mm/prctl_thp_disable.c b/tools/testing/selftests/mm/prctl_thp_disable.c index 82c6e96ea6eb..f9ec1408a6e3 100644 --- a/tools/testing/selftests/mm/prctl_thp_disable.c +++ b/tools/testing/selftests/mm/prctl_thp_disable.c @@ -14,7 +14,6 @@ #include #include "kselftest_harness.h" -#include "hugepage_settings.h" #include "vm_util.h" #ifndef PR_THP_DISABLE_EXCEPT_ADVISED diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index ae6e1530b354..b7882ab97683 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -45,8 +45,8 @@ #include #include #include +#include -#include "hugepage_settings.h" #include "pkey-helpers.h" u64 shadow_pkey_reg; diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index 5f278913c4d7..7f649b673355 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -9,7 +9,6 @@ #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" #define PAGEMAP_FILE_PATH "/proc/self/pagemap" #define TEST_ITERATIONS 10000 diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index a30927514b4f..68f508c9a355 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -21,7 +21,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" uint64_t pagesize; unsigned int pageshift; diff --git a/tools/testing/selftests/mm/thuge-gen.c b/tools/testing/selftests/mm/thuge-gen.c index 50d0805b65db..a04f588df780 100644 --- a/tools/testing/selftests/mm/thuge-gen.c +++ b/tools/testing/selftests/mm/thuge-gen.c @@ -14,7 +14,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" #if !defined(MAP_HUGETLB) #define MAP_HUGETLB 0x40000 diff --git a/tools/testing/selftests/mm/transhuge-stress.c b/tools/testing/selftests/mm/transhuge-stress.c index 8eb0c5630e7e..96f72898ebe0 100644 --- a/tools/testing/selftests/mm/transhuge-stress.c +++ b/tools/testing/selftests/mm/transhuge-stress.c @@ -17,7 +17,6 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" int backing_fd = -1; int mmap_flags = MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE; diff --git a/tools/testing/selftests/mm/uffd-common.h b/tools/testing/selftests/mm/uffd-common.h index 92a21b97f745..0723843a7626 100644 --- a/tools/testing/selftests/mm/uffd-common.h +++ b/tools/testing/selftests/mm/uffd-common.h @@ -37,7 +37,6 @@ #include "kselftest.h" #include "vm_util.h" -#include "hugepage_settings.h" #define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY) diff --git a/tools/testing/selftests/mm/uffd-wp-mremap.c b/tools/testing/selftests/mm/uffd-wp-mremap.c index 572c2516e874..c48eaab8e75c 100644 --- a/tools/testing/selftests/mm/uffd-wp-mremap.c +++ b/tools/testing/selftests/mm/uffd-wp-mremap.c @@ -7,8 +7,8 @@ #include #include #include +#include #include "kselftest.h" -#include "hugepage_settings.h" #include "uffd-common.h" static int pagemap_fd; diff --git a/tools/testing/selftests/mm/va_high_addr_switch.c b/tools/testing/selftests/mm/va_high_addr_switch.c index e24d7ba00b44..5a354a664d1f 100644 --- a/tools/testing/selftests/mm/va_high_addr_switch.c +++ b/tools/testing/selftests/mm/va_high_addr_switch.c @@ -11,7 +11,6 @@ #include "vm_util.h" #include "kselftest.h" -#include "hugepage_settings.h" /* * The hint addr value is used to allocate addresses diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index fe0475f2bdf2..64a86e8a0c41 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -9,6 +9,7 @@ #include "kselftest.h" #include #include +#include #define BIT_ULL(nr) (1ULL << (nr)) #define PM_SOFT_DIRTY BIT_ULL(55) From f1e88f2886516eed285eba44d090c93d697cdd5b Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Fri, 18 Sep 2026 16:52:32 +0530 Subject: [PATCH 502/539] tools/mm: move gup_test from selftests/mm to tools/mm Move tools/testing/selftests/mm/gup_test.c to tools/mm/gup_bench.c. This is the first step in separating its benchmarking and functional testing components. Later patches will make this a purely benchmarking tool and introduce a new functional selftest under selftests/mm. Include hugepage_settings.h directly instead of vm_util.h and use getpagesize() instead of psize(). Adjust the Makefiles in both locations and add gup_bench to tools/mm/.gitignore. Remove the gup_test invocations from run_vmtests.sh and update MAINTAINERS. Also remove the gup_test reference from Documentation/core-api/pin_user_pages.rst. The selftest added later in the series is standalone and does not need per command documentation here. Link: https://lore.kernel.org/20260918112234.195857-5-sarthak.sharma@arm.com Signed-off-by: Sarthak Sharma Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Tested-by: Muhammad Usama Anjum Cc: Anshuman Khandual Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Jason Gunthorpe Cc: John Hubbard Cc: Jonathan Corbet Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Mark Brown Cc: Michal Hocko Cc: Nico Pache Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Zi Yan --- Documentation/core-api/pin_user_pages.rst | 9 ----- MAINTAINERS | 2 +- tools/mm/.gitignore | 1 + tools/mm/Makefile | 11 ++++-- .../mm/gup_test.c => mm/gup_bench.c} | 8 ++--- tools/testing/selftests/mm/Makefile | 1 - tools/testing/selftests/mm/run_vmtests.sh | 36 ------------------- 7 files changed, 14 insertions(+), 54 deletions(-) rename tools/{testing/selftests/mm/gup_test.c => mm/gup_bench.c} (97%) diff --git a/Documentation/core-api/pin_user_pages.rst b/Documentation/core-api/pin_user_pages.rst index c16ca163b55e..e0acedbd1d48 100644 --- a/Documentation/core-api/pin_user_pages.rst +++ b/Documentation/core-api/pin_user_pages.rst @@ -226,15 +226,6 @@ will be pinned longterm, and whose data will be accessed. Unit testing ============ -This file:: - - tools/testing/selftests/mm/gup_test.c - -has the following new calls to exercise the new pin*() wrapper functions: - -* PIN_FAST_BENCHMARK (./gup_test -a) -* PIN_BASIC_TEST (./gup_test -b) - You can monitor how many total dma-pinned pages have been acquired and released since the system was booted, via two new /proc/vmstat entries: :: diff --git a/MAINTAINERS b/MAINTAINERS index a302a955fcd2..f67054eed62b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17191,8 +17191,8 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm F: mm/gup.c F: mm/gup_test.c F: mm/gup_test.h +F: tools/mm/gup_bench.c F: tools/testing/selftests/mm/gup_longterm.c -F: tools/testing/selftests/mm/gup_test.c MEMORY MANAGEMENT - KSM (Kernel Samepage Merging) M: Andrew Morton diff --git a/tools/mm/.gitignore b/tools/mm/.gitignore index 1446a659e540..154d740be02e 100644 --- a/tools/mm/.gitignore +++ b/tools/mm/.gitignore @@ -3,3 +3,4 @@ slabinfo page-types page_owner_sort thp_swap_allocator_test +gup_bench diff --git a/tools/mm/Makefile b/tools/mm/Makefile index 858186a6eefd..f20a32d8cc22 100644 --- a/tools/mm/Makefile +++ b/tools/mm/Makefile @@ -3,13 +3,15 @@ # include ../scripts/Makefile.include -BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test +BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter +BUILD_TARGETS += thp_swap_allocator_test gup_bench INSTALL_TARGETS = $(BUILD_TARGETS) thpmaps LIB_DIR = ../lib/api LIBS = $(LIB_DIR)/libapi.a +GUP_BENCH_OBJS = gup_bench.c ../lib/mm/hugepage_settings.c ../lib/mm/file_utils.c -CFLAGS += -Wall -Wextra -I../lib/ -pthread +CFLAGS += -Wall -Wextra -I../lib/ -I../.. -pthread LDFLAGS += $(LIBS) -pthread all: $(BUILD_TARGETS) @@ -22,8 +24,11 @@ $(LIBS): %: %.c $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) +gup_bench: $(GUP_BENCH_OBJS) $(LIBS) + $(CC) $(CFLAGS) -o $@ $(GUP_BENCH_OBJS) $(LDFLAGS) + clean: - $(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test + $(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test gup_bench make -C $(LIB_DIR) clean sbindir ?= /usr/sbin diff --git a/tools/testing/selftests/mm/gup_test.c b/tools/mm/gup_bench.c similarity index 97% rename from tools/testing/selftests/mm/gup_test.c rename to tools/mm/gup_bench.c index 5f44761dbec0..da56aa5324d3 100644 --- a/tools/testing/selftests/mm/gup_test.c +++ b/tools/mm/gup_bench.c @@ -12,8 +12,8 @@ #include #include #include -#include "kselftest.h" -#include "vm_util.h" +#include +#include "../testing/selftests/kselftest.h" #define MB (1UL << 20) @@ -140,7 +140,7 @@ int main(int argc, char **argv) case 'n': nr_pages = atoi(optarg); if (nr_pages < 0) - nr_pages = size / psize(); + nr_pages = size / getpagesize(); break; case 't': thp = 1; @@ -254,7 +254,7 @@ int main(int argc, char **argv) madvise(p, size, MADV_NOHUGEPAGE); /* Fault them in here, from user space. */ - for (; (unsigned long)p < gup.addr + size; p += psize()) + for (; (unsigned long)p < gup.addr + size; p += getpagesize()) p[0] = 0; tid = malloc(sizeof(pthread_t) * nthreads); diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 67882e52d4ff..d6337156111a 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -59,7 +59,6 @@ endif TEST_GEN_FILES = cow TEST_GEN_FILES += compaction_test TEST_GEN_FILES += gup_longterm -TEST_GEN_FILES += gup_test TEST_GEN_FILES += hmm-tests TEST_GEN_FILES += hugetlb-madvise TEST_GEN_FILES += hugetlb-mmap diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index a1db51655702..4281bb6a859e 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -148,30 +148,6 @@ test_selected() { fi } -run_gup_matrix() { - # -t: thp=on, -T: thp=off, -H: hugetlb=on - local hugetlb_mb=256 - - for huge in -t -T "-H -m $hugetlb_mb"; do - # -u: gup-fast, -U: gup-basic, -a: pin-fast, -b: pin-basic, -L: pin-longterm - for test_cmd in -u -U -a -b -L; do - # -w: write=1, -W: write=0 - for write in -w -W; do - # -S: shared - for share in -S " "; do - # -n: How many pages to fetch together? 512 is special - # because it's default thp size (or 2M on x86), 123 to - # just test partial gup when hit a huge in whatever form - for num in "-n 1" "-n 512" "-n 123" "-n -1"; do - CATEGORY="gup_test" run_test ./gup_test \ - $huge $test_cmd $write $share $num - done - done - done - done - done -} - # filter 64bit architectures ARCH64STR="arm64 mips64 parisc64 ppc64 ppc64le riscv64 s390x sparc64 x86_64" if [ -z "$ARCH" ]; then @@ -293,18 +269,6 @@ fi CATEGORY="mmap" run_test ./map_fixed_noreplace -if $RUN_ALL; then - run_gup_matrix -else - # get_user_pages_fast() benchmark - CATEGORY="gup_test" run_test ./gup_test -u -n 1 - CATEGORY="gup_test" run_test ./gup_test -u -n -1 - # pin_user_pages_fast() benchmark - CATEGORY="gup_test" run_test ./gup_test -a -n 1 - CATEGORY="gup_test" run_test ./gup_test -a -n -1 -fi -# Dump pages 0, 19, and 4096, using pin_user_pages: -CATEGORY="gup_test" run_test ./gup_test -ct -F 0x1 0 19 0x1000 CATEGORY="gup_test" run_test ./gup_longterm CATEGORY="userfaultfd" run_test ./uffd-unit-tests From 84821e767653b790496dd2a3fad681902c3d6046 Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Fri, 18 Sep 2026 16:52:33 +0530 Subject: [PATCH 503/539] tools/mm: make gup_bench a benchmark only tool Remove the functional modes (GUP_BASIC_TEST, PIN_BASIC_TEST and DUMP_USER_PAGES_TEST) from gup_bench. Drop the kselftest dependency and use normal diagnostics and exit statuses. When no arguments are supplied, run a single GUP_FAST_BENCHMARK with the existing default values. Let users select other configurations through command-line options. Report ioctl failures and handle errors without relying on assert(). Link: https://lore.kernel.org/20260918112234.195857-6-sarthak.sharma@arm.com Signed-off-by: Sarthak Sharma Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Tested-by: Muhammad Usama Anjum Cc: Anshuman Khandual Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Jason Gunthorpe Cc: John Hubbard Cc: Jonathan Corbet Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Mark Brown Cc: Michal Hocko Cc: Nico Pache Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Zi Yan --- tools/mm/gup_bench.c | 185 ++++++++++++++++++------------------------- 1 file changed, 79 insertions(+), 106 deletions(-) diff --git a/tools/mm/gup_bench.c b/tools/mm/gup_bench.c index da56aa5324d3..ff6e46654681 100644 --- a/tools/mm/gup_bench.c +++ b/tools/mm/gup_bench.c @@ -10,10 +10,10 @@ #include #include #include -#include +#include +#include #include #include -#include "../testing/selftests/kselftest.h" #define MB (1UL << 20) @@ -37,12 +37,6 @@ static char *cmd_to_str(unsigned long cmd) return "PIN_FAST_BENCHMARK"; case PIN_LONGTERM_BENCHMARK: return "PIN_LONGTERM_BENCHMARK"; - case GUP_BASIC_TEST: - return "GUP_BASIC_TEST"; - case PIN_BASIC_TEST: - return "PIN_BASIC_TEST"; - case DUMP_USER_PAGES_TEST: - return "DUMP_USER_PAGES_TEST"; } return "Unknown command"; } @@ -52,39 +46,29 @@ void *gup_thread(void *data) struct gup_test gup = *(struct gup_test *)data; int i, status; - /* Only report timing information on the *_BENCHMARK commands: */ - if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) || - (cmd == PIN_LONGTERM_BENCHMARK)) { - for (i = 0; i < repeats; i++) { - gup.size = size; - status = ioctl(gup_fd, cmd, &gup); - if (status) - break; - - pthread_mutex_lock(&print_mutex); - ksft_print_msg("%s: Time: get:%lld put:%lld us", - cmd_to_str(cmd), gup.get_delta_usec, - gup.put_delta_usec); - if (gup.size != size) - ksft_print_msg(", truncated (size: %lld)", gup.size); - ksft_print_msg("\n"); - pthread_mutex_unlock(&print_mutex); - } - } else { + for (i = 0; i < repeats; i++) { gup.size = size; status = ioctl(gup_fd, cmd, &gup); - if (status) - goto return_; + if (status) { + int err = errno; + + pthread_mutex_lock(&print_mutex); + fprintf(stderr, "%s ioctl failed: %s\n", cmd_to_str(cmd), + strerror(err)); + pthread_mutex_unlock(&print_mutex); + return data; + } pthread_mutex_lock(&print_mutex); - ksft_print_msg("%s: done\n", cmd_to_str(cmd)); + printf("%s: Time: get:%lld put:%lld us", + cmd_to_str(cmd), gup.get_delta_usec, + gup.put_delta_usec); if (gup.size != size) - ksft_print_msg("Truncated (size: %lld)\n", gup.size); + printf(", truncated (size: %lld)", gup.size); + printf("\n"); pthread_mutex_unlock(&print_mutex); } -return_: - ksft_test_result(!status, "ioctl status %d\n", status); return NULL; } @@ -92,38 +76,21 @@ int main(int argc, char **argv) { struct gup_test gup = { 0 }; int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret; - int flags = MAP_PRIVATE; + int flags = MAP_PRIVATE, started_threads = 0, exit_status = 1; char *file = "/dev/zero"; - bool hugetlb = false; + bool hugetlb = false, thread_error = false; + void *thread_result; pthread_t *tid; char *p; - while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) { + while ((opt = getopt(argc, argv, "m:r:n:F:f:aj:tTLuwWSH")) != -1) { switch (opt) { case 'a': cmd = PIN_FAST_BENCHMARK; break; - case 'b': - cmd = PIN_BASIC_TEST; - break; case 'L': cmd = PIN_LONGTERM_BENCHMARK; break; - case 'c': - cmd = DUMP_USER_PAGES_TEST; - /* - * Dump page 0 (index 1). May be overridden later, by - * user's non-option arguments. - * - * .which_pages is zero-based, so that zero can mean "do - * nothing". - */ - gup.which_pages[0] = 1; - break; - case 'p': - /* works only with DUMP_USER_PAGES_TEST */ - gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN; - break; case 'F': /* strtol, so you can pass flags in hex form */ gup.gup_flags = strtol(optarg, 0, 0); @@ -148,9 +115,6 @@ int main(int argc, char **argv) case 'T': thp = 0; break; - case 'U': - cmd = GUP_BASIC_TEST; - break; case 'u': cmd = GUP_FAST_BENCHMARK; break; @@ -172,52 +136,41 @@ int main(int argc, char **argv) hugetlb = true; break; default: - ksft_exit_fail_msg("Wrong argument\n"); + fprintf(stderr, "Wrong argument\n"); + exit(1); } } - if (optind < argc) { - int extra_arg_count = 0; - /* - * For example: - * - * ./gup_test -c 0 1 0x1001 - * - * ...to dump pages 0, 1, and 4097 - */ - - while ((optind < argc) && - (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) { - /* - * Do the 1-based indexing here, so that the user can - * use normal 0-based indexing on the command line. - */ - long page_index = strtol(argv[optind], 0, 0) + 1; - - gup.which_pages[extra_arg_count] = page_index; - extra_arg_count++; - optind++; - } + if (optind != argc) { + fprintf(stderr, "Unexpected argument '%s'\n", argv[optind]); + exit(1); } - ksft_print_header(); + if (geteuid()) { + fprintf(stderr, "Please run this test as root\n"); + exit(1); + } if (hugetlb) { unsigned long hp_size = default_huge_page_size(); - if (!hp_size) - ksft_exit_skip("HugeTLB is unavailable\n"); + if (!hp_size) { + fprintf(stderr, "Could not determine huge page size\n"); + return 1; + } size = (size + hp_size - 1) & ~(hp_size - 1); - if (!hugetlb_setup_default(size / hp_size)) - ksft_exit_skip("Not enough huge pages\n"); + if (!hugetlb_setup_default(size / hp_size)) { + fprintf(stderr, "Not enough huge pages\n"); + return 1; + } } - ksft_set_plan(nthreads); - filed = open(file, O_RDWR|O_CREAT, 0664); - if (filed < 0) - ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno)); + if (filed < 0) { + fprintf(stderr, "Unable to open %s: %s\n", file, strerror(errno)); + return 1; + } gup.nr_pages_per_call = nr_pages; if (write) @@ -226,26 +179,24 @@ int main(int argc, char **argv) gup_fd = open(GUP_TEST_FILE, O_RDWR); if (gup_fd == -1) { switch (errno) { - case EACCES: - if (getuid()) - ksft_print_msg("Please run this test as root\n"); - break; case ENOENT: if (opendir("/sys/kernel/debug") == NULL) - ksft_print_msg("mount debugfs at /sys/kernel/debug\n"); - ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n"); + fprintf(stderr, "mount debugfs at /sys/kernel/debug\n"); + fprintf(stderr, "check if CONFIG_GUP_TEST is enabled in kernel config\n"); break; default: - ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno)); + fprintf(stderr, "failed to open %s: %s\n", GUP_TEST_FILE, + strerror(errno)); break; } - ksft_test_result_skip("Please run this test as root\n"); - ksft_exit_pass(); + goto err_close_filed; } p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0); - if (p == MAP_FAILED) - ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); + if (p == MAP_FAILED) { + fprintf(stderr, "mmap: %s\n", strerror(errno)); + goto err_close_gup_fd; + } gup.addr = (unsigned long)p; if (thp == 1) @@ -258,17 +209,39 @@ int main(int argc, char **argv) p[0] = 0; tid = malloc(sizeof(pthread_t) * nthreads); - assert(tid); + if (!tid) { + fprintf(stderr, "Failed to allocate %d threads: %s\n", + nthreads, strerror(errno)); + goto err_unmap; + } + for (i = 0; i < nthreads; i++) { ret = pthread_create(&tid[i], NULL, gup_thread, &gup); - assert(ret == 0); + if (ret) { + fprintf(stderr, "pthread_create failed: %s\n", strerror(ret)); + thread_error = true; + break; + } + started_threads++; } - for (i = 0; i < nthreads; i++) { - ret = pthread_join(tid[i], NULL); - assert(ret == 0); + for (i = 0; i < started_threads; i++) { + ret = pthread_join(tid[i], &thread_result); + if (ret) { + fprintf(stderr, "pthread_join failed: %s\n", strerror(ret)); + thread_error = true; + } else if (thread_result) + thread_error = true; } free(tid); + if (!thread_error) + exit_status = 0; - ksft_exit_pass(); +err_unmap: + munmap((void *)gup.addr, size); +err_close_gup_fd: + close(gup_fd); +err_close_filed: + close(filed); + return exit_status; } From d2b82fa39fdaea0407000c29ad0d5b7794996d00 Mon Sep 17 00:00:00 2001 From: Sarthak Sharma Date: Fri, 18 Sep 2026 16:52:34 +0530 Subject: [PATCH 504/539] selftests/mm: add a GUP selftest Add a new GUP selftest which uses kselftest_harness.h. Cover 12 mapping configurations: THP enabled, THP disabled and HugeTLB, each across private/shared mappings and with/without FOLL_WRITE. Run 5 test cases for every variant: get_user_pages, get_user_pages_fast, pin_user_pages, pin_user_pages_fast and pin_user_pages_longterm. Use two default hugeTLB pages and derive the mapping size from their size. This exercises GUP both within a single HugeTLB page and across a HugeTLB boundary, without reserving an excessive number of pages. Sweep four nr_pages_per_call values for each test: 1, 512, 123 and all pages. This preserves the coverage previously provided by run_gup_matrix(): 12 mapping combinations x 5 GUP/PUP operations x 4 batch sizes. In total the selftest reports 60 TAP cases and issues 240 ioctls. Do not carry DUMP_USER_PAGES_TEST into the new selftest because its output is written to the kernel log and the selftest does not verify that output. Add the new gup binary to the selftests/mm build, run_vmtests.sh and MAINTAINERS. Update mm/Kconfig to describe the benchmark and selftest split. Link: https://lore.kernel.org/20260918112234.195857-7-sarthak.sharma@arm.com Signed-off-by: Sarthak Sharma Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Tested-by: Muhammad Usama Anjum Cc: Anshuman Khandual Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Jason Gunthorpe Cc: John Hubbard Cc: Jonathan Corbet Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Mark Brown Cc: Michal Hocko Cc: Nico Pache Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Zi Yan --- MAINTAINERS | 1 + mm/Kconfig | 19 +- tools/testing/selftests/mm/Makefile | 1 + tools/testing/selftests/mm/gup.c | 262 ++++++++++++++++++++++ tools/testing/selftests/mm/run_vmtests.sh | 1 + 5 files changed, 272 insertions(+), 12 deletions(-) create mode 100644 tools/testing/selftests/mm/gup.c diff --git a/MAINTAINERS b/MAINTAINERS index f67054eed62b..d2fc3d8364b3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17192,6 +17192,7 @@ F: mm/gup.c F: mm/gup_test.c F: mm/gup_test.h F: tools/mm/gup_bench.c +F: tools/testing/selftests/mm/gup.c F: tools/testing/selftests/mm/gup_longterm.c MEMORY MANAGEMENT - KSM (Kernel Samepage Merging) diff --git a/mm/Kconfig b/mm/Kconfig index 30170a936f1f..edb4a6c0a870 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1292,24 +1292,19 @@ config PERCPU_STATS be used to help understand percpu memory usage. config GUP_TEST - bool "Enable infrastructure for get_user_pages()-related unit tests" + bool "Enable infrastructure for get_user_pages()-related unit tests and benchmarks" depends on DEBUG_FS help Provides /sys/kernel/debug/gup_test, which in turn provides a way - to make ioctl calls that can launch kernel-based unit tests for - the get_user_pages*() and pin_user_pages*() family of API calls. + to make ioctl calls that can launch kernel-based unit tests and + benchmarks for the get_user_pages*() and pin_user_pages*() families + of API calls. - These tests include benchmark testing of the _fast variants of - get_user_pages*() and pin_user_pages*(), as well as smoke tests of + These include benchmark testing of the _fast variants of + get_user_pages*() and pin_user_pages*(), as well as tests of the non-_fast variants. - There is also a sub-test that allows running dump_page() on any - of up to eight pages (selected by command line args) within the - range of user-space addresses. These pages are either pinned via - pin_user_pages*(), or pinned via get_user_pages*(), as specified - by other command line arguments. - - See tools/testing/selftests/mm/gup_test.c + See tools/testing/selftests/mm/gup.c and tools/mm/gup_bench.c. comment "GUP_TEST needs to have DEBUG_FS enabled" depends on !GUP_TEST && !DEBUG_FS diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index d6337156111a..7d69baeb93f4 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -58,6 +58,7 @@ endif TEST_GEN_FILES = cow TEST_GEN_FILES += compaction_test +TEST_GEN_FILES += gup TEST_GEN_FILES += gup_longterm TEST_GEN_FILES += hmm-tests TEST_GEN_FILES += hugetlb-madvise diff --git a/tools/testing/selftests/mm/gup.c b/tools/testing/selftests/mm/gup.c new file mode 100644 index 000000000000..a6a8ca47d12e --- /dev/null +++ b/tools/testing/selftests/mm/gup.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0 +#define __SANE_USERSPACE_TYPES__ // Use ll64 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "vm_util.h" +#include "kselftest_harness.h" + +#define MB (1UL << 20) + +/* Just the flags we need, copied from the kernel internals. */ +#define FOLL_WRITE 0x01 /* check pte is writable */ + +/* Page counts exercising single, THP-batch, partial, and full-mapping GUP. */ +static const int nr_pages_list[] = { 1, 512, 123, -1 }; + +#define GUP_TEST_FILE "/sys/kernel/debug/gup_test" +#define NR_HUGETLB_PAGES 2 + +static unsigned long hp_size; + +FIXTURE(gup_test) +{ + int gup_fd; + char *addr; + unsigned long size; +}; + +FIXTURE_VARIANT(gup_test) +{ + bool thp; + bool hugetlb; + bool write; + bool shared; +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write) +{ + .thp = false, + .hugetlb = false, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read) +{ + .thp = false, + .hugetlb = false, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write_thp) +{ + .thp = true, + .hugetlb = false, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read_thp) +{ + .thp = true, + .hugetlb = false, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write) +{ + .thp = false, + .hugetlb = false, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read) +{ + .thp = false, + .hugetlb = false, + .write = false, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write_thp) +{ + .thp = true, + .hugetlb = false, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read_thp) +{ + .thp = true, + .hugetlb = false, + .write = false, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = false, + .shared = true, +}; + +FIXTURE_SETUP(gup_test) +{ + int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS; + char *p; + + self->size = 128 * MB; + + if (variant->hugetlb) { + if (!hp_size) + SKIP(return, "HugeTLB not available\n"); + + if (hugetlb_free_default_pages() < NR_HUGETLB_PAGES) + SKIP(return, "Not enough huge pages\n"); + + self->size = NR_HUGETLB_PAGES * hp_size; + mmap_flags |= MAP_HUGETLB; + } + + if (variant->shared) + mmap_flags = (mmap_flags & ~MAP_PRIVATE) | MAP_SHARED; + + /* gup_fd has to be >= 0. Already checked in main() */ + self->gup_fd = open(GUP_TEST_FILE, O_RDWR); + ASSERT_GE(self->gup_fd, 0); + + self->addr = mmap(NULL, self->size, PROT_READ | PROT_WRITE, + mmap_flags, -1, 0); + + ASSERT_NE(self->addr, MAP_FAILED) { + int err = errno; + + close(self->gup_fd); + TH_LOG("mmap failed: %s", strerror(err)); + } + + if (variant->thp) + madvise(self->addr, self->size, MADV_HUGEPAGE); + else if (!variant->hugetlb) + madvise(self->addr, self->size, MADV_NOHUGEPAGE); + + for (p = self->addr; (unsigned long)p < (unsigned long)self->addr + + self->size; p += psize()) + p[0] = 0; +} + +FIXTURE_TEARDOWN(gup_test) +{ + munmap(self->addr, self->size); + close(self->gup_fd); +} + +static void run_gup_cmd(struct __test_metadata *_metadata, + FIXTURE_DATA(gup_test) *self, + const FIXTURE_VARIANT(gup_test) *variant, + unsigned long command) +{ + int i; + + for (i = 0; i < (int)ARRAY_SIZE(nr_pages_list); i++) { + struct gup_test gup = { + .addr = (unsigned long)self->addr, + .size = self->size, + .nr_pages_per_call = nr_pages_list[i] < 0 ? + self->size / psize() : nr_pages_list[i], + .gup_flags = variant->write ? FOLL_WRITE : 0, + }; + + TH_LOG("nr_pages_per_call=%u", gup.nr_pages_per_call); + ASSERT_EQ(ioctl(self->gup_fd, command, &gup), 0); + ASSERT_EQ(gup.size, self->size); + } +} + +TEST_F(gup_test, get_user_pages) +{ + run_gup_cmd(_metadata, self, variant, GUP_BASIC_TEST); +} + +TEST_F(gup_test, pin_user_pages) +{ + run_gup_cmd(_metadata, self, variant, PIN_BASIC_TEST); +} + +TEST_F(gup_test, get_user_pages_fast) +{ + run_gup_cmd(_metadata, self, variant, GUP_FAST_BENCHMARK); +} + +TEST_F(gup_test, pin_user_pages_fast) +{ + run_gup_cmd(_metadata, self, variant, PIN_FAST_BENCHMARK); +} + +TEST_F(gup_test, pin_user_pages_longterm) +{ + run_gup_cmd(_metadata, self, variant, PIN_LONGTERM_BENCHMARK); +} + +int main(int argc, char **argv) +{ + const int fd = open(GUP_TEST_FILE, O_RDWR); + + if (fd == -1) { + ksft_print_header(); + if (errno == EACCES) + ksft_exit_skip("Please run this test as root\n"); + if (errno == ENOENT) { + DIR *debugfs = opendir("/sys/kernel/debug"); + + if (!debugfs) + ksft_exit_skip("Mount debugfs at /sys/kernel/debug\n"); + closedir(debugfs); + ksft_exit_skip("Check CONFIG_GUP_TEST in kernel config\n"); + } + ksft_exit_fail_msg("Failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno)); + } + close(fd); + + hp_size = default_huge_page_size(); + if (hp_size) + hugetlb_setup_default(NR_HUGETLB_PAGES); + + return test_harness_run(argc, argv); +} diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 4281bb6a859e..0d8c93087d05 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -269,6 +269,7 @@ fi CATEGORY="mmap" run_test ./map_fixed_noreplace +CATEGORY="gup_test" run_test ./gup CATEGORY="gup_test" run_test ./gup_longterm CATEGORY="userfaultfd" run_test ./uffd-unit-tests From ebd1a08c192109c83a87ddf42793ed6283180faf Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 18 Sep 2026 04:24:37 -0700 Subject: [PATCH 505/539] mm/shmem: report RCU-tasks quiescent states while undoing a range This shows up in the Meta fleet on ftruncate() of large tmpfs files: INFO: rcu_tasks detected stalls on tasks: 00000000752fd185: .. nvcsw: 59455/59455 holdout: 1 idle_cpu: -1/0 task:rocksdb:bottom state:R running task __folio_split find_get_entry find_get_entries truncate_inode_partial_folio shmem_undo_range shmem_setattr notify_change do_ftruncate __x64_sys_ftruncate do_syscall_64 shmem_undo_range() walks the whole of the requested range in folio_batch sized steps, twice, and its two cond_resched() calls are the only reschedule points in that walk. cond_resched() is not an RCU-tasks quiescent state. Use cond_resched_tasks_rcu_qs() at both points so the walk reports an RCU-tasks quiescent state as it proceeds. Link: https://lore.kernel.org/20260918-shmem-tasks-rcu-v1-1-79acf91a2569@debian.org Fixes: 8315f42295d2 ("rcu: Add call_rcu_tasks()") Signed-off-by: Breno Leitao Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Baolin Wang Cc: Hugh Dickins Cc: "Paul E . McKenney" Cc: --- mm/shmem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index f33dbf5af2cb..f2a36a1b5375 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1367,7 +1367,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, } folio_batch_remove_exceptionals(&fbatch); folio_batch_release(&fbatch); - cond_resched(); + cond_resched_tasks_rcu_qs(); } /* @@ -1408,7 +1408,7 @@ whole_folios: index = start; while (index < end) { - cond_resched(); + cond_resched_tasks_rcu_qs(); if (!find_get_entries(mapping, &index, end - 1, &fbatch, indices)) { From 48bd5dc504cca77b3c3d0df0683c9d29460109d5 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Fri, 18 Sep 2026 12:19:28 +0530 Subject: [PATCH 506/539] mm: constify arguments in default pxdp_get() Generic MM default pxdp_get() helpers fetch the values contained in pgtable entries via READ_ONCE() without modifying them. Just make their arguments explicitly 'const' for some additional protection. Link: https://lore.kernel.org/20260918064928.793742-1-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Suggested-by: David Hildenbrand Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: SJ Park Cc: Pedro Falcato --- include/linux/pgtable.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index e3c8ab96941c..780fe849ff8b 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -490,35 +490,35 @@ static inline int pudp_set_access_flags(struct vm_area_struct *vma, #endif #ifndef ptep_get -static inline pte_t ptep_get(pte_t *ptep) +static inline pte_t ptep_get(const pte_t *ptep) { return READ_ONCE(*ptep); } #endif #ifndef pmdp_get -static inline pmd_t pmdp_get(pmd_t *pmdp) +static inline pmd_t pmdp_get(const pmd_t *pmdp) { return READ_ONCE(*pmdp); } #endif #ifndef pudp_get -static inline pud_t pudp_get(pud_t *pudp) +static inline pud_t pudp_get(const pud_t *pudp) { return READ_ONCE(*pudp); } #endif #ifndef p4dp_get -static inline p4d_t p4dp_get(p4d_t *p4dp) +static inline p4d_t p4dp_get(const p4d_t *p4dp) { return READ_ONCE(*p4dp); } #endif #ifndef pgdp_get -static inline pgd_t pgdp_get(pgd_t *pgdp) +static inline pgd_t pgdp_get(const pgd_t *pgdp) { return READ_ONCE(*pgdp); } From c6e5c0b9a493891634d0a413c3485930a28a8903 Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Wed, 16 Sep 2026 15:55:57 +0800 Subject: [PATCH 507/539] mm/alloc_tag: account for reserved tag ids in the kernel tag check The tag ids stored in the page flags include two reserved markers. Id 0 means the page has no tag and id 1 means the tag was cleared, so real tags start at CODETAG_ID_FIRST. The kernel-side check in alloc_tag_sec_init() compared kernel_tags.count alone against the addressable limit, so with the count at or just under the limit the last tag ids wrapped into those markers. Pages allocated through them then look the same as untagged pages on free, nothing is ever subtracted from the real tag and /proc/allocinfo shows that memory as still allocated. Add the missing CODETAG_ID_FIRST, same as tags_addressable(). Link: https://lore.kernel.org/20260916075557.121316-1-hao.ge@linux.dev Fixes: 4835f747d3ed ("alloc_tag: support for page allocation tag compression") Signed-off-by: Hao Ge Signed-off-by: Andrew Morton Acked-by: Suren Baghdasaryan Cc: --- mm/alloc_tag.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c index b33410310477..82e2c3448dcf 100644 --- a/mm/alloc_tag.c +++ b/mm/alloc_tag.c @@ -621,7 +621,7 @@ void __init alloc_tag_sec_init(void) kernel_tags.count = last_codetag - kernel_tags.first_tag; /* Check if kernel tags fit into page flags */ - if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { + if (CODETAG_ID_FIRST + kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); From e4b92891aaca0d37b5973362dd14747a65406650 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 20 Sep 2026 16:50:02 +0100 Subject: [PATCH 508/539] mm/shmem: don't release a swapin-error marker as a swap entry A failed shmem swapin (eg, EIO) frees the swap slot and leaves a PTE_MARKER_POISONED entry in the page cache. On truncate or eviction shmem_free_swap() passes that marker to swap_put_entries_direct(), which warns because it is not a swap entry. There is nothing left to release either way. Skip the release for non-swap entries, as every other caller already does. Link: https://lore.kernel.org/20260920155002.1030454-1-devnexen@gmail.com Fixes: ac2d3268284b ("mm/swapfile.c: remove the unneeded checking") Signed-off-by: David Carlier Signed-off-by: Andrew Morton Reported-by: syzbot+23b25ba3c6bf971f9c57@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=23b25ba3c6bf971f9c57 Reviewed-by: Baolin Wang Cc: Baoquan he Cc: Barry Song Cc: Chis Li (Google) Cc: Hugh Dickens Cc: Kairui Song Cc: Kemeng Shi Cc: Nhat Pham Cc: --- mm/shmem.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index f2a36a1b5375..07b2855dfb7b 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1184,6 +1184,7 @@ static long shmem_free_swap(struct address_space *mapping, pgoff_t index, pgoff_t end, void *radswap) { XA_STATE(xas, &mapping->i_pages, index); + const softleaf_t swp = radix_to_swp_entry(radswap); unsigned int nr_pages = 0; pgoff_t base; void *entry; @@ -1200,8 +1201,9 @@ static long shmem_free_swap(struct address_space *mapping, } xas_unlock_irq(&xas); - if (nr_pages) - swap_put_entries_direct(radix_to_swp_entry(radswap), nr_pages); + /* A swapin-error marker holds no swap slot, so just drop it. */ + if (nr_pages && softleaf_is_swap(swp)) + swap_put_entries_direct(swp, nr_pages); return nr_pages; } From cbb0045eb8d7069c422180e66e15cea7268d9785 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Sat, 19 Sep 2026 13:43:11 +0300 Subject: [PATCH 509/539] docs/mm: describe set_memory() and set_direct_map() APIs The set_memory() and set_direct_map() APIs change permissions of existing kernel mappings, but their semantics are only described by the code, and that code differs from architecture to architecture. Add Documentation/mm/kernel-page-tables.rst that briefly describes what the kernel page tables consist of, defines the semantics both APIs have in common, including the parts that are easy to get wrong, and lists the differences between the architecture implementations. Add kernel-doc comments for the generic set_memory() and set_direct_map() stubs and link them into Documentation/core-api/mm-api.rst. [rppt@kernel.org: fix phrasing, per Kevin] Link: https://lore.kernel.org/arJNn2QD_pY6e14R@kernel.org Link: https://lore.kernel.org/20260919-set-memory-docs-v2-1-a2a4b3657690@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Reviewed-by: Kevin Brodsky Assisted-by: copilot:claude-opus Cc: "David Hildenbrand (arm)" Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Randy Dunlap Cc: Shuah khan Cc: Suren Baghdasaryan Cc: "Vlastimil Babka (SUSE)" --- Documentation/core-api/mm-api.rst | 9 + Documentation/mm/index.rst | 1 + Documentation/mm/kernel-page-tables.rst | 410 ++++++++++++++++++++++++ include/linux/set_memory.h | 110 +++++++ 4 files changed, 530 insertions(+) create mode 100644 Documentation/mm/kernel-page-tables.rst diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index c1d03a5a2a19..e135b61bc87d 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -52,6 +52,15 @@ Virtually Contiguous Mappings .. kernel-doc:: mm/vmalloc.c :export: +Kernel Page Table Permissions +============================= + +.. kernel-doc:: include/linux/set_memory.h + :doc: Kernel page table permissions + +.. kernel-doc:: include/linux/set_memory.h + :internal: + File Mapping and Page Cache =========================== diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 13a79f5d092c..e9ae5cd82163 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -25,6 +25,7 @@ see the :doc:`admin guide <../admin-guide/mm/index>`. physical_memory page_tables + kernel-page-tables process_addrs bootmem page_allocation diff --git a/Documentation/mm/kernel-page-tables.rst b/Documentation/mm/kernel-page-tables.rst new file mode 100644 index 000000000000..c16de34b4f79 --- /dev/null +++ b/Documentation/mm/kernel-page-tables.rst @@ -0,0 +1,410 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================== +Kernel Page Tables +================== + +Introduction +============ + +The kernel page tables are created early during boot and, unlike the page +tables of user processes, most of them remain static throughout the system +lifetime. + +Every architecture has a direct map (also called linear map) that maps the +physical memory at a fixed offset, so that a physical address can be +translated to a kernel virtual address with simple arithmetic. On most +architectures the direct map is a part of the kernel page tables, with a few +exceptions described in the `Direct map`_ section below. + +On 32-bit systems with high memory the direct map covers only a part of the +physical memory, see Documentation/mm/highmem.rst. + +The vmalloc area, present on every architecture with an MMU, is used for +allocations of virtually contiguous memory whose backing pages are not +necessarily physically contiguous, and for mapping of the device memory. Its +page tables are created and torn down at runtime, see +Documentation/mm/vmalloc.rst. + +Architectures that use the `SPARSEMEM_VMEMMAP` memory model reserve a range of +kernel address space for the memory map, so that `struct page` objects appear +as a virtually contiguous array indexed by the page frame number, see +Documentation/mm/memory-model.rst. + +Besides these, the kernel image may be mapped in a dedicated part of the +kernel address space rather than accessed through the direct map. In that +case its mapping is an alias of the direct map of the physical memory the +image occupies. + +The rest of the kernel address space is architecture specific. For instance, +x86 has a region for the EFI runtime services and s390 has a region for the +code that has to run in the 31-bit addressing mode. + +Direct map +========== + +On most architectures the direct map is an ordinary part of the kernel page +tables. It is created early during boot with the largest pages the hardware +and the kernel configuration allow. + +Several architectures are different. + +MIPS +---- + +MIPS does not map the physical memory with page tables at all. Instead, a +part of the kernel virtual address space is a window into the physical +address space: the hardware translates the addresses that fall into that +window by a fixed transformation of the address bits, without walking the +page tables and without using the TLB. The memory attributes, such as +cacheability and the privilege level required to access the memory, are a +property of the window rather than of an individual page. + +There are no page table entries describing the direct map, so its properties +cannot be changed for an individual page. On 32-bit systems the window covers +only 512 MiB of the physical memory, so everything above that is high memory. + +LoongArch +--------- + +Like MIPS, LoongArch maps the physical memory with a hardware window. The +window covers 256 TiB of the physical address space on 64-bit and 512 MiB on +32-bit systems, and everything above that is high memory. + +The window occupies the lower part of the kernel address space. The upper +part, which includes the vmalloc area, is mapped with kernel page tables. + +PowerPC with the hash MMU +------------------------- + +On 64-bit PowerPC systems with the hash MMU the direct map does not exist in +the Linux page tables. It is installed into the hardware hash page table early +during boot. + +Modifying such mappings requires updating the hash page table directly, and +the hash MMU code implements this only for the kernel image permissions, +`debug_pagealloc` and KFENCE. + +With the radix MMU the direct map is a part of the ordinary kernel page +tables. + +Modifying the kernel page tables +================================ + +Except for the vmalloc area, the kernel page tables are mostly static. Still, +there are cases when the permissions of existing kernel mappings have to be +updated, for instance when a module is loaded and its text becomes read-only +and executable, or when a page is temporarily removed from the direct map to +reduce its exposure. + +There are two families of functions for this, both declared in +`include/linux/set_memory.h`: + +* `set_memory_*()` change permissions of an arbitrary kernel mapping. They + take a kernel virtual address and the number of pages. + +* `set_direct_map_*()` change permissions of the direct mapping for the range + of page frames starting at the page represented by a `struct page`. They take + a `struct page` pointer and the number of pages. + +Architectures that implement `set_memory()` select `CONFIG_ARCH_HAS_SET_MEMORY` + +Architectures that implement `set_direct_map()` select +`CONFIG_ARCH_HAS_SET_DIRECT_MAP`. + +Common semantics +---------------- + +Ranges +~~~~~~ + +The `set_memory()` functions expect a range described by a start address and a +number of pages. The address must be page aligned and the entire range must be +covered by page table entries the architecture knows how to update. + +The entries may be marked as not present, set_memory_p() and set_memory_valid() +exist exactly to bring such a mapping back. + +When a range does not qualify, an architecture will usually say so by returning +an error and sometimes by a WARN()ing as well, unless it prefers to keep it to +itself and return success, see `Architecture specific differences`_. + +The `set_direct_map()` functions expect a range described by the first +`struct page` and a number of pages, and they update the direct map starting +at that page. + +The pages that follow the first one are updated regardless of what they are, so +the caller has to make sure that the range does not extend beyond the memory it +owns. + +Some architectures cannot split a large mapping, and they reject a range that +is a part of one, see `Architecture specific differences`_. + +Calling `set_memory()` with the number of pages set to zero is a no-op that +returns success, except on arm64, where doing nothing to the wrong address is +still an error. + +Aliases +~~~~~~~ + +A physical page may be mapped several times, for instance in the direct map +and in the vmalloc area, and the permissions of these mappings may differ. + +Whether the direct map alias is updated by a `set_memory()` call, and +which permission bits make it there, is entirely up to the architecture, and +there is not much agreement between them, see +`Architecture specific differences`_. + +Relying on that is a gamble; code that needs the direct map alias to change +should say so with the `set_direct_map()` APIs. + +Failures and partial updates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Both families of functions return 0 on success and a negative error code on +failure. The most common failures are `-EINVAL` for a range that cannot be +handled and `-ENOMEM` when splitting a large mapping fails to allocate a page +table. + +Some of the range checks happen upfront, so their failure leaves the page +tables unchanged. + +**The update is not atomic and there is no rollback.** + +The architecture implementations walk the range and update the page tables as +they go, and they stop at the first entry that cannot be updated. When an error +is returned, an arbitrary prefix of the range may have been updated already, +and the same is true for the direct map alias when the architecture updates it. + +For example, when a range spans two large mappings and splitting the second +one fails because there is no memory for a page table, the first one is +already split and updated by the time the error is returned. + +None of the APIs inform the caller where in the range they failed, so reverting +such a partial update is possible in principle but unreliable in practice. + +The caller may try to restore the original permissions over the entire range, +but that revert goes through the very code that has just failed, which does +not inspire much confidence. + +The callers should therefore be prepared to give up on the memory in question: +leak it or panic, but never return it to the allocator before the permissions +are restored and never assume that the requested permissions are in effect. +Neither option is appealing, but both beat handing out a page whose +permissions nobody knows. + +TLB flushing +~~~~~~~~~~~~ + +The `set_memory()` functions flush the TLB for the affected range before +they return, so that the new permissions are in effect for every CPU. + +The `set_direct_map()` functions have `_noflush` in their names because when +they were first introduced on x86, the intention was that the TLB flushing +could be optimized by letting the caller handle it. + +For example, vfree() batches the TLB flushes for the areas allocated with +`VM_FLUSH_RESET_PERMS`, folding the flush of the direct map into the flush it +has to do for the vmalloc mapping anyway. + +Some architectures flush the TLB in the `_noflush` functions anyway, so the +name is best read as a suggestion. It does not make the flush by the caller +unnecessary, it only makes it more expensive. + +A caller that changes the permissions to more restrictive ones must flush the +TLB itself. + +Context +~~~~~~~ + +Architectures use different locking mechanisms to synchronize kernel page table +updates, and both families of functions may sleep, for instance when they +allocate memory to split a large mapping. + +The caller cannot presume it is safe to call these APIs from an atomic context. + +The `set_direct_map()` functions must not be called for high memory pages, +which have no direct map alias to update. + +Unimplemented APIs silently succeed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When an architecture does not implement these APIs, the generic stubs in +`include/linux/set_memory.h` return 0, that is, they report success for work +they have no intention of doing. + +The same happens inside several architecture implementations. The arm64 +`set_direct_map()` functions return 0 when can_set_direct_map() is false, and +the LoongArch `set_memory()` functions return 0 for the addresses in its +windowed direct mapping, which is not backed by page tables at all. + +Returning success without actually updating the page tables is a deliberate +trade-off that keeps the callers free of `#ifdef`\ s, but they have to realize: + +**a return value of 0 does not imply that the permissions were actually +changed.** + +For best-effort hardening that is good enough. When correctness or security +depends on the permissions, the caller has to make sure the architecture +really implements what it needs. For instance, secretmem depends on +`CONFIG_ARCH_HAS_SET_DIRECT_MAP` and calls can_set_direct_map() at runtime. + +Architecture specific differences +================================= + +The APIs are implemented by seven architectures and, beyond the common +semantics described above, their behaviour differs in several respects. + +Which of the APIs are implemented: + +========= ===================== ========================= +Arch `ARCH_HAS_SET_MEMORY` `ARCH_HAS_SET_DIRECT_MAP` +========= ===================== ========================= +arm yes no +arm64 yes yes +loongarch yes yes +powerpc yes no +riscv yes (MMU only) yes (MMU only) +s390 yes yes +x86 yes yes +========= ===================== ========================= + +Only set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() are +available everywhere, and even these are not universal: some architectures +restrict the ranges that can be modified, for instance arm64 rejects direct map +addresses and accepts only addresses in vmalloc space. The architectures that +may run on hardware without an execute permission bit, like x86 and s390, +silently skip the update of the executable bit. + +set_memory_rox() has a generic implementation that calls set_memory_ro() and +set_memory_x() in turn; PowerPC, s390 and x86 override it with a single-pass +version. + +Several architectures define additional APIs. Some of those may have identical +semantics but different names. For example, making a mapping present or not +present is spelled differently: set_memory_p() and set_memory_np() on x86 and +PowerPC, set_memory_valid() on arm and arm64. + +The direct map and the kernel image are normally mapped with the largest +possible pages, and changing the permissions of a single page inside such a +mapping requires splitting it, which not every architecture can do. + +arm +--- + +* Does not implement `set_direct_map()`. +* Provides set_memory_valid(). +* set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() accept + only vmalloc and module addresses. +* set_memory_valid() accepts any address. +* Does not update mapping aliases. + +arm64 +----- + +* Provides set_memory_valid(). +* Provides the memory encryption helpers, which are effective only when the + kernel runs as a confidential guest. +* set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() accept + only vmalloc and module addresses: + + - the range must fit in the VM area that contains its start + - the VM area must have `VM_ALLOC` set and `VM_ALLOW_HUGE_VMAP` clear + +* set_memory_valid() accepts any address. +* The encryption helpers accept only direct map addresses. +* The `set_memory()` functions propagate the read-only and the read-write + changes to the direct map alias when `rodata=full` is in effect. +* Splits leaf mappings before the update on the hardware that supports it. + The split itself may be partial when it fails midway, but the permissions are + left untouched in that case. + Without support for splitting large mappings, a range that covers a leaf + entry only partially fails with a WARN()ing and `-EINVAL`. + If a range spans one or more full leaf entries and a partial leaf entry, the + permissions of the full leaf entries are updated before the failure. +* The `set_direct_map()` functions return 0 without doing anything when the + direct map cannot be modified, see can_set_direct_map(). +* Skips the TLB flush in `set_memory()` when the update only turns an invalid + mapping into a valid one. +* Does not flush TLB in `set_direct_map()`. + +LoongArch +--------- + +* Accepts only the addresses above the hardware window and silently returns + success for the rest, see `Direct map`_. +* Does not update mapping aliases. +* Does not split anything: a leaf entry is updated as a whole, which changes + the permissions of the entire large mapping. +* Flushes the TLB in `set_direct_map()`. + +PowerPC +------- + +* Does not implement `set_direct_map()`. +* Provides set_memory_np() and set_memory_p(). +* Rejects huge vmalloc mappings. +* With the hash MMU on 64-bit systems accepts nothing but the vmalloc and the + I/O regions. +* With the radix MMU accepts direct map addresses, but still cannot split a + large mapping. +* Does not update mapping aliases. + +riscv +----- + +* Implements both APIs only when the MMU is enabled. +* Provides set_memory_rw_nx(). +* The `set_memory()` functions accept any mapped kernel address, including the + direct map, but a vmalloc range must have the `pages` array of its VM area + populated, which rules out vmap() and ioremap() mappings. +* On 64-bit systems the `set_memory()` functions update the direct map alias of + a vmalloc range, including the executable bit. +* Does not split vmalloc ranges: a leaf entry is updated as a whole, which + changes the permissions of the entire large mapping. +* Splits the direct map on 64-bit systems. +* Flushes the TLB in `set_direct_map()`. + +s390 +---- + +* Provides set_memory_4k(), set_memory_rwnx() and the + `__set_memory_*(start, end)` variants that take a range rather than a page + count. +* The `set_memory()` functions accept any mapped kernel address, including the + direct map. +* Skips the update of the executable bit when the hardware has no support for + it. +* The `set_memory()` functions propagate only the read-only and read-write + changes to the direct map alias of a `VM_ALLOC` area, and deliberately not + the executable bit. +* Splits leaf PUD and PMD entries when the range is not aligned to them or when + set_memory_4k() is requested. +* Updates the page table entries with instructions that invalidate the + corresponding TLB entries, so no separate flush is needed anywhere. + +x86 +--- + +* Provides the largest set of operations on top of the common ones: + + - the cache attribute helpers: set_memory_uc(), set_memory_wc(), + set_memory_wb() + - presence control: set_memory_np() and set_memory_p() + - set_memory_4k() + - set_memory_global() and set_memory_nonglobal() + - the array variants that operate on `struct page` arrays or arrays of + virtual addresses + - memory encryption: set_memory_encrypted() and set_memory_decrypted() + +* The `set_memory()` functions accept any mapped kernel address, including the + direct map, and silently succeed for the unmapped holes inside it. +* Does nothing in set_memory_x() and set_memory_nx() when the CPU has no + execute permission bit. +* The `set_memory()` functions apply the change to the direct map alias and, + for the kernel image, to the high kernel mapping. The NX bit is never + propagated, so that the direct map stays non-executable. +* Splits large mappings on demand and can collapse them back when the + permissions become uniform again. +* Does not flush the TLB in `set_direct_map()`, but splitting a large + mapping flushes it anyway. diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 3fe293cfed8c..27c32fbabfec 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -5,16 +5,92 @@ #ifndef _LINUX_SET_MEMORY_H_ #define _LINUX_SET_MEMORY_H_ +/** + * DOC: Kernel page table permissions + * + * The set_memory() and set_direct_map() APIs update permissions of existing + * kernel mappings. + * + * The set_memory() functions operate on a range of kernel virtual addresses, + * the set_direct_map() functions operate on the direct map. + * + * The updates are not atomic: when a call fails, an arbitrary prefix of the + * range may have been updated already and there is no automatic rollback. + * A caller must restore the required permissions before reusing or freeing + * the memory. + * + * When an architecture does not implement these APIs they succeed without + * doing anything, so a return value of 0 does not mean that the permissions + * were actually changed. + * + * Callers that depend on the permissions being applied must ensure that the + * architecture supports the required operation for the target addresses. The + * Kconfig symbols alone do not guarantee this. + * + * See Documentation/mm/kernel-page-tables.rst for the details and for the + * differences between the architecture implementations. + */ + #ifdef CONFIG_ARCH_HAS_SET_MEMORY #include #else +/** + * set_memory_ro - make a kernel mapping read-only + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_ro(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_rw - make a kernel mapping writable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_rw(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_x - make a kernel mapping executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_x(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_nx - make a kernel mapping non-executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_nx(unsigned long addr, int numpages) { return 0; } #endif #ifndef set_memory_rox +/** + * set_memory_rox - make a kernel mapping read-only and executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * A failure may leave the range read-only but not executable. + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int set_memory_rox(unsigned long addr, int numpages) { int ret = set_memory_ro(addr, numpages); @@ -25,11 +101,33 @@ static inline int set_memory_rox(unsigned long addr, int numpages) #endif #ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP +/** + * set_direct_map_invalid_noflush - remove pages from the direct map + * @page: first page to update + * @nr: number of pages to update + * + * Makes the direct mapping of @nr pages starting at @page not present. + * The caller is responsible for any required TLB flushing. + * + * Return: 0 on success, negative error code on failure. + */ static inline int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { return 0; } + +/** + * set_direct_map_default_noflush - restore the direct map of pages + * @page: first page to update + * @nr: number of pages to update + * + * Restores the default kernel permissions of the direct mapping of @nr + * pages starting at @page. + * The caller is responsible for any required TLB flushing. + * + * Return: 0 on success, negative error code on failure. + */ static inline int set_direct_map_default_noflush(struct page *page, unsigned int nr) { @@ -46,6 +144,18 @@ static inline bool kernel_page_present(struct page *page) * boot time. Let them overrive this query. */ #ifndef can_set_direct_map +/** + * can_set_direct_map - check if the direct map can be modified + * + * Available with CONFIG_ARCH_HAS_SET_DIRECT_MAP. Architectures may override + * this to report whether direct map updates are enabled at runtime. + * Even though the generic implementation returns true this does not guarantee + * that every address can be updated. + * + * See Documentation/mm/kernel-page-tables.rst for the details + * + * Return: true unless the architecture reports direct map updates disabled. + */ static inline bool can_set_direct_map(void) { return true; From 59e24520b8f13ca6a42efb297c3403cd5b56c0b8 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:31 +0100 Subject: [PATCH 510/539] selftests/mm: raise the khugepaged test-case cap Patch series "selftests/mm: improve khugepaged coverage", v6. khugepaged collapses to mTHP orders since 7.2, and 7.3 added three selftest cases for it: the generic collapse cases run at one order named by -c, with the result detected by counting folios of that order. That leaves the collapse path largely untested. The suite does not run where a PMD is 512M. A folio count cannot say where a collapse landed. Fixed sleeps cannot tell "not collapsed" from "not scanned yet". And nothing exercises collapse under contention. Close those gaps in order: - Make the suite run at a 512M PMD: scale the collapse wait with the PMD size, skip what such a PMD cannot serve, make the swapout the swap cases depend on deterministic, and keep khugepaged out of the MADV_COLLAPSE cases. - Detect results per window rather than by count, with folio-order helpers in vm_util that are checked against the kernel before any collapse test trusts them. - Drive khugepaged deterministically: a completion barrier that wakes the daemon and waits for a full pass, and a check that one pass yields one attributed collapse. - Cover collapse at every supported order by default: which window collapses, occupancy at both limits, sources that are already large folios, and a fork-shared source under concurrent writes. - Race collapse against everything that can touch its sources, at both occupancy limits and over whole-table zaps, checked by content and by the kernel's own assertions. Everything passes on an unmodified kernel. This patch (of 19): TEST() ends the run with "MAX_TEST_CASES is too small" when the table fills, and the table holds 64. A full invocation already registers 63, so the next case added anywhere aborts the whole suite before a single test runs. Raise the cap to 256. The table is a static array of small structs, so the room costs nothing worth counting. Link: https://lore.kernel.org/20260919002451.496763-1-kirill@shutemov.name Link: https://lore.kernel.org/20260919002451.496763-2-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Acked-by: Usama Arif Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Baolin Wang Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Michal Hocko Cc: Muhammad Usama Anjum Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 525108cace54..c44bc18f7536 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -1290,7 +1290,7 @@ struct test_case { test_fn fn; }; -#define MAX_TEST_CASES 64 +#define MAX_TEST_CASES 256 static struct test_case test_cases[MAX_TEST_CASES]; static int nr_test_cases; From b36bec6a01b1e1d8d5827ad3237a0bd1d4e2cca2 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:32 +0100 Subject: [PATCH 511/539] selftests/mm: skip collapse_compound_extreme() where the PMD is too large collapse_compound_extreme() builds a PTE table full of distinct PTE-mapped compound pages by cycling hpage_pmd_nr fault-time THPs through mremap. It therefore needs hpage_pmd_nr PMD-order allocations in a row. That is fine at a 2M PMD (4K base pages) or a 32M one (16K). A 512M PMD -- arm64 with 64K base pages -- makes each of those an order-13 allocation, which the allocator cannot reliably hand out even once, let alone 8192 times. The failure is not a quiet one: the case calls ksft_exit_fail_msg(), so the whole binary stops and every case after it is lost. Skip the case where the PMD is larger than 32M. The MADV_COLLAPSE cases still cover PMD-order collapse on those configurations, and 4K and 16K PMDs are unaffected. Link: https://lore.kernel.org/20260919002451.496763-3-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Baolin Wang Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Michal Hocko Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index c44bc18f7536..8a6d708026b7 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -940,6 +940,16 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops void *p; int i; + /* + * This needs hpage_pmd_nr PMD-order allocations in a row, which the + * allocator will not supply if the PMD is very large. + */ + if (hpage_pmd_size > (32UL << 20)) { + ksft_test_result_skip("%s: PMD too large for fault-time THP construction\n", + __func__); + return; + } + p = ops->setup_area(1); ksft_print_msg("Construct PTE page table full of different PTE-mapped compound pages\n"); for (i = 0; i < hpage_pmd_nr; i++) { From e8e83ce6f4a271f733e0bca6440070aed8056a49 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:33 +0100 Subject: [PATCH 512/539] selftests/mm: scale khugepaged's collapse wait with the PMD size wait_for_scan() gives every case the same three seconds, whatever the huge page costs to build. collapse_full() asks for four of them: 8M at a 2M PMD, but 2G at a 512M PMD -- arm64 with 64K base pages. Three seconds is thin at that size, and the case has reported a failure for a collapse that was still going. The timeout is a ceiling on a poll loop, not a sleep: the loop stops as soon as ops->check_huge() sees the collapse, or as soon as full_scans has advanced by two. Raising it costs a passing case nothing. Across 80 runs of collapse_full() on arm64 with 64K pages the wait was half a second in 73 of them, with a tail to two seconds. Keep three seconds as the floor and add a second per 128M collapsed. A 2M PMD is unchanged, so x86-64 is too; a 512M PMD gets 19 seconds. On arm64 with 64K pages a passing ./khugepaged all:anon takes 49 seconds under TCG before and after this change. Link: https://lore.kernel.org/20260919002451.496763-4-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Baolin Wang Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Michal Hocko Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 8a6d708026b7..189cc4fee18f 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -561,8 +561,11 @@ static bool wait_for_scan(const char *msg, char *p, size_t len, int nr_hpages, int collap_order, struct mem_ops *ops) { unsigned long hpage_size = page_size << collap_order; - int full_scans; - int timeout = 6; /* 3 seconds */ + unsigned long bytes = (unsigned long)nr_hpages * hpage_size; + int timeout, full_scans; + + /* Half-second ticks: three seconds floor, plus a second per 128M */ + timeout = 6 + 2 * (bytes / (128UL << 20)); /* Sanity check */ if (!ops->check_huge(p, len, 0, hpage_size)) From ceffb8ff832ed5ea7d81b5ec7794399bccb5b955 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:34 +0100 Subject: [PATCH 513/539] selftests/mm: skip khugepaged page cache cases without a PMD folio The page cache caps folio order at MAX_PAGECACHE_ORDER, which is below the PMD order on arm64 with 64K pages, where a PMD is 512M. A PMD-sized page cache folio is impossible there, so the kernel refuses these collapses: MADV_COLLAPSE answers -EINVAL and khugepaged passes over the range. Four shmem cases ask for a PMD-sized folio anyway, fail, and the run bails out in the middle. Skip the shmem and file mem types where the cap is below the PMD order. The cap is not shmem-specific: it applies to every file folio. Add thp_file_supported_orders() to read the orders the page cache allows. Anonymous collapse is unaffected: its orders are not capped this way. Link: https://lore.kernel.org/20260919002451.496763-5-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Baolin Wang Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/lib/mm/hugepage_settings.h | 9 +++++++++ tools/testing/selftests/mm/khugepaged.c | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tools/lib/mm/hugepage_settings.h b/tools/lib/mm/hugepage_settings.h index 548e9d288d1d..94d9fc747f49 100644 --- a/tools/lib/mm/hugepage_settings.h +++ b/tools/lib/mm/hugepage_settings.h @@ -87,6 +87,15 @@ void thp_set_read_ahead_path(char *path); unsigned long thp_supported_orders(void); unsigned long thp_shmem_supported_orders(void); +/* + * The per-order shmem_enabled attribute is created for the orders the page + * cache can hold, not just for shmem, so it answers for regular files too. + */ +static inline unsigned long thp_file_supported_orders(void) +{ + return thp_shmem_supported_orders(); +} + bool thp_available(void); bool thp_is_enabled(void); diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 189cc4fee18f..5318f3cfc0d0 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -1358,6 +1358,25 @@ int main(int argc, char **argv) setbuf(stdout, NULL); + /* + * Without a PMD-order page cache folio the kernel refuses these + * collapses, so there is nothing to test. + */ + if (!(thp_file_supported_orders() & (1UL << hpage_pmd_order))) { + if (shmem_ops) { + ksft_print_msg("no PMD-order page cache folio: skipping shmem\n"); + shmem_ops = NULL; + } + if (read_only_file_ops) { + ksft_print_msg("no PMD-order page cache folio: skipping file\n"); + read_only_file_ops = NULL; + read_write_file_read_ops = NULL; + read_write_file_write_ops = NULL; + } + if (!anon_ops && !shmem_ops && !read_only_file_ops) + ksft_exit_skip("No mem_type left to run\n"); + } + default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1; default_settings.khugepaged.max_ptes_swap = hpage_pmd_nr / 8; default_settings.khugepaged.max_ptes_shared = hpage_pmd_nr / 2; From 1648a3366defe50730ba73941508c2fd92140d33 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:35 +0100 Subject: [PATCH 514/539] selftests/mm: make the swap cases' swapout reliable collapse_swapin_single_pte() and collapse_max_ptes_swap() swap a range out and then require smaps to report exactly the count they asked for. Two things keep that count from arriving. MADV_PAGEOUT is best effort, so the count often turns up a moment late. And wait_for_scan() leaves the range eligible for collapsing, so khugepaged is still working on it. Collapsing reads the swapped-out pages back in, so the daemon empties the swap as fast as the case fills it. On arm64 with 64K pages max_ptes_swap is 1024 pages, which is 64M a step, and the case loses the race: # Swapout 1024 of 8192 pages... Fail not ok 10 collapse_max_ptes_swap Retry for up to two seconds, holding the range out of khugepaged's reach meanwhile. The collapse each case runs next restores MADV_HUGEPAGE, so only the setup is affected. If the pages still won't swap out, skip: no swap, swap too small or full, a memcg cap or busy writeback. None of that is a kernel bug. Link: https://lore.kernel.org/20260919002451.496763-6-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Reviewed-by: Muhammad Usama Anjum Reviewed-by: Baolin Wang Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 41 +++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 5318f3cfc0d0..13a2a47ab110 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -221,6 +221,29 @@ err_out: return swap; } +static bool swapout_range(void *p, unsigned long size) +{ + int i; + + /* keep khugepaged from collapsing the range and swapping it back in */ + if (madvise(p, size, MADV_NOHUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_NOHUGEPAGE)"); + + /* + * Retry several times because MADV_PAGEOUT is best effort. Sleep + * between the retries to give outstanding writeback a chance to + * finish. + */ + for (i = 0; i < 40; i++) { + if (madvise(p, size, MADV_PAGEOUT)) + ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); + if (check_swap(p, size)) + return true; + usleep(50 * 1000); + } + return false; +} + static void *alloc_mapping(int nr) { void *p; @@ -828,12 +851,10 @@ static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_op ops->fault(p, 0, hpage_pmd_size); ksft_print_msg("Swapout one page..."); - if (madvise(p, page_size, MADV_PAGEOUT)) - ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); - if (check_swap(p, page_size)) { + if (swapout_range(p, page_size)) { success("OK"); } else { - fail("Fail"); + skip("Could not swap out"); goto out; } @@ -854,12 +875,10 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o ops->fault(p, 0, hpage_pmd_size); ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); - if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) - ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); - if (check_swap(p, (max_ptes_swap + 1) * page_size)) { + if (swapout_range(p, (max_ptes_swap + 1) * page_size)) { success("OK"); } else { - fail("Fail"); + skip("Could not swap out"); goto out; } @@ -871,12 +890,10 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o ops->fault(p, 0, hpage_pmd_size); ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); - if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) - ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); - if (check_swap(p, max_ptes_swap * page_size)) { + if (swapout_range(p, max_ptes_swap * page_size)) { success("OK"); } else { - fail("Fail"); + skip("Could not swap out"); goto out; } From 7af29a132253ee847d140ee35d79f9e556e821f2 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:36 +0100 Subject: [PATCH 515/539] selftests/mm: stop khugepaged during the MADV_COLLAPSE cases __madvise_collapse() turns THP off before each MADV_COLLAPSE, both to keep khugepaged out of the range and to prove MADV_COLLAPSE ignores the setting. It clears the global controls only, which is no longer enough. A per-order control overrides them, and -s, which makes the cases fault in folios of one order, leaves that order's control at "always". khugepaged then collapses the very range the case is working on, and the case fails on a collapse that was interfered with rather than refused. Clear the per-order controls too. MADV_COLLAPSE does not consult them: anon never did, and shmem stopped with "mm: shmem: ignore sysfs configs for shmem forced collapse". Link: https://lore.kernel.org/20260919002451.496763-7-kirill@shutemov.name Fixes: b7f16963efe7 ("mm/khugepaged: run khugepaged for all orders") Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Muhammad Usama Anjum Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 13a2a47ab110..e013eebc7136 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -538,8 +538,8 @@ static bool is_anon(struct mem_ops *ops) static void __madvise_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { - int ret; struct thp_settings settings = *thp_current_settings(); + int ret, i; ksft_print_msg("%s...", msg); @@ -555,6 +555,10 @@ static void __madvise_collapse(const char *msg, char *p, int nr_hpages, */ settings.thp_enabled = THP_NEVER; settings.shmem_enabled = SHMEM_NEVER; + for (i = 0; i < NR_ORDERS; i++) { + settings.hugepages[i].enabled = THP_NEVER; + settings.shmem_hugepages[i].enabled = SHMEM_NEVER; + } thp_push_settings(&settings); /* Clear VM_NOHUGEPAGE */ From 22206a92859e8f002c6d2a64ff375f2de2e2b0f9 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:37 +0100 Subject: [PATCH 516/539] selftests/mm: move is_backed_by_folio() into vm_util Checking that an address range is backed by a folio of a given order is useful to any test that builds or collapses large folios. mTHP collapse coverage in the khugepaged selftest needs exactly that. split_huge_page_test.c already has the building block: is_backed_by_folio() reads the compound head and tail flags from /proc/kpageflags to classify the folio behind a page. Move it into vm_util so other tests can use it. No functional change. Link: https://lore.kernel.org/20260919002451.496763-8-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Acked-by: Mike Rapoport (Microsoft) Acked-by: Lorenzo Stoakes (ARM) Reviewed-by: Baolin Wang Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Michal Hocko Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- .../selftests/mm/split_huge_page_test.c | 61 ------------------- tools/testing/selftests/mm/vm_util.c | 61 +++++++++++++++++++ tools/testing/selftests/mm/vm_util.h | 2 + 3 files changed, 63 insertions(+), 61 deletions(-) diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 68f508c9a355..c5d96a4b1db3 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -41,67 +41,6 @@ const char *kpageflags_proc = "/proc/kpageflags"; int pagemap_fd; int kpageflags_fd; -static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, - int kpageflags_fd) -{ - const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD; - const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL; - const unsigned long nr_pages = 1UL << order; - unsigned long pfn_head; - uint64_t pfn_flags; - unsigned long pfn; - unsigned long i; - - pfn = pagemap_get_pfn(pagemap_fd, vaddr); - - /* non present page */ - if (pfn == -1UL) - return false; - - if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) - goto fail; - - /* check for order-0 pages */ - if (!order) { - if (pfn_flags & (folio_head_flags | folio_tail_flags)) - return false; - return true; - } - - /* non THP folio */ - if (!(pfn_flags & KPF_THP)) - return false; - - pfn_head = pfn & ~(nr_pages - 1); - - if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) - goto fail; - - /* head PFN has no compound_head flag set */ - if ((pfn_flags & folio_head_flags) != folio_head_flags) - return false; - - /* check all tail PFN flags */ - for (i = 1; i < nr_pages; i++) { - if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) - goto fail; - if ((pfn_flags & folio_tail_flags) != folio_tail_flags) - return false; - } - - /* - * check the PFN after this folio, but if its flags cannot be obtained, - * assume this folio has the expected order - */ - if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) - return true; - - /* If we find another tail page, then the folio is larger. */ - return (pfn_flags & folio_tail_flags) != folio_tail_flags; -fail: - ksft_exit_fail_msg("Failed to get folio info\n"); -} - static int check_after_split_folio_orders(char *vaddr_start, size_t len, int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) { diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index f8916b2abc2e..d2c5a20d724a 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -490,6 +490,67 @@ int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags) return 0; } +bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, + int kpageflags_fd) +{ + const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD; + const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL; + const unsigned long nr_pages = 1UL << order; + unsigned long pfn_head; + uint64_t pfn_flags; + unsigned long pfn; + unsigned long i; + + pfn = pagemap_get_pfn(pagemap_fd, vaddr); + + /* non present page */ + if (pfn == -1UL) + return false; + + if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) + goto fail; + + /* check for order-0 pages */ + if (!order) { + if (pfn_flags & (folio_head_flags | folio_tail_flags)) + return false; + return true; + } + + /* non THP folio */ + if (!(pfn_flags & KPF_THP)) + return false; + + pfn_head = pfn & ~(nr_pages - 1); + + if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) + goto fail; + + /* head PFN has no compound_head flag set */ + if ((pfn_flags & folio_head_flags) != folio_head_flags) + return false; + + /* check all tail PFN flags */ + for (i = 1; i < nr_pages; i++) { + if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) + goto fail; + if ((pfn_flags & folio_tail_flags) != folio_tail_flags) + return false; + } + + /* + * check the PFN after this folio, but if its flags cannot be obtained, + * assume this folio has the expected order + */ + if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) + return true; + + /* If we find another tail page, then the folio is larger. */ + return (pfn_flags & folio_tail_flags) != folio_tail_flags; +fail: + ksft_exit_fail_msg("Failed to get folio info\n"); +} + /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor, uint64_t *ioctls) diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 64a86e8a0c41..f12979a70135 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -99,6 +99,8 @@ int64_t allocate_transhuge(void *ptr, int pagemap_fd); int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags); int gather_folio_orders(char *vaddr_start, size_t len, int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders); +bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, + int kpageflags_fd); int uffd_register(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor); From 1649257f27ab7239937109fa10a1efd16c3ee374 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:38 +0100 Subject: [PATCH 517/539] selftests/mm: add folio-order check for address ranges An mTHP collapse test needs to know that a range is backed by folios of the target order, and that they sit where a collapse would put them. Nothing answers both: is_backed_by_folio() classifies the folio behind a single page, and check_huge_anon() counts the folios of an order in a range without saying where they start. Add is_range_backed_by_order(). It requires every folio-sized, folio- aligned part of the range to map one folio of that order, head to tail, with the head at the start of the part. A part backed by two smaller folios fails, and so does a folio mapped off its natural alignment. The mTHP cases need both to tell a collapsed range from the one beside it. Link: https://lore.kernel.org/20260919002451.496763-9-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport (Microsoft) Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Baolin Wang Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/vm_util.c | 47 ++++++++++++++++++++++++++++ tools/testing/selftests/mm/vm_util.h | 2 ++ 2 files changed, 49 insertions(+) diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index d2c5a20d724a..1f88330fd027 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -551,6 +551,53 @@ fail: ksft_exit_fail_msg("Failed to get folio info\n"); } +/** + * is_range_backed_by_order() - check that a range is backed by @order folios + * @start: start of the range, a multiple of the folio size + * @len: length of the range in bytes, a multiple of the folio size + * @order: the folio order to check for + * @pagemap_fd: open /proc//pagemap of the range's owner + * @kpageflags_fd: open /proc/kpageflags + * + * Every folio-sized, folio-aligned part of the range must map one folio of + * @order, head to tail, with the head at the start of the part. A part + * backed by several smaller folios fails, and so does a folio mapped off + * its natural alignment. + * + * Returns: true if the whole range is backed that way, false otherwise. + */ +bool is_range_backed_by_order(char *start, size_t len, int order, + int pagemap_fd, int kpageflags_fd) +{ + const unsigned long nr_pages = 1UL << order; + const size_t folio_size = nr_pages * psize(); + char *vaddr; + + if ((uintptr_t)start % folio_size || len % folio_size) + return false; + + for (vaddr = start; vaddr < start + len; vaddr += folio_size) { + const unsigned long pfn = pagemap_get_pfn(pagemap_fd, vaddr); + unsigned long i; + + /* Not present, or a tail page */ + if (pfn == -1UL || pfn % nr_pages) + return false; + + for (i = 1; i < nr_pages; i++) { + char *page = vaddr + i * psize(); + + if (pagemap_get_pfn(pagemap_fd, page) != pfn + i) + return false; + } + + if (!is_backed_by_folio(vaddr, order, pagemap_fd, kpageflags_fd)) + return false; + } + + return true; +} + /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor, uint64_t *ioctls) diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index f12979a70135..0172003c16dc 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -101,6 +101,8 @@ int gather_folio_orders(char *vaddr_start, size_t len, int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders); bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, int kpageflags_fd); +bool is_range_backed_by_order(char *start, size_t len, int order, + int pagemap_fd, int kpageflags_fd); int uffd_register(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor); From 9912e3d928a738eafe528a5de631ec1931053030 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:39 +0100 Subject: [PATCH 518/539] selftests/mm: add folio-order detection self-check The khugepaged mTHP tests detect collapse results with the vm_util folio-order helpers rather than smaps AnonHugePages, which only sees PMD mappings. If those helpers are wrong, every case built on them is wrong the same way, and nothing says so. Check them directly. For every anon THP order the kernel supports, fault memory in with only that order enabled. Require the helpers to classify the backing as exactly that order: not the order below it, and base-page memory as order 0. Run it in the thp category, ahead of ./khugepaged, so a broken helper is reported as itself rather than as a collapse failure. Verified on x86-64 4K (orders 0, 2-9) and arm64 64K (orders 0, 2-13). The test needs ALIGN(), which hmm-tests.c and migration.c each defined privately. Move it to vm_util.h and drop both copies. Link: https://lore.kernel.org/20260919002451.496763-10-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Baolin Wang Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/Makefile | 1 + .../testing/selftests/mm/folio_order_check.c | 122 ++++++++++++++++++ tools/testing/selftests/mm/hmm-tests.c | 1 - tools/testing/selftests/mm/migration.c | 1 - tools/testing/selftests/mm/run_vmtests.sh | 2 + tools/testing/selftests/mm/vm_util.h | 2 + 6 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/mm/folio_order_check.c diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 7d69baeb93f4..cb32cf5d867e 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -105,6 +105,7 @@ TEST_GEN_FILES += guard-regions TEST_GEN_FILES += merge TEST_GEN_FILES += rmap TEST_GEN_FILES += folio_split_race_test +TEST_GEN_FILES += folio_order_check TEST_GEN_FILES += soft-dirty ifeq ($(ARCH),x86_64) diff --git a/tools/testing/selftests/mm/folio_order_check.c b/tools/testing/selftests/mm/folio_order_check.c new file mode 100644 index 000000000000..5eafbcc1b4f3 --- /dev/null +++ b/tools/testing/selftests/mm/folio_order_check.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Self-check for the vm_util folio-order helpers, is_backed_by_folio() and + * is_range_backed_by_order(), which the khugepaged mTHP cases use to detect + * collapse results. For every anon THP order the kernel supports, fault + * memory in with only that order enabled and require the helpers to report + * exactly that order. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include "kselftest.h" +#include "vm_util.h" +#include + +static int pagemap_fd; +static int kpageflags_fd; + +static char *alloc_aligned(size_t size) +{ + size_t len = size * 2; + char *p, *aligned; + + p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (p == MAP_FAILED) + ksft_exit_fail_perror("mmap()"); + + aligned = (char *)ALIGN((uintptr_t)p, size); + if (aligned != p) + munmap(p, aligned - p); + if (aligned + size != p + len) + munmap(aligned + size, p + len - aligned - size); + + return aligned; +} + +static void check_order(int order) +{ + struct thp_settings settings = *thp_current_settings(); + size_t size = psize() << order; + bool ok = true; + char *p; + int i; + + for (i = 0; i < NR_ORDERS; i++) + settings.hugepages[i].enabled = THP_NEVER; + if (order) + settings.hugepages[order].enabled = THP_ALWAYS; + thp_push_settings(&settings); + + p = alloc_aligned(size); + *p = 1; + + if (!is_range_backed_by_order(p, size, order, pagemap_fd, kpageflags_fd)) { + ksft_print_msg("order %d not detected after fault\n", order); + ok = false; + } + + /* A lower order must be rejected: the folio is larger */ + if (order && is_range_backed_by_order(p, size, order - 1, + pagemap_fd, kpageflags_fd)) { + ksft_print_msg("order %d also reported as order %d\n", + order, order - 1); + ok = false; + } + + /* A large folio must not pass as order 0 */ + if (order && is_range_backed_by_order(p, size, 0, + pagemap_fd, kpageflags_fd)) { + ksft_print_msg("order %d also reported as order 0\n", order); + ok = false; + } + + munmap(p, size); + thp_pop_settings(); + + ksft_test_result(ok, "order %d classified\n", order); +} + +int main(void) +{ + struct thp_settings settings; + unsigned long orders; + int order; + + ksft_print_header(); + + if (!thp_available()) + ksft_exit_skip("Transparent Hugepages not available\n"); + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_perror("open(/proc/self/pagemap)"); + kpageflags_fd = open("/proc/kpageflags", O_RDONLY); + if (kpageflags_fd < 0) + ksft_exit_skip("open(/proc/kpageflags) requires root\n"); + + orders = thp_supported_orders(); + if (!orders) + ksft_exit_skip("No supported THP orders\n"); + + ksft_set_plan(__builtin_popcountl(orders) + 1); + + thp_save_settings(); + thp_read_settings(&settings); + /* Base of the settings stack; the bottom entry is never popped */ + thp_push_settings(&settings); + + check_order(0); + for (order = 1; order < NR_ORDERS; order++) { + if (!(orders & (1UL << order))) + continue; + check_order(order); + } + + ksft_finished(); +} diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index fa1a651963fd..e5f273ca84c1 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -65,7 +65,6 @@ enum { #define HMM_PATH_MAX 64 #define NTIMES 10 -#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) /* Just the flags we need, copied from mm.h: */ #ifndef FOLL_WRITE diff --git a/tools/testing/selftests/mm/migration.c b/tools/testing/selftests/mm/migration.c index a35e2b57e05b..d1d0989ed2ca 100644 --- a/tools/testing/selftests/mm/migration.c +++ b/tools/testing/selftests/mm/migration.c @@ -20,7 +20,6 @@ #define TWOMEG (2<<20) #define RUNTIME (20) -#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) HUGETLB_SETUP_DEFAULT_PAGES(1) diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 0d8c93087d05..6b80cf2eab14 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -382,6 +382,8 @@ CATEGORY="pfnmap" run_test ./pfnmap # COW tests CATEGORY="cow" run_test ./cow +CATEGORY="thp" run_test ./folio_order_check + CATEGORY="thp" run_test ./khugepaged CATEGORY="thp" run_test ./khugepaged -s 2 diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 0172003c16dc..ea48e6a7527e 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -12,6 +12,8 @@ #include #define BIT_ULL(nr) (1ULL << (nr)) +#define ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) + #define PM_SOFT_DIRTY BIT_ULL(55) #define PM_MMAP_EXCLUSIVE BIT_ULL(56) #define PM_UFFD_WP BIT_ULL(57) From b79d51e92e5f480b0d91ca272b4081493563699a Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:40 +0100 Subject: [PATCH 519/539] selftests/mm: add khugepaged completion barrier helper A khugepaged test has to tell "not collapsed" from "not scanned yet", and nothing in the selftests can. wait_for_scan() in khugepaged.c comes closest: it polls full_scans until the counter has advanced by two, since the pass in progress may already have passed the test's mm. But it only returns in time if scan_sleep_millisecs happens to be short, and it is private to that one test. Add khugepaged_full_pass() to hugepage_settings, built on the same advance-by-two wait but driven through sysfs: a store to scan_sleep_millisecs wakes the daemon, so the barrier completes whatever the scan cadence. A store made while the daemon is scanning rather than sleeping is lost, so the helper keeps storing until the pass lands. One wake completes one pass only if pages_to_scan covers every mm on the list, so callers need it large. Settings pushes must not start passes of their own. A store to either sleep knob wakes the daemon, so thp_write_settings() now writes a khugepaged knob only when its value changes. Link: https://lore.kernel.org/20260919002451.496763-11-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/lib/mm/hugepage_settings.c | 60 +++++++++++++++++++++++++++----- tools/lib/mm/hugepage_settings.h | 2 ++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/tools/lib/mm/hugepage_settings.c b/tools/lib/mm/hugepage_settings.c index 656442c8d395..77918677a9cd 100644 --- a/tools/lib/mm/hugepage_settings.c +++ b/tools/lib/mm/hugepage_settings.c @@ -217,6 +217,13 @@ void thp_read_settings(struct thp_settings *settings) } } +/* A store to either sleep knob wakes khugepaged, so write only on change */ +static void thp_update_num(const char *name, unsigned long num) +{ + if (thp_read_num(name) != num) + thp_write_num(name, num); +} + void thp_write_settings(struct thp_settings *settings) { struct khugepaged_settings *khugepaged = &settings->khugepaged; @@ -232,15 +239,15 @@ void thp_write_settings(struct thp_settings *settings) shmem_enabled_strings[settings->shmem_enabled]); thp_write_num("use_zero_page", settings->use_zero_page); - thp_write_num("khugepaged/defrag", khugepaged->defrag); - thp_write_num("khugepaged/alloc_sleep_millisecs", - khugepaged->alloc_sleep_millisecs); - thp_write_num("khugepaged/scan_sleep_millisecs", - khugepaged->scan_sleep_millisecs); - thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none); - thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap); - thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); - thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); + thp_update_num("khugepaged/defrag", khugepaged->defrag); + thp_update_num("khugepaged/alloc_sleep_millisecs", + khugepaged->alloc_sleep_millisecs); + thp_update_num("khugepaged/scan_sleep_millisecs", + khugepaged->scan_sleep_millisecs); + thp_update_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none); + thp_update_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap); + thp_update_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); + thp_update_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); if (dev_queue_read_ahead_path[0]) { int ret = write_num(dev_queue_read_ahead_path, @@ -271,6 +278,41 @@ void thp_write_settings(struct thp_settings *settings) } } +/* + * Wait for a full khugepaged scan pass that started after this call: the + * pass in progress may already have passed this mm, so full_scans has to + * advance twice. + * + * A store to scan_sleep_millisecs wakes the daemon, but one made while it + * is scanning rather than sleeping is lost, so keep storing until the pass + * lands. + * + * One wake is one pass only if pages_to_scan covers every mm on the list. + */ +bool khugepaged_full_pass(unsigned int timeout_s) +{ + unsigned long deadline_ms = timeout_s * 1000UL; + unsigned long elapsed_ms = 0, poll_ms = 10; + unsigned long sleep_ms; + int pass; + + sleep_ms = thp_read_num("khugepaged/scan_sleep_millisecs"); + for (pass = 0; pass < 2; pass++) { + unsigned long target = + thp_read_num("khugepaged/full_scans") + 1; + + while (thp_read_num("khugepaged/full_scans") < target) { + if (elapsed_ms >= deadline_ms) + return false; + thp_write_num("khugepaged/scan_sleep_millisecs", + sleep_ms); + usleep(poll_ms * 1000); + elapsed_ms += poll_ms; + } + } + return true; +} + struct thp_settings *thp_current_settings(void) { if (!settings_index) { diff --git a/tools/lib/mm/hugepage_settings.h b/tools/lib/mm/hugepage_settings.h index 94d9fc747f49..8f4581099b7a 100644 --- a/tools/lib/mm/hugepage_settings.h +++ b/tools/lib/mm/hugepage_settings.h @@ -83,6 +83,8 @@ static inline void thp_save_settings(void) hugepage_save_settings(/* thp = */ true, /* hugetlb = */ false); } +bool khugepaged_full_pass(unsigned int timeout_s); + void thp_set_read_ahead_path(char *path); unsigned long thp_supported_orders(void); unsigned long thp_shmem_supported_orders(void); From 6e9ed3d8fdd4325ea8367d44e160eeea021d6a36 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:41 +0100 Subject: [PATCH 520/539] selftests/mm: add order-parameterized khugepaged collapse cases The mthp_khugepaged context runs the generic cases at a sub-PMD order, which answers how many folios of that order a range ends up with. It cannot say which order-sized window they landed in, so "the populated window collapsed" and "the empty window next to it collapsed instead" look alike. Add four cases that check each window on its own, with the folio-order helpers in vm_util: - collapse_order_single_window(): only the populated window collapses; - collapse_order_partial_window(): the default max_ptes_none lets a window with one present PTE collapse; - collapse_order_max_ptes_none(): with max_ptes_none=0 a full window collapses and one missing a page does not; - collapse_order_mixed_sources(): sources that are already large folios of a smaller order collapse to the target. Each case faults its region before MADV_HUGEPAGE with only the target order enabled, so the sources are order 0 and the result can only come from khugepaged. They wait for a full pass rather than for the result to appear: without a completed pass, "not collapsed" and "not scanned yet" are the same thing. Link: https://lore.kernel.org/20260919002451.496763-12-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Tested-by: Muhammad Usama Anjum Tested-by: Baolin Wang Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 212 ++++++++++++++++++++++++ 1 file changed, 212 insertions(+) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index e013eebc7136..51bda01446cd 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -30,6 +30,8 @@ static unsigned long page_size; static int hpage_pmd_nr; static int anon_order; static int collapse_order; +static int pagemap_fd = -1; +static int kpageflags_fd = -1; #define PID_SMAPS "/proc/self/smaps" #define TEST_FILE "collapse_test_file" @@ -1209,6 +1211,198 @@ static void madvise_retracted_page_tables(struct collapse_context *c, ksft_test_result_report(exit_status, "%s\n", __func__); } +/* Smallest order khugepaged will consider for mTHP collapse */ +#define MIN_MTHP_ORDER 2 + +/* Time budget for one khugepaged pass in the collapse_order_* cases */ +#define MTHP_PASS_TIMEOUT_S 30 + +static size_t mthp_window_size(void) +{ + return page_size << collapse_order; +} + +static void mthp_push_target_order(void) +{ + struct thp_settings settings = *thp_current_settings(); + int i; + + /* + * Only the target order, and only for madvise: the cases fault their + * region first, so the sources stay order 0 whatever -s asked for. + */ + settings.thp_enabled = THP_NEVER; + for (i = 0; i < NR_ORDERS; i++) + settings.hugepages[i].enabled = THP_NEVER; + settings.hugepages[collapse_order].enabled = THP_MADVISE; + thp_push_settings(&settings); +} + +static bool all_windows_at_order(void *p, size_t len) +{ + return is_range_backed_by_order(p, len, collapse_order, + pagemap_fd, kpageflags_fd); +} + +static bool any_window_at_order(void *p, size_t len) +{ + size_t window = mthp_window_size(); + char *addr = p; + + for (; len >= window; addr += window, len -= window) { + if (all_windows_at_order(addr, window)) + return true; + } + return false; +} + +static void collapse_order_single_window(struct collapse_context *c, + struct mem_ops *ops) +{ + size_t window = mthp_window_size(); + void *p; + + mthp_push_target_order(); + + p = ops->setup_area(1); + ops->fault(p, window, 2 * window); + if (any_window_at_order(p, hpage_pmd_size)) + ksft_exit_fail_msg("Unexpected large folio after fault\n"); + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + ksft_print_msg("Collapse one fully populated window..."); + if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) + fail("Timeout"); + else if (all_windows_at_order(p + window, window) && + !any_window_at_order(p, window) && + !any_window_at_order(p + 2 * window, + hpage_pmd_size - 2 * window)) + success("OK"); + else + fail("Fail"); + + validate_memory(p, window, 2 * window); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + +static void collapse_order_partial_window(struct collapse_context *c, + struct mem_ops *ops) +{ + void *p; + + mthp_push_target_order(); + + p = ops->setup_area(1); + ops->fault(p, 0, page_size); + if (any_window_at_order(p, hpage_pmd_size)) + ksft_exit_fail_msg("Unexpected large folio after fault\n"); + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + ksft_print_msg("Collapse window with single PTE entry present..."); + if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) + fail("Timeout"); + else if (all_windows_at_order(p, mthp_window_size())) + success("OK"); + else + fail("Fail"); + + validate_memory(p, 0, page_size); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + +static void collapse_order_max_ptes_none(struct collapse_context *c, + struct mem_ops *ops) +{ + struct thp_settings settings; + size_t window = mthp_window_size(); + void *p; + + mthp_push_target_order(); + settings = *thp_current_settings(); + settings.khugepaged.max_ptes_none = 0; + thp_push_settings(&settings); + + p = ops->setup_area(1); + ops->fault(p, 0, 2 * window - page_size); + if (any_window_at_order(p, hpage_pmd_size)) + ksft_exit_fail_msg("Unexpected large folio after fault\n"); + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + ksft_print_msg("Collapse full window, not the one missing a page..."); + if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) + fail("Timeout"); + else if (all_windows_at_order(p, window) && + !any_window_at_order(p + window, window)) + success("OK"); + else + fail("Fail"); + + validate_memory(p, 0, 2 * window - page_size); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + +static void collapse_order_mixed_sources(struct collapse_context *c, + struct mem_ops *ops) +{ + struct thp_settings settings; + void *p; + + if (collapse_order <= MIN_MTHP_ORDER) { + ksft_test_result_skip("%s: no source order below target\n", + __func__); + return; + } + + mthp_push_target_order(); + + settings = *thp_current_settings(); + settings.hugepages[MIN_MTHP_ORDER].enabled = THP_ALWAYS; + thp_push_settings(&settings); + p = ops->setup_area(1); + ops->fault(p, 0, hpage_pmd_size); + thp_pop_settings(); + + /* + * The allocator can fall back to smaller folios under fragmentation; + * having nothing to collapse from is not a failure. + */ + if (!is_range_backed_by_order(p, hpage_pmd_size, MIN_MTHP_ORDER, + pagemap_fd, kpageflags_fd)) { + ksft_print_msg("No order-%d sources to collapse...", + MIN_MTHP_ORDER); + skip("Skip"); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); + return; + } + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + ksft_print_msg("Collapse region backed by smaller large folios..."); + if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) + fail("Timeout"); + else if (all_windows_at_order(p, hpage_pmd_size)) + success("OK"); + else + fail("Fail"); + + validate_memory(p, 0, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); + thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + static void usage(void) { fprintf(stderr, "\nUsage: ./khugepaged [OPTIONS] [dir]\n\n"); @@ -1377,6 +1571,20 @@ int main(int argc, char **argv) parse_test_type(argc, argv); + if (mthp_khugepaged_context && + !(thp_supported_orders() & (1UL << collapse_order))) + ksft_exit_skip("Order %d is not a supported anon THP order\n", + collapse_order); + + if (mthp_khugepaged_context) { + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_perror("open(/proc/self/pagemap)"); + kpageflags_fd = open("/proc/kpageflags", O_RDONLY); + if (kpageflags_fd < 0) + ksft_exit_fail_perror("open(/proc/kpageflags)"); + } + setbuf(stdout, NULL); /* @@ -1427,6 +1635,10 @@ int main(int argc, char **argv) TEST(collapse_empty, madvise_context, anon_ops); TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_single_window, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_partial_window, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_max_ptes_none, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_mixed_sources, mthp_khugepaged_context, anon_ops); TEST(collapse_single_pte_entry, khugepaged_context, anon_ops); TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops); From d2a9d28f53852b16f0e9a99a6855a4e4bf12cb4e Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:42 +0100 Subject: [PATCH 521/539] selftests/mm: parameterize the mixed-source collapse case by source order collapse_order_mixed_sources() faults its region as order-2 folios and collapses them to the -c target. Order 2 is below the contpte size on every arm64 page size, so nothing in this suite collapses a contpte-mapped source on purpose. Let -s name the source order alongside -c. The case then faults at that order, keeping order 2 when -s is absent, and the source order has to be a supported mTHP order below the target. The other mTHP cases are unaffected: mthp_push_target_order() enables only the target order. A -c at or below -s is refused before any case runs: the sources would already be the size being asked for. Without the check the generic cases fail on that one by one instead of saying why. "-s 5 -c 7" on arm64/64K then collapses contpte-mapped sources into a larger mTHP. Link: https://lore.kernel.org/20260919002451.496763-13-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Acked-by: Lorenzo Stoakes (ARM) Tested-by: Muhammad Usama Anjum Reviewed-by: Baolin Wang Tested-by: Baolin Wang Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 51bda01446cd..9d1c47bd5013 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -1354,11 +1354,13 @@ static void collapse_order_max_ptes_none(struct collapse_context *c, static void collapse_order_mixed_sources(struct collapse_context *c, struct mem_ops *ops) { + int source_order = anon_order ? anon_order : MIN_MTHP_ORDER; struct thp_settings settings; void *p; - if (collapse_order <= MIN_MTHP_ORDER) { - ksft_test_result_skip("%s: no source order below target\n", + if (source_order >= collapse_order || + !(thp_supported_orders() & (1UL << source_order))) { + ksft_test_result_skip("%s: no supported source order below target\n", __func__); return; } @@ -1366,7 +1368,7 @@ static void collapse_order_mixed_sources(struct collapse_context *c, mthp_push_target_order(); settings = *thp_current_settings(); - settings.hugepages[MIN_MTHP_ORDER].enabled = THP_ALWAYS; + settings.hugepages[source_order].enabled = THP_ALWAYS; thp_push_settings(&settings); p = ops->setup_area(1); ops->fault(p, 0, hpage_pmd_size); @@ -1376,10 +1378,9 @@ static void collapse_order_mixed_sources(struct collapse_context *c, * The allocator can fall back to smaller folios under fragmentation; * having nothing to collapse from is not a failure. */ - if (!is_range_backed_by_order(p, hpage_pmd_size, MIN_MTHP_ORDER, + if (!is_range_backed_by_order(p, hpage_pmd_size, source_order, pagemap_fd, kpageflags_fd)) { - ksft_print_msg("No order-%d sources to collapse...", - MIN_MTHP_ORDER); + ksft_print_msg("No order-%d sources to collapse...", source_order); skip("Skip"); ops->cleanup_area(p, hpage_pmd_size); thp_pop_settings(); @@ -1389,7 +1390,8 @@ static void collapse_order_mixed_sources(struct collapse_context *c, if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); - ksft_print_msg("Collapse region backed by smaller large folios..."); + ksft_print_msg("Collapse region backed by order-%d sources...", + source_order); if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) fail("Timeout"); else if (all_windows_at_order(p, hpage_pmd_size)) @@ -1420,6 +1422,7 @@ static void usage(void) fprintf(stderr, "\t\t-s: mTHP size, expressed as page order.\n"); fprintf(stderr, "\t\t Defaults to 0. Use this size for anon or shmem allocations.\n"); fprintf(stderr, "\t\t-c: collapse order for mTHP collapse, expressed as page order.\n"); + fprintf(stderr, "\t\t -s, if set, is the source order for the mixed-source case.\n"); exit(1); } @@ -1575,6 +1578,9 @@ int main(int argc, char **argv) !(thp_supported_orders() & (1UL << collapse_order))) ksft_exit_skip("Order %d is not a supported anon THP order\n", collapse_order); + if (mthp_khugepaged_context && collapse_order <= anon_order) + ksft_exit_skip("-c %d needs a source order below it, -s says %d\n", + collapse_order, anon_order); if (mthp_khugepaged_context) { pagemap_fd = open("/proc/self/pagemap", O_RDONLY); From e1fe35d6c8474a0d1c9291f8425f0b008d48cb16 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:43 +0100 Subject: [PATCH 522/539] selftests/mm: cover a shared-source collapse write race collapse_fork() checks that a fork-shared range collapses in the child while the parent keeps its own pages, but the parent sits still while that happens. Nothing checks that CoW isolation survives a collapse racing with writes to the shared source. Add a case where the parent writes to the shared range throughout the child's collapse. CoW has to keep the two apart: the child must see the content from before the fork, and the parent only its own writes. The parent unshares one page every 10ms, starting only once the child says it is about to collapse. Writing the range in a burst would break CoW on all of it before the collapse begins, leaving the child to collapse pages that are already exclusive to it. Preparation for changing how collapse handles fork-shared sources. Link: https://lore.kernel.org/20260919002451.496763-14-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Baolin Wang Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 106 ++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 9d1c47bd5013..21ae258bd56e 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -1166,6 +1166,109 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops ksft_test_result_report(exit_status, "%s\n", __func__); } +/* + * The parent writes to the fork-shared range throughout the child's + * collapse. CoW must keep the two apart: the child sees the pre-fork + * content, the parent only its own writes. + */ +static void collapse_fork_cow_race(struct collapse_context *c, struct mem_ops *ops) +{ + const int stride = page_size / sizeof(int); + int wstatus, child_status, i, n; + unsigned long shared; + volatile int *ip; + pid_t child; + int sync[2]; + char go = 1; + void *p; + + /* At a page per 10 ms, 64 pages spread the writes across the collapse */ + n = 64; + shared = n * page_size; + + p = ops->setup_area(1); + /* Shared prefix, with the pre-fork pattern */ + ops->fault(p, 0, shared); + if (pipe(sync)) + ksft_exit_fail_perror("pipe()"); + + /* A volatile pointer so the stores are not merged or dropped */ + ip = p; + + ksft_print_msg("Fork, collapse in the child while the parent rewrites..."); + child = fork(); + if (!child) { + int collapse_status; + + close(sync[0]); + /* Private remainder */ + ops->fault(p, shared, hpage_pmd_size); + /* Start the parent unsharing, and give it a head start */ + if (write(sync[1], &go, 1) != 1) + _exit(KSFT_FAIL); + usleep(5000); + c->collapse("Collapse a range the parent is writing to", + p, 1, ops, true); + collapse_status = exit_status; + for (i = 0; i < n; i++) + if (ip[i * stride] != i + 0xdead0000) + break; + if (i == n) + success("OK"); + else + fail("Fail: child content"); + /* The content check must not bury a failed collapse */ + if (exit_status != KSFT_FAIL) + exit_status = collapse_status; + ops->cleanup_area(p, hpage_pmd_size); + _exit(exit_status); + } + + close(sync[1]); + if (read(sync[0], &go, 1) != 1) + ksft_exit_fail_msg("child never reached the collapse\n"); + close(sync[0]); + + /* + * Unshare one page at a time: a burst would break CoW on the whole + * range before the collapse starts, leaving nothing shared to collapse. + */ + i = 0; + for (;;) { + pid_t ret; + + if (i < n) + ip[i * stride] = i + 0xbeef0000; + i++; + usleep(10 * 1000); + ret = waitpid(child, &wstatus, WNOHANG); + if (ret == child) + break; + if (ret < 0) + ksft_exit_fail_perror("waitpid()"); + } + + /* Finish whatever the paced sweep did not reach */ + for (; i < n; i++) + ip[i * stride] = i + 0xbeef0000; + /* A child that died reading the racing pages is a failure, not a zero */ + child_status = WIFEXITED(wstatus) ? WEXITSTATUS(wstatus) : KSFT_FAIL; + + ksft_print_msg("Check the parent sees only its own writes..."); + for (i = 0; i < n; i++) + if (ip[i * stride] != i + 0xbeef0000) + break; + if (i == n) + success("OK"); + else + fail("Fail: parent content"); + ops->cleanup_area(p, hpage_pmd_size); + /* The parent's check must not bury the child's verdict */ + if (exit_status != KSFT_FAIL) + exit_status = child_status; + ksft_test_result_report(exit_status, "%s\n", __func__); +} + static void madvise_collapse_existing_thps(struct collapse_context *c, struct mem_ops *ops) { @@ -1700,6 +1803,9 @@ int main(int argc, char **argv) TEST(collapse_max_ptes_shared, khugepaged_context, anon_ops); TEST(collapse_max_ptes_shared, madvise_context, anon_ops); + TEST(collapse_fork_cow_race, khugepaged_context, anon_ops); + TEST(collapse_fork_cow_race, madvise_context, anon_ops); + TEST(madvise_collapse_existing_thps, madvise_context, anon_ops); TEST(madvise_collapse_existing_thps, madvise_context, read_only_file_ops); TEST(madvise_collapse_existing_thps, madvise_context, read_write_file_read_ops); From e5d767fe33c1666860bcdef5f5c5c6183bc4a718 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:44 +0100 Subject: [PATCH 523/539] selftests/mm: run every supported collapse order by default The mTHP collapse cases only run when the caller names both the context and an order, so a plain ./khugepaged covers the PMD contexts on anon and nothing else. run_vmtests.sh pinned order 4 and covered no other. Run the mTHP cases once per supported anon THP order below the PMD when -c is absent, and pull that context into both the no-argument invocation and "all". Also: - -c still pins one order, and now says what is wrong instead of printing the usage text. - Both orders end up as array indices and shift counts, so -s and -c are range-checked before they get there. - The mTHP context has only anon cases, so a run that names a different mem_type -- "all:shmem", say -- drops it again rather than refusing to start. Naming both explicitly still refuses. - A case carries the order it was registered at, so a result names it: # Run test: collapse_single_mthp (mthp_khugepaged:anon, order 6) On x86-64 with 4K pages that is orders 2 through 8, and ./khugepaged goes from 29 results in 17 seconds to 77 in 29, so run_vmtests.sh can drop its pinned order-4 line. Link: https://lore.kernel.org/20260919002451.496763-15-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Reviewed-by: Baolin Wang Tested-by: Muhammad Usama Anjum Tested-by: Baolin Wang Assisted-by: LLM Cc: Alexander Gordeev Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged.c | 101 +++++++++++++++++----- tools/testing/selftests/mm/run_vmtests.sh | 2 - 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 21ae258bd56e..a2ac3b3ca5de 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -30,6 +30,9 @@ static unsigned long page_size; static int hpage_pmd_nr; static int anon_order; static int collapse_order; +static bool collapse_order_set; +static int collapse_orders[NR_ORDERS]; +static int nr_collapse_orders; static int pagemap_fd = -1; static int kpageflags_fd = -1; @@ -1525,12 +1528,14 @@ static void usage(void) fprintf(stderr, "\t\t-s: mTHP size, expressed as page order.\n"); fprintf(stderr, "\t\t Defaults to 0. Use this size for anon or shmem allocations.\n"); fprintf(stderr, "\t\t-c: collapse order for mTHP collapse, expressed as page order.\n"); + fprintf(stderr, "\t\t Defaults to every supported order below the PMD.\n"); fprintf(stderr, "\t\t -s, if set, is the source order for the mixed-source case.\n"); exit(1); } static void parse_test_type(int argc, char **argv) { + bool mthp_context_implied = false; int opt; char *buf; const char *token; @@ -1542,6 +1547,7 @@ static void parse_test_type(int argc, char **argv) break; case 'c': collapse_order = atoi(optarg); + collapse_order_set = true; break; case 'h': default: @@ -1549,12 +1555,25 @@ static void parse_test_type(int argc, char **argv) } } + /* + * Both orders end up as array indices and shift counts, so neither + * can be negative, and a zero collapse order asks for base pages. + */ + if (anon_order < 0 || anon_order > hpage_pmd_order) + ksft_exit_fail_msg("-s takes an order in 0..%d, not %d\n", + hpage_pmd_order, anon_order); + if (collapse_order_set && + (collapse_order <= 0 || collapse_order >= hpage_pmd_order)) + ksft_exit_fail_msg("-c takes an order in 1..%d, not %d\n", + hpage_pmd_order - 1, collapse_order); + argv += optind; argc -= optind; if (argc == 0) { - /* Backwards compatibility */ + /* No arguments: anon under every context */ khugepaged_context = &__khugepaged_context; + mthp_khugepaged_context = &__mthp_khugepaged_context; madvise_context = &__madvise_context; anon_ops = &__anon_ops; return; @@ -1565,13 +1584,14 @@ static void parse_test_type(int argc, char **argv) if (!strcmp(token, "all")) { khugepaged_context = &__khugepaged_context; + mthp_khugepaged_context = &__mthp_khugepaged_context; madvise_context = &__madvise_context; + /* The mTHP context has only anon cases; let other mem_types drop it */ + mthp_context_implied = true; } else if (!strcmp(token, "khugepaged")) { khugepaged_context = &__khugepaged_context; } else if (!strcmp(token, "mthp_khugepaged")) { mthp_khugepaged_context = &__mthp_khugepaged_context; - if (collapse_order <= 0 || collapse_order >= hpage_pmd_order) - usage(); } else if (!strcmp(token, "madvise")) { madvise_context = &__madvise_context; } else { @@ -1587,20 +1607,20 @@ static void parse_test_type(int argc, char **argv) read_write_file_write_ops = &__read_write_file_write_ops; anon_ops = &__anon_ops; shmem_ops = &__shmem_ops; - if (mthp_khugepaged_context) - usage(); } else if (!strcmp(buf, "anon")) { anon_ops = &__anon_ops; } else if (!strcmp(buf, "file")) { read_only_file_ops = &__read_only_file_ops; read_write_file_read_ops = &__read_write_file_read_ops; read_write_file_write_ops = &__read_write_file_write_ops; - if (mthp_khugepaged_context) + if (mthp_khugepaged_context && !mthp_context_implied) usage(); + mthp_khugepaged_context = NULL; } else if (!strcmp(buf, "shmem")) { shmem_ops = &__shmem_ops; - if (mthp_khugepaged_context) + if (mthp_khugepaged_context && !mthp_context_implied) usage(); + mthp_khugepaged_context = NULL; } else { usage(); } @@ -1622,6 +1642,7 @@ struct test_case { struct mem_ops *ops; const char *desc; test_fn fn; + int order; /* mTHP contexts: the collapse order */ }; #define MAX_TEST_CASES 256 @@ -1637,6 +1658,7 @@ static int nr_test_cases; .ops = o, \ .desc = #t, \ .fn = t, \ + .order = collapse_order, \ }; \ } \ } while (0) @@ -1677,13 +1699,35 @@ int main(int argc, char **argv) parse_test_type(argc, argv); - if (mthp_khugepaged_context && - !(thp_supported_orders() & (1UL << collapse_order))) - ksft_exit_skip("Order %d is not a supported anon THP order\n", - collapse_order); - if (mthp_khugepaged_context && collapse_order <= anon_order) - ksft_exit_skip("-c %d needs a source order below it, -s says %d\n", - collapse_order, anon_order); + if (mthp_khugepaged_context) { + unsigned long orders = thp_supported_orders(); + + if (collapse_order_set) { + if (!(orders & (1UL << collapse_order))) + ksft_exit_skip("Order %d is not a supported anon THP order\n", + collapse_order); + if (collapse_order <= anon_order) + ksft_exit_skip("-c %d needs a source order below it, -s says %d\n", + collapse_order, anon_order); + collapse_orders[nr_collapse_orders++] = collapse_order; + } else { + /* + * Every supported order above the source: -s makes the + * fault path hand out folios of that order, so a target + * at or below it has nothing to collapse. + */ + int first = anon_order + 1; + + if (first < MIN_MTHP_ORDER) + first = MIN_MTHP_ORDER; + for (int i = first; i < hpage_pmd_order; i++) { + if (orders & (1UL << i)) + collapse_orders[nr_collapse_orders++] = i; + } + if (!nr_collapse_orders) + ksft_print_msg("mTHP cases skipped: no order above the source\n"); + } + } if (mthp_khugepaged_context) { pagemap_fd = open("/proc/self/pagemap", O_RDONLY); @@ -1732,7 +1776,17 @@ int main(int argc, char **argv) TEST(collapse_full, khugepaged_context, read_write_file_read_ops); TEST(collapse_full, khugepaged_context, read_write_file_write_ops); TEST(collapse_full, khugepaged_context, shmem_ops); - TEST(collapse_full, mthp_khugepaged_context, anon_ops); + for (int i = 0; i < nr_collapse_orders; i++) { + collapse_order = collapse_orders[i]; + TEST(collapse_full, mthp_khugepaged_context, anon_ops); + TEST(collapse_empty, mthp_khugepaged_context, anon_ops); + TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_single_window, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_partial_window, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_max_ptes_none, mthp_khugepaged_context, anon_ops); + TEST(collapse_order_mixed_sources, mthp_khugepaged_context, anon_ops); + } + TEST(collapse_full, madvise_context, anon_ops); TEST(collapse_full, madvise_context, read_only_file_ops); TEST(collapse_full, madvise_context, read_write_file_read_ops); @@ -1740,15 +1794,8 @@ int main(int argc, char **argv) TEST(collapse_full, madvise_context, shmem_ops); TEST(collapse_empty, khugepaged_context, anon_ops); - TEST(collapse_empty, mthp_khugepaged_context, anon_ops); TEST(collapse_empty, madvise_context, anon_ops); - TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops); - TEST(collapse_order_single_window, mthp_khugepaged_context, anon_ops); - TEST(collapse_order_partial_window, mthp_khugepaged_context, anon_ops); - TEST(collapse_order_max_ptes_none, mthp_khugepaged_context, anon_ops); - TEST(collapse_order_mixed_sources, mthp_khugepaged_context, anon_ops); - TEST(collapse_single_pte_entry, khugepaged_context, anon_ops); TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops); TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_read_ops); @@ -1821,7 +1868,15 @@ int main(int argc, char **argv) for (int i = 0; i < nr_test_cases; i++) { struct test_case *t = &test_cases[i]; - ksft_print_msg("\n# Run test: %s (%s:%s)\n", t->desc, t->ctx->name, t->ops->name); + if (t->ctx == &__mthp_khugepaged_context) { + collapse_order = t->order; + ksft_print_msg("\n# Run test: %s (%s:%s, order %d)\n", + t->desc, t->ctx->name, t->ops->name, + t->order); + } else { + ksft_print_msg("\n# Run test: %s (%s:%s)\n", t->desc, + t->ctx->name, t->ops->name); + } t->fn(t->ctx, t->ops); } diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 6b80cf2eab14..f11384127885 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -392,8 +392,6 @@ CATEGORY="thp" run_test ./khugepaged all:shmem CATEGORY="thp" run_test ./khugepaged -s 4 all:shmem -CATEGORY="thp" run_test ./khugepaged -c 4 mthp_khugepaged:anon - # Try to create XFS if not provided if [ -z "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then if test_selected "thp"; then From 4e316c12768b7237e2f4e11864ac0b93d120d2b4 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:45 +0100 Subject: [PATCH 524/539] selftests/mm: check that one khugepaged pass collapses one window khugepaged_full_pass() drives the daemon through sysfs: a store to scan_sleep_millisecs wakes it, and full_scans advancing by two marks one pass that started after setup. Every mTHP collapse result in the suite rests on that pair, and nothing checks it. Add khugepaged_sync_check. Each step: - prepare one aligned window - record its source PFNs from pagemap - run one khugepaged_full_pass() barrier - require the window came out collapsed, with exactly one collapse attempt attributed to it The anon events carry no virtual address, so an attempt is matched by the source folio PFN and order that the mm_collapse_huge_page_isolate tracepoint reports. Reading the trace buffer takes four small helpers in vm_util: open an event subsystem's enable file, flip it, clear the buffer, and open it for reading. scan_sleep_millisecs is set to a minute, so a step that took a sleep instead of a wake would blow the budget. Passes 5/5 on x86-64 4K and arm64 64K. Link: https://lore.kernel.org/20260919002451.496763-16-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Baolin Wang Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/Makefile | 1 + .../selftests/mm/khugepaged_sync_check.c | 179 ++++++++++++++++++ tools/testing/selftests/mm/run_vmtests.sh | 2 + tools/testing/selftests/mm/vm_util.c | 38 ++++ tools/testing/selftests/mm/vm_util.h | 4 + 5 files changed, 224 insertions(+) create mode 100644 tools/testing/selftests/mm/khugepaged_sync_check.c diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index cb32cf5d867e..2ed881324911 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -106,6 +106,7 @@ TEST_GEN_FILES += merge TEST_GEN_FILES += rmap TEST_GEN_FILES += folio_split_race_test TEST_GEN_FILES += folio_order_check +TEST_GEN_FILES += khugepaged_sync_check TEST_GEN_FILES += soft-dirty ifeq ($(ARCH),x86_64) diff --git a/tools/testing/selftests/mm/khugepaged_sync_check.c b/tools/testing/selftests/mm/khugepaged_sync_check.c new file mode 100644 index 000000000000..28a9b1ff5d44 --- /dev/null +++ b/tools/testing/selftests/mm/khugepaged_sync_check.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Check that khugepaged_full_pass() drives khugepaged in step: one barrier + * over one prepared window must collapse it with exactly one collapse + * attempt attributed to its source pages, step after step. + * + * scan_sleep_millisecs is a minute so that a step which slept instead of + * being woken blows the budget. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" +#include "vm_util.h" +#include + +#define BASE_ADDR ((void *)(1UL << 30)) +/* Smallest order khugepaged considers */ +#define TARGET_ORDER 2 +#define NR_ITERATIONS 5 +#define PASS_TIMEOUT_S 30 + +static int pagemap_fd; +static int kpageflags_fd; +static int trace_events_fd = -1; +static unsigned long hpage_pmd_size; + +/* + * The events are system-wide: switch them off however the test ends, + * including from inside a helper that gives up. + */ +static void trace_events_off(void) +{ + if (trace_events_fd >= 0) + tracing_events_enable(trace_events_fd, false); +} + +/* Count the isolate events whose scan_pfn is one of the window's source PFNs */ +static int count_attributed(unsigned long *pfns, int nr_pfns, + unsigned int order) +{ + char line[1024]; + int count = 0; + FILE *fp; + + fp = tracing_open_trace(); + if (!fp) + ksft_exit_fail_msg("Cannot open trace buffer\n"); + + while (fgets(line, sizeof(line), fp)) { + unsigned long val; + unsigned int ord; + char *s, *o; + int i; + + s = strstr(line, "mm_collapse_huge_page_isolate:"); + if (!s) + continue; + if (sscanf(s, "mm_collapse_huge_page_isolate: scan_pfn=0x%lx", + &val) != 1) + continue; + o = strstr(s, "order="); + if (!o || sscanf(o, "order=%u", &ord) != 1 || ord != order) + continue; + for (i = 0; i < nr_pfns; i++) { + if (val == pfns[i]) { + count++; + break; + } + } + } + fclose(fp); + return count; +} + +static void one_step(int iteration) +{ + const size_t window = getpagesize() << TARGET_ORDER; + const int nr_pages = 1 << TARGET_ORDER; + unsigned long pfns[1 << TARGET_ORDER]; + bool collapsed, passed; + int attributed; + char *p; + int i; + + p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0); + if (p != BASE_ADDR) + ksft_exit_fail_perror("mmap() window"); + + for (i = 0; i < nr_pages; i++) { + p[i * getpagesize()] = i + 1; + pfns[i] = pagemap_get_pfn(pagemap_fd, p + i * getpagesize()); + if (pfns[i] == -1UL) + ksft_exit_fail_msg("Source page not present\n"); + } + + /* Clear before enabling so the buffer holds only this step's events */ + if (tracing_clear_trace()) + ksft_exit_fail_msg("Cannot clear the trace buffer\n"); + if (tracing_events_enable(trace_events_fd, true)) + ksft_exit_fail_msg("Cannot enable huge_memory events\n"); + + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + passed = khugepaged_full_pass(PASS_TIMEOUT_S); + + /* Off before anything that can give up: the events are system-wide */ + if (tracing_events_enable(trace_events_fd, false)) + ksft_exit_fail_msg("Cannot disable huge_memory events\n"); + if (!passed) + ksft_exit_fail_msg("khugepaged did not complete a full pass\n"); + + collapsed = is_range_backed_by_order(p, window, TARGET_ORDER, + pagemap_fd, kpageflags_fd); + attributed = count_attributed(pfns, nr_pages, TARGET_ORDER); + + ksft_test_result(collapsed && attributed == 1, + "step %d: window collapsed, %d attributed result(s)\n", + iteration, attributed); + + munmap(p, hpage_pmd_size); +} + +int main(void) +{ + struct thp_settings settings; + int i; + + ksft_print_header(); + + if (!thp_available()) + ksft_exit_skip("Transparent Hugepages not available\n"); + if (!(thp_supported_orders() & (1UL << TARGET_ORDER))) + ksft_exit_skip("Order %d is not a supported anon THP order\n", + TARGET_ORDER); + + hpage_pmd_size = read_pmd_pagesize(); + if (!hpage_pmd_size) + ksft_exit_fail_msg("Reading PMD pagesize failed\n"); + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_perror("open(/proc/self/pagemap)"); + kpageflags_fd = open("/proc/kpageflags", O_RDONLY); + if (kpageflags_fd < 0) + ksft_exit_skip("open(/proc/kpageflags) requires root\n"); + trace_events_fd = tracing_events_open("huge_memory"); + if (trace_events_fd < 0) + ksft_exit_skip("huge_memory events require tracefs and root\n"); + atexit(trace_events_off); + + ksft_set_plan(NR_ITERATIONS); + + thp_save_settings(); + thp_read_settings(&settings); + settings.thp_enabled = THP_MADVISE; + settings.thp_defrag = THP_DEFRAG_ALWAYS; + settings.khugepaged.defrag = 1; + settings.khugepaged.scan_sleep_millisecs = 60 * 1000; + settings.khugepaged.alloc_sleep_millisecs = 60 * 1000; + settings.khugepaged.max_ptes_none = (hpage_pmd_size / getpagesize()) - 1; + /* One wake must complete one full pass; see khugepaged_full_pass() */ + settings.khugepaged.pages_to_scan = 1UL << 24; + for (i = 0; i < NR_ORDERS; i++) + settings.hugepages[i].enabled = THP_NEVER; + settings.hugepages[TARGET_ORDER].enabled = THP_INHERIT; + /* Base of the settings stack; the bottom entry is never popped */ + thp_push_settings(&settings); + + for (i = 0; i < NR_ITERATIONS; i++) + one_step(i); + + ksft_finished(); +} diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index f11384127885..3e8c76cf4948 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -384,6 +384,8 @@ CATEGORY="cow" run_test ./cow CATEGORY="thp" run_test ./folio_order_check +CATEGORY="thp" run_test ./khugepaged_sync_check + CATEGORY="thp" run_test ./khugepaged CATEGORY="thp" run_test ./khugepaged -s 2 diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 1f88330fd027..a0ab78ceedb1 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -598,6 +598,44 @@ bool is_range_backed_by_order(char *start, size_t len, int order, return true; } +#define TRACEFS_ROOT "/sys/kernel/tracing" + +/* + * Returns -1 without tracefs or the subsystem. The events are system-wide: + * whoever switches them on has to switch them off again, on every exit path. + */ +int tracing_events_open(const char *subsys) +{ + char path[256]; + + snprintf(path, sizeof(path), TRACEFS_ROOT "/events/%s/enable", + subsys); + return open(path, O_WRONLY); +} + +int tracing_events_enable(int fd, bool enable) +{ + if (pwrite(fd, enable ? "1" : "0", 1, 0) != 1) + return -1; + return 0; +} + +/* Drop what the trace buffer holds so far */ +int tracing_clear_trace(void) +{ + int fd = open(TRACEFS_ROOT "/trace", O_WRONLY | O_TRUNC); + + if (fd < 0) + return -1; + close(fd); + return 0; +} + +FILE *tracing_open_trace(void) +{ + return fopen(TRACEFS_ROOT "/trace", "r"); +} + /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor, uint64_t *ioctls) diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index ea48e6a7527e..072a6c756c51 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -121,6 +121,10 @@ int close_procmap(struct procmap_fd *procmap); int write_sysfs(const char *file_path, unsigned long val); int read_sysfs(const char *file_path, unsigned long *val); bool softdirty_supported(void); +int tracing_events_open(const char *subsys); +int tracing_events_enable(int fd, bool enable); +int tracing_clear_trace(void); +FILE *tracing_open_trace(void); static inline int open_self_procmap(struct procmap_fd *procmap_out) { From 66c37446443889131133d5bc2870e9da8cc1360e Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:46 +0100 Subject: [PATCH 525/539] selftests/mm: add khugepaged race harness Collapse serialises against faults, GUP, fork, mremap and zapping through a protocol of locks, TLB flushes and refcount checks. No khugepaged selftest exercises any of it under contention. Add khugepaged_race. Six racing threads work the same address space: - two faulters - an MADV_DONTNEED thread - a transient FOLL_PIN thread (gup_test) - a forker - an mremap thread One of three drivers collapses under them: stepped khugepaged, one full pass at a time via khugepaged_full_pass(), so each step covers a known extent; free khugepaged left to run (scan_sleep_millisecs=0), for soak; madvise an MADV_COLLAPSE and MADV_DONTNEED loop. Every mode runs in turn unless -m names one, five seconds each. Every supported anon THP order is set to inherit and max_ptes_none is 0, so a window collapses only once fully populated and the racing MADV_DONTNEED steers selection across orders. The rule is that a racing page reads as its pattern or as zero, never anything else. The faulters and fork children check it throughout, and a final sweep checks it again. The other half of the check is the kernel's own assertions, so read dmesg too. The pin thread goes through gup_test, so the harness skips without CONFIG_GUP_TEST or root. The default playground is three shared PMD-sized areas plus the mremap thread's, over two gigabytes at a 512M PMD; -a shrinks it. Link: https://lore.kernel.org/20260919002451.496763-17-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Baolin Wang Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/Makefile | 1 + tools/testing/selftests/mm/khugepaged_race.c | 415 +++++++++++++++++++ tools/testing/selftests/mm/run_vmtests.sh | 2 + 3 files changed, 418 insertions(+) create mode 100644 tools/testing/selftests/mm/khugepaged_race.c diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 2ed881324911..beacc0f87304 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -107,6 +107,7 @@ TEST_GEN_FILES += rmap TEST_GEN_FILES += folio_split_race_test TEST_GEN_FILES += folio_order_check TEST_GEN_FILES += khugepaged_sync_check +TEST_GEN_FILES += khugepaged_race TEST_GEN_FILES += soft-dirty ifeq ($(ARCH),x86_64) diff --git a/tools/testing/selftests/mm/khugepaged_race.c b/tools/testing/selftests/mm/khugepaged_race.c new file mode 100644 index 000000000000..66c9e2c10f66 --- /dev/null +++ b/tools/testing/selftests/mm/khugepaged_race.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Race collapse against faults, GUP pins, fork, mremap and MADV_DONTNEED + * over the same ranges. A racing page must read as its pattern or as + * zero, never anything else; the kernel's own assertions in dmesg are the + * other half of the check. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" +#include "vm_util.h" +#include +#include "../../../../mm/gup_test.h" + +#ifndef FOLL_WRITE +#define FOLL_WRITE 0x01 +#endif + +#define BASE_ADDR ((void *)(1UL << 30)) +#define PASS_TIMEOUT_S 30 + +/* + * PMD-sized areas the racing threads share, plus one for the mremap + * thread. -a shrinks it where a PMD is 512M. + */ +#define DEFAULT_SHARED_AREAS 3 +static int nr_shared_areas; +static int nr_areas; + +static unsigned long hpage_pmd_size; +static unsigned long page_size; +/* nr_areas PMD-sized areas; the last one belongs to the mremap thread */ +static char *region; +static char *mremap_area; +static char *mremap_scratch; +static int gup_fd = -1; +static volatile int stop; +static volatile int corrupted; + +static unsigned int pattern(unsigned long page_idx) +{ + unsigned int val = (unsigned int)page_idx * 2654435761U; + + return val ? val : 1; /* never collides with the zero-fill */ +} + +/* Zero means never written; anything else must be this page's pattern */ +static bool page_is_corrupt(unsigned long page_idx, unsigned int *val) +{ + *val = *(unsigned int *)(region + page_idx * page_size); + + return *val && *val != pattern(page_idx); +} + +static void check_page(unsigned long page_idx) +{ + unsigned int val; + + if (page_is_corrupt(page_idx, &val)) { + corrupted = 1; + ksft_print_msg("Corruption at page %lu: %#x != %#x\n", + page_idx, val, pattern(page_idx)); + } +} + +static unsigned long shared_pages(void) +{ + return nr_shared_areas * hpage_pmd_size / page_size; +} + +static unsigned long rand_page(unsigned int *seed) +{ + return (unsigned long)rand_r(seed) % shared_pages(); +} + +/* Clamp so a range never reaches the mremap thread's area */ +static unsigned long room_from(unsigned long page_idx, unsigned long want) +{ + unsigned long left = shared_pages() - page_idx; + + return want < left ? want : left; +} + +static void *faulter_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + unsigned long page_idx = rand_page(&seed); + + if (rand_r(&seed) & 1) + *(unsigned int *)(region + page_idx * page_size) = + pattern(page_idx); + else + check_page(page_idx); + } + return NULL; +} + +static void *dontneed_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + unsigned long page_idx = rand_page(&seed); + unsigned long nr = 1UL << (rand_r(&seed) % 6); /* 1..32 pages */ + + madvise(region + page_idx * page_size, + room_from(page_idx, nr) * page_size, MADV_DONTNEED); + usleep(rand_r(&seed) % 500); + } + return NULL; +} + +static void *pinner_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + struct gup_test gup = {}; + unsigned long page_idx = rand_page(&seed); + unsigned long nr = room_from(page_idx, 16); + + gup.addr = (unsigned long)(region + page_idx * page_size); + gup.size = nr * page_size; + gup.nr_pages_per_call = nr; + gup.gup_flags = FOLL_WRITE; + /* Racing MADV_DONTNEED makes transient failures expected */ + ioctl(gup_fd, PIN_FAST_BENCHMARK, &gup); + usleep(rand_r(&seed) % 200); + } + return NULL; +} + +static void *forker_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + pid_t pid = fork(); + + if (pid == 0) { + unsigned int val; + int bad = 0; + + /* + * No stdio in the child: a thread may hold stdout's + * lock across the fork, and printing under it hangs. + */ + for (int i = 0; i < 16; i++) + bad |= page_is_corrupt(rand_page(&seed), &val); + _exit(bad); + } + if (pid > 0) { + int wstatus; + + if (waitpid(pid, &wstatus, 0) < 0) + ksft_exit_fail_perror("waitpid()"); + /* A child killed on the read counts too */ + if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus)) + corrupted = 1; + } + usleep(rand_r(&seed) % 2000); + } + return NULL; +} + +static void *mremapper_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + + while (!stop) { + void *p; + + p = mremap(mremap_area, hpage_pmd_size, hpage_pmd_size, + MREMAP_MAYMOVE | MREMAP_FIXED, mremap_scratch); + if (p == MAP_FAILED) + ksft_exit_fail_perror("mremap() away"); + for (int i = 0; i < 8; i++) + mremap_scratch[(rand_r(&seed) % + (hpage_pmd_size / page_size)) * page_size] = 1; + p = mremap(mremap_scratch, hpage_pmd_size, hpage_pmd_size, + MREMAP_MAYMOVE | MREMAP_FIXED, mremap_area); + if (p == MAP_FAILED) + ksft_exit_fail_perror("mremap() back"); + /* The move back unmapped the scratch address: claim it again */ + if (mmap(mremap_scratch, hpage_pmd_size, PROT_NONE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, + -1, 0) != (void *)mremap_scratch) + ksft_exit_fail_perror("mmap() mremap scratch"); + usleep(rand_r(&seed) % 2000); + } + return NULL; +} + +static unsigned long now_ms(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000UL + tv.tv_usec / 1000; +} + +static void usage(void) +{ + fprintf(stderr, + "Usage: khugepaged_race [-d seconds] [-m stepped|free|madvise] [-a areas] [-t mask]\n" + "\tWithout -m, every mode runs in turn.\n" + "\t-d: seconds per mode (default 5)\n" + "\t-a: number of shared PMD-sized playground areas (default 3)\n" + "\t-t: bitmask of racing threads to start, for bisecting a failure\n"); + exit(1); +} + +int main(int argc, char **argv) +{ + static const char * const thread_names[] = { + "faulter", "faulter2", "dontneed", "pinner", "forker", + "mremapper", + }; + void *(*const thread_fns[])(void *) = { + faulter_fn, faulter_fn, dontneed_fn, pinner_fn, forker_fn, + mremapper_fn, + }; + const int nr_threads = ARRAY_SIZE(thread_names); + pthread_t threads[ARRAY_SIZE(thread_names)]; + static const char * const all_modes[] = { "stepped", "free", "madvise" }; + const char *one_mode[1]; + const char * const *modes = all_modes; + int nr_modes = ARRAY_SIZE(all_modes); + const char *mode_arg = NULL; + struct thp_settings settings; + unsigned long end_ms; + int duration_s = 5; + unsigned long thread_mask = ~0UL; + int nr_areas_arg = 0; + unsigned long i; + int steps = 0; + int opt; + + while ((opt = getopt(argc, argv, "a:d:m:t:h")) != -1) { + switch (opt) { + case 'a': + nr_areas_arg = atoi(optarg); + break; + case 'd': + duration_s = atoi(optarg); + break; + case 'm': + mode_arg = optarg; + break; + case 't': + thread_mask = strtoul(optarg, NULL, 0); + break; + default: + usage(); + } + } + + if (mode_arg) { + if (strcmp(mode_arg, "stepped") && strcmp(mode_arg, "free") && + strcmp(mode_arg, "madvise")) + usage(); + one_mode[0] = mode_arg; + modes = one_mode; + nr_modes = 1; + } + + ksft_print_header(); + if (!thp_available()) + ksft_exit_skip("Transparent Hugepages not available\n"); + + page_size = getpagesize(); + hpage_pmd_size = read_pmd_pagesize(); + if (!hpage_pmd_size) + ksft_exit_fail_msg("Reading PMD pagesize failed\n"); + + gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); + if (gup_fd < 0) + ksft_exit_skip("/sys/kernel/debug/gup_test requires CONFIG_GUP_TEST and root\n"); + + nr_shared_areas = nr_areas_arg > 0 ? nr_areas_arg : DEFAULT_SHARED_AREAS; + nr_areas = nr_shared_areas + 1; + + /* + * MREMAP_FIXED unmaps whatever is in the way without saying so, so + * claim the mremap thread's scratch address up front. + */ + mremap_scratch = (char *)BASE_ADDR + 2 * nr_areas * hpage_pmd_size; + if (mmap(mremap_scratch, hpage_pmd_size, PROT_NONE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, + -1, 0) != (void *)mremap_scratch) + ksft_exit_fail_perror("mmap() mremap scratch"); + + ksft_set_plan(nr_modes); + + thp_save_settings(); + thp_read_settings(&settings); + + /* Base of the settings stack; the bottom entry is never popped */ + thp_push_settings(&settings); + + for (int m = 0; m < nr_modes; m++) { + const char *mode = modes[m]; + + thp_read_settings(&settings); + settings.thp_enabled = THP_MADVISE; + settings.thp_defrag = THP_DEFRAG_ALWAYS; + settings.shmem_enabled = SHMEM_NEVER; + settings.khugepaged.defrag = 1; + settings.khugepaged.scan_sleep_millisecs = + strcmp(mode, "free") ? 1000 : 0; + settings.khugepaged.alloc_sleep_millisecs = 10; + /* + * mTHP collapse honours only 0 or HPAGE_PMD_NR - 1 here, and 0 + * keeps a step from being spent on PMD allocations that racing + * MADV_DONTNEED will not let succeed. + */ + settings.khugepaged.max_ptes_none = 0; + /* One wake, one pass: the playground plus the forked children's copies */ + settings.khugepaged.pages_to_scan = + nr_areas * (hpage_pmd_size / page_size) * 8; + for (i = 0; i < NR_ORDERS; i++) { + if (thp_supported_orders() & (1UL << i)) + settings.hugepages[i].enabled = THP_INHERIT; + } + thp_push_settings(&settings); + + region = mmap(BASE_ADDR, nr_areas * hpage_pmd_size, + PROT_READ | PROT_WRITE, MAP_ANONYMOUS | + MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0); + if (region != BASE_ADDR) + ksft_exit_fail_perror("mmap() playground"); + mremap_area = region + nr_shared_areas * hpage_pmd_size; + + /* Populate so the first pass has something to collapse */ + for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++) + *(unsigned int *)(region + i * page_size) = pattern(i); + memset(mremap_area, 1, hpage_pmd_size); + if (madvise(region, nr_areas * hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); + + for (i = 0; i < nr_threads; i++) { + if (!(thread_mask & (1UL << i))) { + threads[i] = 0; + continue; + } + if (pthread_create(&threads[i], NULL, thread_fns[i], + (void *)(i + 1))) + ksft_exit_fail_perror(thread_names[i]); + } + + end_ms = now_ms() + duration_s * 1000UL; + if (!strcmp(mode, "stepped")) { + while (now_ms() < end_ms && !corrupted) { + if (!khugepaged_full_pass(PASS_TIMEOUT_S)) + ksft_exit_fail_msg("khugepaged pass timed out\n"); + steps++; + } + } else if (!strcmp(mode, "free")) { + while (now_ms() < end_ms && !corrupted) + usleep(100 * 1000); + } else { /* madvise */ + while (now_ms() < end_ms && !corrupted) { + for (i = 0; i < nr_shared_areas; i++) { + madvise(region + i * hpage_pmd_size, + hpage_pmd_size, MADV_COLLAPSE); + } + madvise(region, nr_shared_areas * hpage_pmd_size, + MADV_DONTNEED); + steps++; + } + } + + stop = 1; + for (i = 0; i < nr_threads; i++) { + if (threads[i]) + pthread_join(threads[i], NULL); + } + + for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++) + check_page(i); + + ksft_test_result(!corrupted, + "%s: %ds, %d steps, no corruption\n", + mode, duration_s, steps); + + /* The next mode maps the same fixed address with its own settings */ + munmap(region, nr_areas * hpage_pmd_size); + thp_pop_settings(); + stop = 0; + steps = 0; + + if (corrupted) { + /* Memory is suspect; the rest would prove nothing */ + while (++m < nr_modes) + ksft_test_result_skip("%s: skipped after corruption\n", + modes[m]); + break; + } + } + + ksft_finished(); +} diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 3e8c76cf4948..a1b45a3dedae 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -386,6 +386,8 @@ CATEGORY="thp" run_test ./folio_order_check CATEGORY="thp" run_test ./khugepaged_sync_check +CATEGORY="thp" run_test ./khugepaged_race + CATEGORY="thp" run_test ./khugepaged CATEGORY="thp" run_test ./khugepaged -s 2 From 86e45ac4ea9f63cfe3aab167492e50c612231baa Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:47 +0100 Subject: [PATCH 526/539] selftests/mm: race the collapse of windows with holes The harness pins max_ptes_none to 0, so khugepaged only collapses a window once every PTE in it is present. A window with holes takes a different route, and never gets raced. A hole is zero-filled in the new folio rather than copied. Which slots count as holes keeps moving under the racing MADV_DONTNEED, right up to the moment the PMD is detached. Run both ends of the occupancy scale for every driver mode, one after the other. mTHP collapse supports only those two, 0 and HPAGE_PMD_NR - 1, and coerces anything between them to 0. Each result says which end it ran: ok 1 stepped/strict: 5s, 231 steps, no corruption ok 2 stepped/holes: 5s, 194 steps, no corruption Link: https://lore.kernel.org/20260919002451.496763-18-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Baolin Wang Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged_race.c | 32 ++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged_race.c b/tools/testing/selftests/mm/khugepaged_race.c index 66c9e2c10f66..6575045c0482 100644 --- a/tools/testing/selftests/mm/khugepaged_race.c +++ b/tools/testing/selftests/mm/khugepaged_race.c @@ -236,6 +236,8 @@ int main(int argc, char **argv) const int nr_threads = ARRAY_SIZE(thread_names); pthread_t threads[ARRAY_SIZE(thread_names)]; static const char * const all_modes[] = { "stepped", "free", "madvise" }; + static const bool occupancies[] = { false, true }; /* strict, holes */ + const int nr_occupancies = ARRAY_SIZE(occupancies); const char *one_mode[1]; const char * const *modes = all_modes; int nr_modes = ARRAY_SIZE(all_modes); @@ -303,7 +305,7 @@ int main(int argc, char **argv) -1, 0) != (void *)mremap_scratch) ksft_exit_fail_perror("mmap() mremap scratch"); - ksft_set_plan(nr_modes); + ksft_set_plan(nr_modes * nr_occupancies); thp_save_settings(); thp_read_settings(&settings); @@ -311,8 +313,9 @@ int main(int argc, char **argv) /* Base of the settings stack; the bottom entry is never popped */ thp_push_settings(&settings); - for (int m = 0; m < nr_modes; m++) { - const char *mode = modes[m]; + for (int run = 0; run < nr_modes * nr_occupancies; run++) { + const char *mode = modes[run / nr_occupancies]; + bool holes = occupancies[run % nr_occupancies]; thp_read_settings(&settings); settings.thp_enabled = THP_MADVISE; @@ -322,12 +325,14 @@ int main(int argc, char **argv) settings.khugepaged.scan_sleep_millisecs = strcmp(mode, "free") ? 1000 : 0; settings.khugepaged.alloc_sleep_millisecs = 10; + /* - * mTHP collapse honours only 0 or HPAGE_PMD_NR - 1 here, and 0 - * keeps a step from being spent on PMD allocations that racing - * MADV_DONTNEED will not let succeed. + * mTHP collapse honours only 0 or HPAGE_PMD_NR - 1 here. The two + * ends race different paths: a strict window has every PTE + * present, a hole-heavy one is mostly zero-filled. */ - settings.khugepaged.max_ptes_none = 0; + settings.khugepaged.max_ptes_none = holes ? + (hpage_pmd_size / page_size) - 1 : 0; /* One wake, one pass: the playground plus the forked children's copies */ settings.khugepaged.pages_to_scan = nr_areas * (hpage_pmd_size / page_size) * 8; @@ -393,8 +398,9 @@ int main(int argc, char **argv) check_page(i); ksft_test_result(!corrupted, - "%s: %ds, %d steps, no corruption\n", - mode, duration_s, steps); + "%s/%s: %ds, %d steps, no corruption\n", + mode, holes ? "holes" : "strict", + duration_s, steps); /* The next mode maps the same fixed address with its own settings */ munmap(region, nr_areas * hpage_pmd_size); @@ -404,9 +410,11 @@ int main(int argc, char **argv) if (corrupted) { /* Memory is suspect; the rest would prove nothing */ - while (++m < nr_modes) - ksft_test_result_skip("%s: skipped after corruption\n", - modes[m]); + while (++run < nr_modes * nr_occupancies) + ksft_test_result_skip("%s/%s: skipped after corruption\n", + modes[run / nr_occupancies], + occupancies[run % nr_occupancies] ? + "holes" : "strict"); break; } } From ddf1adcf35f4f3bf32502ee1cd8c15cd6c7dae1b Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:48 +0100 Subject: [PATCH 527/539] selftests/mm: add memory-pressure threads to the khugepaged race harness The harness races collapse against faults, pins, fork, mremap and MADV_DONTNEED, but nothing in it runs reclaim or compaction against the collapse. Add two more threads, and run every mode and occupancy limit both with and without them: - pageout: cycles MADV_PAGEOUT over a region of its own, faults it back in and checks the content each round, since a page's pattern must survive the trip through swap. Left out when the host has no swap, because then there is no anon reclaim to drive. - compactor: writes /proc/sys/vm/compact_memory in a loop. Compaction isolates and migrates folios, so it competes with a collapse for the pages it is gathering, with refcount elevations and migration entries of its own. Each result says whether it ran under pressure: ok 2 stepped/strict/pressure: 5s, 88 steps, no corruption A full run is now twelve combinations; -m picks one mode, -d shortens each run. Link: https://lore.kernel.org/20260919002451.496763-19-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Baolin Wang Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged_race.c | 142 ++++++++++++++++--- 1 file changed, 122 insertions(+), 20 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged_race.c b/tools/testing/selftests/mm/khugepaged_race.c index 6575045c0482..68046f0bf6d3 100644 --- a/tools/testing/selftests/mm/khugepaged_race.c +++ b/tools/testing/selftests/mm/khugepaged_race.c @@ -44,6 +44,8 @@ static unsigned long page_size; static char *region; static char *mremap_area; static char *mremap_scratch; +static char *pageout_area; +static size_t pageout_size; static int gup_fd = -1; static volatile int stop; static volatile int corrupted; @@ -204,6 +206,69 @@ static void *mremapper_fn(void *arg) return NULL; } +/* + * Swap traffic and LRU churn on a region nothing else writes, so a page's + * pattern must survive the trip through swap exactly. + */ +static void *pageout_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + unsigned long nr = pageout_size / page_size; + unsigned long i; + + for (i = 0; i < nr; i++) + *(unsigned int *)(pageout_area + i * page_size) = pattern(i); + + while (!stop) { + madvise(pageout_area, pageout_size, MADV_PAGEOUT); + for (i = 0; i < nr && !stop; i++) { + unsigned int val = *(unsigned int *)(pageout_area + + i * page_size); + + if (val != pattern(i)) { + corrupted = 1; + ksft_print_msg("Pageout corruption at page %lu: %#x != %#x\n", + i, val, pattern(i)); + } + } + usleep(rand_r(&seed) % 2000); + } + return NULL; +} + +/* Compaction migrates the collapse sources while they are being gathered */ +static void *compactor_fn(void *arg) +{ + unsigned int seed = (unsigned long)arg; + int fd = open("/proc/sys/vm/compact_memory", O_WRONLY); + + if (fd < 0) { + ksft_print_msg("No compact_memory; compactor idle\n"); + return NULL; + } + while (!stop) { + if (write(fd, "1", 1) < 0) + break; + usleep(10000 + rand_r(&seed) % 100000); + } + close(fd); + return NULL; +} + +static bool swap_available(void) +{ + char line[256]; + int lines = 0; + FILE *fp = fopen("/proc/swaps", "r"); + + if (!fp) + return false; + while (fgets(line, sizeof(line), fp)) + lines++; + fclose(fp); + return lines > 1; +} + static unsigned long now_ms(void) { struct timeval tv; @@ -227,17 +292,23 @@ int main(int argc, char **argv) { static const char * const thread_names[] = { "faulter", "faulter2", "dontneed", "pinner", "forker", - "mremapper", + "mremapper", "pageout", "compactor", }; void *(*const thread_fns[])(void *) = { faulter_fn, faulter_fn, dontneed_fn, pinner_fn, forker_fn, - mremapper_fn, + mremapper_fn, pageout_fn, compactor_fn, }; + enum { T_FAULTER, T_FAULTER2, T_DONTNEED, T_PINNER, T_FORKER, + T_MREMAPPER, T_PAGEOUT, T_COMPACTOR }; + const unsigned long pageout_bit = 1UL << T_PAGEOUT; + const unsigned long compactor_bit = 1UL << T_COMPACTOR; const int nr_threads = ARRAY_SIZE(thread_names); pthread_t threads[ARRAY_SIZE(thread_names)]; static const char * const all_modes[] = { "stepped", "free", "madvise" }; static const bool occupancies[] = { false, true }; /* strict, holes */ + static const bool pressures[] = { false, true }; /* quiet, under pressure */ const int nr_occupancies = ARRAY_SIZE(occupancies); + const int nr_pressures = ARRAY_SIZE(pressures); const char *one_mode[1]; const char * const *modes = all_modes; int nr_modes = ARRAY_SIZE(all_modes); @@ -246,6 +317,9 @@ int main(int argc, char **argv) unsigned long end_ms; int duration_s = 5; unsigned long thread_mask = ~0UL; + unsigned long base_mask; + bool have_swap; + char label[64]; int nr_areas_arg = 0; unsigned long i; int steps = 0; @@ -305,7 +379,13 @@ int main(int argc, char **argv) -1, 0) != (void *)mremap_scratch) ksft_exit_fail_perror("mmap() mremap scratch"); - ksft_set_plan(nr_modes * nr_occupancies); + base_mask = thread_mask; + have_swap = swap_available(); + if (!have_swap) + /* No swap, no anon reclaim: compaction-only pressure */ + ksft_print_msg("no swap: the pageout thread is not started\n"); + + ksft_set_plan(nr_modes * nr_occupancies * nr_pressures); thp_save_settings(); thp_read_settings(&settings); @@ -313,9 +393,25 @@ int main(int argc, char **argv) /* Base of the settings stack; the bottom entry is never popped */ thp_push_settings(&settings); - for (int run = 0; run < nr_modes * nr_occupancies; run++) { - const char *mode = modes[run / nr_occupancies]; - bool holes = occupancies[run % nr_occupancies]; + for (int run = 0; run < nr_modes * nr_occupancies * nr_pressures; run++) { + int rem = run % (nr_occupancies * nr_pressures); + const char *mode = modes[run / (nr_occupancies * nr_pressures)]; + bool holes = occupancies[rem / nr_pressures]; + bool pressure = pressures[rem % nr_pressures]; + + snprintf(label, sizeof(label), "%s/%s%s", mode, + holes ? "holes" : "strict", pressure ? "/pressure" : ""); + if (corrupted) { + /* Memory is suspect; the rest would prove nothing */ + ksft_test_result_skip("%s: skipped after corruption\n", label); + continue; + } + + thread_mask = base_mask; + if (!pressure) + thread_mask &= ~(pageout_bit | compactor_bit); + else if (!have_swap) + thread_mask &= ~pageout_bit; thp_read_settings(&settings); settings.thp_enabled = THP_MADVISE; @@ -349,6 +445,20 @@ int main(int argc, char **argv) ksft_exit_fail_perror("mmap() playground"); mremap_area = region + nr_shared_areas * hpage_pmd_size; + if (thread_mask & pageout_bit) { + /* Enough to drive real reclaim without swamping a small guest */ + pageout_size = 4 * hpage_pmd_size; + if (pageout_size < 16UL << 20) + pageout_size = 16UL << 20; + if (pageout_size > 64UL << 20) + pageout_size = 64UL << 20; + pageout_area = mmap(NULL, pageout_size, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (pageout_area == MAP_FAILED) + ksft_exit_fail_perror("mmap() pageout area"); + } + /* Populate so the first pass has something to collapse */ for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++) *(unsigned int *)(region + i * page_size) = pattern(i); @@ -397,26 +507,18 @@ int main(int argc, char **argv) for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++) check_page(i); - ksft_test_result(!corrupted, - "%s/%s: %ds, %d steps, no corruption\n", - mode, holes ? "holes" : "strict", - duration_s, steps); + ksft_test_result(!corrupted, "%s: %ds, %d steps, no corruption\n", + label, duration_s, steps); /* The next mode maps the same fixed address with its own settings */ munmap(region, nr_areas * hpage_pmd_size); + if (pageout_area) { + munmap(pageout_area, pageout_size); + pageout_area = NULL; + } thp_pop_settings(); stop = 0; steps = 0; - - if (corrupted) { - /* Memory is suspect; the rest would prove nothing */ - while (++run < nr_modes * nr_occupancies) - ksft_test_result_skip("%s/%s: skipped after corruption\n", - modes[run / nr_occupancies], - occupancies[run % nr_occupancies] ? - "holes" : "strict"); - break; - } } ksft_finished(); From 9640bb559a5df1931252b777760d334ffa00914e Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Sat, 19 Sep 2026 01:24:49 +0100 Subject: [PATCH 528/539] selftests/mm: zap whole PTE tables in the khugepaged race harness The harness's MADV_DONTNEED thread zaps 1 to 32 pages at a time, never a whole PMD-aligned area, and only a zap that covers a full table frees the table itself (CONFIG_PT_RECLAIM). Make the thread zap a whole PMD-aligned area about one iteration in 64, and keep the fine-grained zaps as the common case. The new case frees page tables, racing that against a collapse walking the same table. Link: https://lore.kernel.org/20260919002451.496763-20-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Signed-off-by: Andrew Morton Tested-by: Muhammad Usama Anjum Assisted-by: LLM Cc: Alexander Gordeev Cc: Baolin Wang Cc: Barry Song Cc: "David Hildenbrand (arm)" Cc: Dev Jain Cc: Hugh Dickens Cc: Jason Gunthorpe Cc: kernel-team@meta.com Cc: Lance Yang Cc: Leon Romanovsky Cc: Liam Howlett Cc: Lorenzo Stoakes (ARM) Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Shuah Khan (Samsung OSG) Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" Cc: Zi Yan --- tools/testing/selftests/mm/khugepaged_race.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged_race.c b/tools/testing/selftests/mm/khugepaged_race.c index 68046f0bf6d3..abb0678522cd 100644 --- a/tools/testing/selftests/mm/khugepaged_race.c +++ b/tools/testing/selftests/mm/khugepaged_race.c @@ -118,8 +118,21 @@ static void *dontneed_fn(void *arg) unsigned long page_idx = rand_page(&seed); unsigned long nr = 1UL << (rand_r(&seed) % 6); /* 1..32 pages */ - madvise(region + page_idx * page_size, - room_from(page_idx, nr) * page_size, MADV_DONTNEED); + /* + * Now and then zap a whole PMD-aligned area: only a zap that + * covers the full table frees the table itself (CONFIG_PT_RECLAIM). + */ + if (!(rand_r(&seed) % 64)) { + unsigned long area = page_idx / + (hpage_pmd_size / page_size); + + madvise(region + area * hpage_pmd_size, + hpage_pmd_size, MADV_DONTNEED); + } else { + madvise(region + page_idx * page_size, + room_from(page_idx, nr) * page_size, + MADV_DONTNEED); + } usleep(rand_r(&seed) % 500); } return NULL; From 7b466529307be9e723c94e5f6d4d63517c0950f8 Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Mon, 21 Sep 2026 13:42:25 +0800 Subject: [PATCH 529/539] mm: disallow raw PFN mappings of huge/shared zeropage Handling the huge/shared zeropage correctly in vmf_insert_pfn_pmd() and vmf_insert_pfn_prot() is more involved. We would need to check whether the VMA allows it and keep the mapping read-only, similar to the checks in vm_mixed_ok(). No in-tree user needs that support, so reject these mappings with VM_FAULT_SIGBUS rather than complicate the code for now. Link: https://lore.kernel.org/20260921054225.28537-1-lance.yang@linux.dev Link: https://lore.kernel.org/all/20260917121010.60966-1-lance.yang@linux.dev/ Signed-off-by: Lance Yang Signed-off-by: Andrew Morton Suggested-by: Kiryl Shutsemau (Meta) Suggested-by: David Hildenbrand (Arm) Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: David Hildenbrand (Arm) Acked-by: Zi Yan Reviewed-by: Lorenzo Stoakes (ARM) Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Liam Howlett Cc: Michal Hocko Cc: "Mike Rapoport (IBM)" Cc: Nico Pache (Red Hat) Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Usama Arif Cc: "Vlastimil Babka (SUSE)" --- mm/huge_memory.c | 3 +++ mm/memory.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 26523137b758..d7901b8171b2 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1727,6 +1727,9 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, (VM_PFNMAP|VM_MIXEDMAP)); BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); + if (unlikely(is_huge_zero_pfn(pfn))) + return VM_FAULT_SIGBUS; + pfnmap_setup_cachemode_pfn(pfn, &pgprot); return insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write); diff --git a/mm/memory.c b/mm/memory.c index 926276d41920..7d821b110df0 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2860,6 +2860,9 @@ vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); + if (unlikely(is_zero_pfn(pfn))) + return VM_FAULT_SIGBUS; + if (addr < vma->vm_start || addr >= vma->vm_end) return VM_FAULT_SIGBUS; From 4ff3b4a54a32142497fb44b79d0920aa6ba3393d Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Mon, 21 Sep 2026 08:24:41 -0700 Subject: [PATCH 530/539] mm/damon/core: charge only the part of a region the filter left Patch series "mm/damon/core: fix the size charged for a filter-trimmed region", v2. An address range DAMOS filter that partially overlaps a monitoring region splits it, so the scheme action reaches only one side of the boundary. damos_apply_scheme() reads the region size before that split and never re-reads it, so the quota charge and schemes//stats/sz_tried account for the whole original region. Patch 1 re-reads the size after the core filters have run. Patch 2 adds KUnit coverage for what the function charges and reports, for the trimmed cases and for the cases that must not change. This patch (of 2): An address range DAMOS filter that partially overlaps a monitoring region splits the region at the filter boundary, so the scheme action is applied to only one side of it. damos_apply_scheme() reads the region size once on entry, before damos_core_filter_out() performs that split. The quota charge and the statistics update therefore account for the region as it was before the trim. schemes//stats/sz_tried counts memory the filter excluded, which Documentation/mm/damon/design.rst says is not counted as tried, and with quotas/bytes set the excluded part is charged against the budget, throttling the scheme to a fraction of what was configured. User impact is that DAMOS could unexpectedly slowly run, due to over-charged quota. DAMOS stat could also be confusing. No critical events such as crashes or leask happen. Re-read the region size after the core filters have run. Link: https://lore.kernel.org/20260921152443.80132-1-sj@kernel.org Link: https://lore.kernel.org/20260921152443.80132-2-sj@kernel.org Fixes: ab9bda001b68 ("mm/damon/core: introduce address range type damos filter") Signed-off-by: Donggeun Yoo Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Brendan Higgins Cc: David Gow Cc: --- mm/damon/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index add1b7afb957..8d8b0cba1a56 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2784,6 +2784,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, } if (damos_core_filter_out(c, t, r, s)) return; + sz = damon_sz_region(r); ktime_get_coarse_ts64(&begin); trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses, damon_nr_regions(t), do_trace); From 1c335228206392b96d327c6b49cea68cbb806d8a Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Mon, 21 Sep 2026 08:24:42 -0700 Subject: [PATCH 531/539] mm/damon/tests/core-kunit: test the size charged for a filter-trimmed region damos_test_filter_out() checks that an address range filter splits a region at the filter boundary, and stops there. Nothing checks what damos_apply_scheme() then charges to the quota and reports as tried. damos_test_apply_scheme_filtered_sz() covers the two ways such a filter trims a region: one that starts before the filter's range, and one that starts inside it. damos_test_apply_scheme_filter_sz_unchanged() and damos_test_apply_scheme_quota_sz() cover the cases whose accounting must not change: a region wholly inside a reject range, one wholly inside an allow range, two filters trimming in a single call, a filter type that never splits, no filter at all, a region the quota trims, and a quota remainder too small for one region. Link: https://lore.kernel.org/20260921152443.80132-3-sj@kernel.org Signed-off-by: Donggeun Yoo Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Brendan Higgins Cc: David Gow Cc: --- mm/damon/tests/core-kunit.h | 316 ++++++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 5ff0436c5844..df84d9cc7d20 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1706,6 +1706,319 @@ out: damos_free_filter(f); } +static unsigned long damos_test_apply_scheme_stub(struct damon_ctx *c, + struct damon_target *t, struct damon_region *r, + struct damos *s, unsigned long *sz_filter_passed) +{ + return 0; +} + +static void damos_test_apply_scheme_filtered_sz(struct kunit *test) +{ + struct damos_access_pattern pattern = { + .min_sz_region = 0, + .max_sz_region = ULONG_MAX, + .min_nr_accesses = 0, + .max_nr_accesses = UINT_MAX, + .min_age_region = 0, + .max_age_region = UINT_MAX, + }; + unsigned long min_sz = DAMON_MIN_REGION_SZ; + struct damos_watermarks wmarks = {}; + struct damos_quota quota = {}; + struct damon_ctx *c; + struct damon_target *t; + struct damon_region *r; + struct damos_filter *f; + struct damos *s; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + c->ops.apply_scheme = damos_test_apply_scheme_stub; + + s = damon_new_scheme(&pattern, DAMOS_STAT, 0, "a, &wmarks, + NUMA_NO_NODE); + if (!s) { + damon_destroy_ctx(c); + kunit_skip(test, "scheme alloc fail"); + } + damon_add_scheme(c, s); + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + damon_add_target(c, t); + + f = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false); + if (!f) { + damon_destroy_ctx(c); + kunit_skip(test, "filter alloc fail"); + } + f->addr_range = (struct damon_addr_range){ + .start = 2 * min_sz, + .end = 8 * min_sz + }; + damos_add_filter(s, f); + damos_set_filters_default_reject(s); + + /* reject filter, region starting before the range */ + r = damon_new_region(0, 4 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + + if (damon_nr_regions(t) != 2) + goto out; + damon_destroy_region(damon_next_region(r), t); + damon_destroy_region(r, t); + s->stat = (struct damos_stat){}; + + /* allow filter, region starting inside the range */ + f->allow = true; + damos_set_filters_default_reject(s); + r = damon_new_region(2 * min_sz, 10 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + KUNIT_EXPECT_EQ(test, r->ar.end, 8 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 6 * min_sz); + +out: + damon_destroy_ctx(c); +} + +static void damos_test_apply_scheme_filter_sz_unchanged(struct kunit *test) +{ + struct damos_access_pattern pattern = { + .min_sz_region = 0, + .max_sz_region = ULONG_MAX, + .min_nr_accesses = 0, + .max_nr_accesses = UINT_MAX, + .min_age_region = 0, + .max_age_region = UINT_MAX, + }; + unsigned long min_sz = DAMON_MIN_REGION_SZ; + struct damos_watermarks wmarks = {}; + struct damos_quota quota = {}; + struct damos_filter *f, *f2; + struct damon_ctx *c; + struct damon_target *t; + struct damon_region *r; + struct damos *s; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + c->ops.apply_scheme = damos_test_apply_scheme_stub; + + s = damon_new_scheme(&pattern, DAMOS_STAT, 0, "a, &wmarks, + NUMA_NO_NODE); + if (!s) { + damon_destroy_ctx(c); + kunit_skip(test, "scheme alloc fail"); + } + damon_add_scheme(c, s); + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + damon_add_target(c, t); + + f = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false); + if (!f) { + damon_destroy_ctx(c); + kunit_skip(test, "filter alloc fail"); + } + f->addr_range = (struct damon_addr_range){ + .start = 2 * min_sz, .end = 8 * min_sz}; + damos_add_filter(s, f); + damos_set_filters_default_reject(s); + + /* wholly inside a reject range: not counted at all */ + r = damon_new_region(4 * min_sz, 6 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.nr_tried, 0); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 0); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1); + + /* wholly inside an allow range: counted whole, not split */ + f->allow = true; + damos_set_filters_default_reject(s); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1); + + /* two filters, each trimming: the size left after both is counted */ + damon_destroy_region(r, t); + s->stat = (struct damos_stat){}; + f->allow = false; + f->addr_range = (struct damon_addr_range){ + .start = 4 * min_sz, .end = 12 * min_sz}; + f2 = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false); + if (!f2) { + damon_destroy_ctx(c); + kunit_skip(test, "filter alloc fail"); + } + f2->addr_range = (struct damon_addr_range){ + .start = 2 * min_sz, .end = 3 * min_sz}; + damos_add_filter(s, f2); + damos_set_filters_default_reject(s); + + r = damon_new_region(0, 8 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3); + KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + + /* a core filter that never splits: counted whole */ + damos_destroy_filter(f2); + damos_destroy_filter(f); + f = damos_new_filter(DAMOS_FILTER_TYPE_TARGET, true, true); + if (!f) { + damon_destroy_ctx(c); + kunit_skip(test, "filter alloc fail"); + } + f->target_idx = 0; + damos_add_filter(s, f); + damos_set_filters_default_reject(s); + s->stat = (struct damos_stat){}; + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3); + KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + + damon_destroy_ctx(c); +} + +static void damos_test_apply_scheme_quota_sz(struct kunit *test) +{ + struct damos_access_pattern pattern = { + .min_sz_region = 0, + .max_sz_region = ULONG_MAX, + .min_nr_accesses = 0, + .max_nr_accesses = UINT_MAX, + .min_age_region = 0, + .max_age_region = UINT_MAX, + }; + unsigned long min_sz = DAMON_MIN_REGION_SZ; + struct damos_watermarks wmarks = {}; + struct damos_quota quota = {}; + struct damon_region *r, *next; + struct damon_ctx *c; + struct damon_target *t; + struct damos *s; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + + s = damon_new_scheme(&pattern, DAMOS_STAT, 0, "a, &wmarks, + NUMA_NO_NODE); + if (!s) { + damon_destroy_ctx(c); + kunit_skip(test, "scheme alloc fail"); + } + damon_add_scheme(c, s); + damos_set_filters_default_reject(s); + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + damon_add_target(c, t); + + /* no apply_scheme operation: the whole region is counted */ + r = damon_new_region(0, 4 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 4 * min_sz); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1); + + /* no filter and no quota: the whole region is counted */ + c->ops.apply_scheme = damos_test_apply_scheme_stub; + s->stat = (struct damos_stat){}; + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 4 * min_sz); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1); + + /* the quota trims the region: the trimmed size is counted */ + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + s->stat = (struct damos_stat){}; + s->quota.esz = 2 * min_sz; + s->quota.charged_sz = 0; + + r = damon_new_region(0, 8 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz); + + /* quota remainder below one region: tried, but nothing counted */ + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + s->stat = (struct damos_stat){}; + s->quota.esz = 1; + s->quota.charged_sz = 0; + + r = damon_new_region(0, 4 * min_sz); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + damon_add_region(r, t); + + damos_apply_scheme(c, t, r, s); + KUNIT_EXPECT_EQ(test, s->stat.nr_tried, 1); + KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 0); + KUNIT_EXPECT_EQ(test, r->ar.end, 4 * min_sz); + + damon_destroy_ctx(c); +} + static void damon_test_feed_loop_next_input(struct kunit *test) { unsigned long last_input = 900000, current_score = 200; @@ -1959,6 +2272,9 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_commit_ctx), KUNIT_CASE(damon_test_valid_probe_params), KUNIT_CASE(damos_test_filter_out), + KUNIT_CASE(damos_test_apply_scheme_filtered_sz), + KUNIT_CASE(damos_test_apply_scheme_filter_sz_unchanged), + KUNIT_CASE(damos_test_apply_scheme_quota_sz), KUNIT_CASE(damon_test_feed_loop_next_input), KUNIT_CASE(damon_test_set_filters_default_reject), KUNIT_CASE(damon_test_apply_min_nr_regions), From a9799d2d248533c6d3f8f8d1272a0d27871a8ebc Mon Sep 17 00:00:00 2001 From: Liew Rui Yan Date: Mon, 21 Sep 2026 08:15:43 -0700 Subject: [PATCH 532/539] mm/damon/core: skip quota score setup when the quota is full Patch series "mm/damon: improvements in efficiency, error handling, documents". Four various improvements. Patch 1 from Liew Rui Yan skips unnecessary quota setup work when relevant. Patch 2 from Xuewen Wang propagates ignored damon_call() error to sysfs users. Patch 3 from Adrian Huang (Lenovo) fixes typos in comments. Patch 4 from Karthikeyan KS clarifies zero sample interval acceptance on kernel-doc comment. This patch (of 4): In damos_adjust_quota(), the quota could already be full. In this situation, damos_adjust_quota() will still calculates quota->min_score. However, this min_score will not be used in this window, because in damon_do_apply_schemes(), damos_quota_is_full() will always returns true, preventing the scheme from being applied to any region. Therefore, add a short circuit for 'esz < min_region_sz' schemes to early return from damos_adjust_quota() before calculating min_score. Link: https://lore.kernel.org/20260921151547.78472-1-sj@kernel.org Link: https://lore.kernel.org/20260921151547.78472-2-sj@kernel.org Signed-off-by: Liew Rui Yan Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Adrian Huang Cc: Karthikeyan KS Cc: Xuewen Wang Cc: Zenghui Yu (Huawei) --- mm/damon/core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 8d8b0cba1a56..749897846b4c 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3385,6 +3385,8 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) damos_trace_esz(c, s, quota); } + if (damos_quota_is_full(quota, c->min_region_sz)) + return; if (!c->ops.get_scheme_score) return; From 639b0cca7d11d6c5ee539a371c818ab9b3e69645 Mon Sep 17 00:00:00 2001 From: Xuewen Wang Date: Mon, 21 Sep 2026 08:15:44 -0700 Subject: [PATCH 533/539] mm/damon/sysfs: propagate damon_call() error in turn_damon_on damon_sysfs_turn_damon_on() ignored the return value of damon_call() for the repeat call control registration and always returned the stale result of damon_start() (0 at that point). When damon_call() fails, e.g., the kdamond is already exiting, the user still gets success from the state file write while monitoring is not actually on. Save and return the damon_call() result instead. No rollback of damon_start() is needed since a failed damon_call() guarantees the context is stopped. Link: https://lore.kernel.org/20260921151547.78472-3-sj@kernel.org Signed-off-by: Xuewen Wang Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Adrian Huang Cc: Karthikeyan KS Cc: Liew Rui Yan Cc: Zenghui Yu (Huawei) --- mm/damon/sysfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 43519afb9eb7..70a4b64fb8ea 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2607,7 +2607,8 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond) repeat_call_control->data = kdamond; repeat_call_control->repeat = true; repeat_call_control->dealloc_on_cancel = true; - if (damon_call(ctx, repeat_call_control)) + err = damon_call(ctx, repeat_call_control); + if (err) kfree(repeat_call_control); return err; } From 76ba2763d594bb7ec9f01a64f7ae7ae8ecc02b37 Mon Sep 17 00:00:00 2001 From: "Adrian Huang (Lenovo)" Date: Mon, 21 Sep 2026 08:15:45 -0700 Subject: [PATCH 534/539] mm/damon: fix typos in comments Correct spelling mistakes. No functional changes. Link: https://lore.kernel.org/20260921151547.78472-4-sj@kernel.org Signed-off-by: Adrian Huang (Lenovo) Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: Zenghui Yu (Huawei) Reviewed-by: SJ Park Cc: Karthikeyan KS Cc: Liew Rui Yan Cc: Xuewen Wang --- mm/damon/core.c | 6 +++--- mm/damon/lru_sort.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 749897846b4c..733025b36745 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2486,7 +2486,7 @@ static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r, * This function checks if a given region should be skipped or not for the * reason. If only the starting part of the region has previously charged, * this function splits the region into two so that the second one covers the - * area that not charged in the previous charge widnow, and return true. The + * area that not charged in the previous charge window, and return true. The * caller can see the second one on the next iteration of the region walk. * Note that this means the caller should use damon_for_each_region() instead * of damon_for_each_region_safe(). If damon_for_each_region_safe() is used, @@ -2652,7 +2652,7 @@ static void damos_walk_call_walk(struct damon_ctx *ctx, struct damon_target *t, * This function is called when kdamond finished applying the action of a DAMOS * scheme to all regions that eligible for the given &damos->apply_interval_us. * If every scheme of @ctx including @s now finished walking for at least one - * &damos->apply_interval_us, this function makrs the handling of the given + * &damos->apply_interval_us, this function marks the handling of the given * DAMOS walk request is done, so that damos_walk() can wake up and return. */ static void damos_walk_complete(struct damon_ctx *ctx, struct damos *s) @@ -4194,7 +4194,7 @@ static bool damon_find_system_rams_range(unsigned long *start, * This function sets the region of @t as requested by @start and @end. If the * values of @start and @end are zero, however, this function finds 'System * RAM' resources and sets the region to cover all the resource. In the latter - * case, this function saves the start and the end addresseses of the first and + * case, this function saves the start and the end addresses of the first and * the last resources in @start and @end, respectively. * * Return: 0 on success, negative error code otherwise. diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index ad8e86dd3a93..273efa3c913e 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -260,7 +260,7 @@ static int damon_lru_sort_add_filters(struct damos *hot_scheme, return -ENOMEM; damos_add_filter(hot_scheme, filter); - /* disabllow de-prioritizing young pages */ + /* disallow de-prioritizing young pages */ filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, false); if (!filter) return -ENOMEM; From 6fbcf40619ad8de2617745d6aba563564898980d Mon Sep 17 00:00:00 2001 From: Karthikeyan KS Date: Mon, 21 Sep 2026 08:15:46 -0700 Subject: [PATCH 535/539] mm/damon: document that a zero sample_interval is accepted damon_set_attrs() accepts sample_interval == 0. This was reported as a bug in v1 of this patch (rejecting it in damon_set_attrs()). A similar patch was already declined for the same reason: a zero interval is intentionally supported [1]. Document the behavior instead of changing it. Link: https://lore.kernel.org/20260921151547.78472-5-sj@kernel.org Link: https://lore.kernel.org/all/20260722094304.3132750-1-dayou5941@163.com/ [1] Signed-off-by: Karthikeyan KS Signed-off-by: SJ Park Signed-off-by: Andrew Morton Reviewed-by: SJ Park Cc: Adrian Huang Cc: Liew Rui Yan Cc: Xuewen Wang Cc: Zenghui Yu (Huawei) --- include/linux/damon.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 836353c4ab9a..844b175120f0 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -836,7 +836,8 @@ struct damon_probe { /** * struct damon_attrs - Monitoring attributes for accuracy/overhead control. * - * @sample_interval: The time between access samplings. + * @sample_interval: The time between access samplings. Zero is + * accepted. * @aggr_interval: The time between monitor results aggregations. * @ops_update_interval: The time between monitoring operations updates. * @intervals_goal: Intervals auto-tuning goal. From 1b4bbae111f13f01732c04a6663af209772e259a Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Mon, 21 Sep 2026 05:30:26 -0700 Subject: [PATCH 536/539] mm: kmemleak: move the struct page scan into a helper Patch series "mm: kmemleak: batch the struct page scan". kmemleak walks the struct page array as a scan root and hands it to scan_block() one page at a time. scan_block() takes kmemleak_lock with interrupts disabled for the duration of the call, so the scanner acquires the lock once per online PFN in order to look at a single 64-byte struct page. Batching adjacent pages up to that size keeps exactly the same pages under the same maximum lock hold time, with MAX_SCAN_SIZE / pagesize fewer acquisitions. Patch 1 lifts the loop out of __kmemleak_scan() into scan_zone_pages() with no functional change. Patch 2 does the batching, which is then contained in that one function. This improves the performance due to less atomic operations, which is not a big deal on a regular machine, but, given kmemleak usually comes with extra debug options, such as PROVE_LOCKING, DEBUG_SPINLOCK, etc. For instance, measuring Meta's "debug kernel flavor" on an arm64 hosts, this improve the scan time by 20%. It is safe to get more work into scan_block(), given it has the protections, added by commit eb11f56eeca560 ("mm/kmemleak: stop the task stack scan early when interrupted") MAX_SCAN_SIZE is also not a new maximum for a single scan_block() call. kmemleak_scan_task_stacks() already hands it a whole task stack in one go. On arm64 and x86_64 THREAD_SIZE is never below 16 KiB, four times MAX_SCAN_SIZE, and it is 64 KiB on arm64 with 64K pages. This patch (of 2): The struct page scanning loop sits inline in __kmemleak_scan(), nested three levels deep and sharing the caller's "stop" variable with the zone walk around it. Move it into scan_zone_pages(), which scans one zone and returns 1 if scanning should stop. No functional change; this only makes room for changing how the pages are handed to scan_block(). Link: https://lore.kernel.org/20260921-b4-kmemleak-page-scan-v1-1-fb97d4801b3a@debian.org Signed-off-by: Breno Leitao Signed-off-by: Andrew Morton Reviewed-by: Catalin Marinas --- mm/kmemleak.c | 57 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 5d0daea93c47..fe34c5650f42 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1855,6 +1855,39 @@ static void dedup_flush(struct xarray *dedup) } } +/* + * Scan the struct pages of a zone, skipping memory holes, pages that belong to + * another zone and pages that are not in use. Returns 1 if the scan should be + * stopped. + */ +static int scan_zone_pages(struct zone *zone) +{ + unsigned long start_pfn = zone->zone_start_pfn; + unsigned long end_pfn = zone_end_pfn(zone); + unsigned long pfn; + + for (pfn = start_pfn; pfn < end_pfn; pfn++) { + struct page *page = pfn_to_online_page(pfn); + + if (!(pfn & 63)) + cond_resched_tasks_rcu_qs(); + + if (!page) + continue; + + /* only scan pages belonging to this zone */ + if (page_zone(page) != zone) + continue; + /* only scan if page is in use */ + if (page_count(page) == 0) + continue; + if (scan_block(page, page + 1, NULL)) + return 1; + } + + return 0; +} + /* * Scan data sections and all the referenced memory blocks allocated via the * kernel's standard allocators. This function must be called with the @@ -1928,29 +1961,7 @@ static int __kmemleak_scan(bool full) */ get_online_mems(); for_each_populated_zone(zone) { - unsigned long start_pfn = zone->zone_start_pfn; - unsigned long end_pfn = zone_end_pfn(zone); - unsigned long pfn; - - for (pfn = start_pfn; pfn < end_pfn; pfn++) { - struct page *page = pfn_to_online_page(pfn); - - if (!(pfn & 63)) - cond_resched_tasks_rcu_qs(); - - if (!page) - continue; - - /* only scan pages belonging to this zone */ - if (page_zone(page) != zone) - continue; - /* only scan if page is in use */ - if (page_count(page) == 0) - continue; - stop = scan_block(page, page + 1, NULL); - if (stop) - break; - } + stop = scan_zone_pages(zone); if (stop) break; } From 213753c71e098a5bca7866a147bfd262cf9d347c Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Mon, 21 Sep 2026 05:30:27 -0700 Subject: [PATCH 537/539] mm: kmemleak: scan the struct page array in MAX_SCAN_SIZE batches scan_zone_pages() scans the struct page array one page per scan_block() call: if (scan_block(page, page + 1, NULL)) scan_block() takes kmemleak_lock with interrupts disabled for the duration of the call, so this acquires the lock once per online PFN to scan a single struct page. Gather runs of adjacent eligible struct pages and pass each run to scan_block() in one call, capped at MAX_SCAN_SIZE. The longest kmemleak_lock hold duration is now MAX_SCAN_SIZE worth of bytes. The same bound scan_large_block() already applies to the data sections and the per-CPU areas. This can make the scan faster. On an arm64 VM with 24 GiB and ~17 GiB in use, the struct page phase goes from 4390k scan_block() calls down to 69k for the same 4390k pages scanned, and the whole scan about 20% faster (on debug kernel). The win is in the per-acquisition cost, so on a kernel built without the lock debugging options the scan time is unchanged. I don't think it will be a problem doing more on scan_block(), given we have the scan_should_stop() protection. Coverage is unchanged: a temporary assertion comparing the number of eligible pages against the number actually passed to scan_block() matched on every zone of every scan, including while memory was being freed underneath the scan. Link: https://lore.kernel.org/20260921-b4-kmemleak-page-scan-v1-2-fb97d4801b3a@debian.org Signed-off-by: Breno Leitao Signed-off-by: Andrew Morton Reviewed-by: Catalin Marinas --- mm/kmemleak.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index fe34c5650f42..20c0eb41c873 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1862,8 +1862,11 @@ static void dedup_flush(struct xarray *dedup) */ static int scan_zone_pages(struct zone *zone) { + const unsigned int max_batch = MAX_SCAN_SIZE / sizeof(struct page); unsigned long start_pfn = zone->zone_start_pfn; unsigned long end_pfn = zone_end_pfn(zone); + struct page *first = NULL, *last = NULL; + unsigned int batch = 0; unsigned long pfn; for (pfn = start_pfn; pfn < end_pfn; pfn++) { @@ -1872,19 +1875,29 @@ static int scan_zone_pages(struct zone *zone) if (!(pfn & 63)) cond_resched_tasks_rcu_qs(); - if (!page) - continue; + /* only scan in-use pages belonging to this zone */ + if (page && (page_zone(page) != zone || + page_count(page) == 0)) + page = NULL; - /* only scan pages belonging to this zone */ - if (page_zone(page) != zone) + if (page && first && page == last + 1 && + batch < max_batch) { + last = page; + batch++; continue; - /* only scan if page is in use */ - if (page_count(page) == 0) - continue; - if (scan_block(page, page + 1, NULL)) + } + + if (first && scan_block(first, last + 1, NULL)) return 1; + + first = page; + last = page; + batch = page ? 1 : 0; } + if (first && scan_block(first, last + 1, NULL)) + return 1; + return 0; } From 664208a30604cb0e3ae35b65a7c535dbf3094fc8 Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Sun, 20 Sep 2026 21:25:19 +0800 Subject: [PATCH 538/539] mm: vmscan: put rotation-missed folios at the LRU tail The page reclaim isolates a batch of folios from the tail of an LRU list and works on them one by one. For a suitable swap-backed folio on an async swap device, it queues the folio for writeback and, after finishing the batch, puts the folio back to the head of the original LRU list. Meanwhile the page writeback flushes the queued folios in its own, independent batches. For each folio it writes back it calls folio_rotate_reclaimable(), which tries to rotate the folio to the LRU tail. But folio_rotate_reclaimable() only takes effect once the folio has been put back by reclaim. If the async swap device is fast enough, the writeback can complete a folio while reclaim is still working on the rest of the batch that contains it. In that case the folio stays near the head and reclaim will not revisit it before wrapping around, causing a cold/hot inversion: a clean, written-back folio that should be a prime reclaim candidate is kept ahead of hotter folios. commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") addressed this for MGLRU only. The traditional active/inactive LRU has the same problem, reported at [1]. A reproducer is available at [2]. Rather than re-reclaiming those folios (which would drop the swap cache that may still be useful for a future hit [4]), restore the rotation that was missed: when move_folios_to_lru() puts a folio back, add it to the LRU tail if it looks like it missed folio_rotate_reclaimable() (inactive, not mapped, not dirty and not under writeback). A referenced folio is left at the head so it still gets a second chance, and a folio with an unexpected reference (e.g. a GUP or speculative pin) is left at the head because it cannot be reclaimed yet anyway. A new do_rotate parameter gates this so it only applies on the reclaim put-back path (shrink_inactive_list()), not on shrink_active_list() where the list order is already deliberate. This approach was suggested by Barry Song [3]. Only the traditional LRU is handled here. MGLRU already retries such folios via its own clean-list retry pass in evict_folios(), so it is left unchanged. The same do_rotate scheme could later replace that retry pass to unify both LRUs, which is left for a follow-up. Test result with [2]: Without patch: cat memory.usage_in_bytes 1073700864 cat memory.memsw.usage_in_bytes 1413124096 free -h total used free Mem: 1.6Gi 1.2Gi 299Mi Swap: 1.0Gi 678Mi 346Mi With patch: cat memory.usage_in_bytes 1071140864 cat memory.memsw.usage_in_bytes 1413423104 free -h total used free Mem: 1.6Gi 1.2Gi 322Mi Swap: 1.0Gi 328Mi 695Mi After applying the patch, the difference between memory.memsw.usage_in_bytes and memory.usage_in_bytes is close to the swap "used" value reported by 'free -h'. Link: https://lore.kernel.org/20260920132519.3369946-1-ridong.chen@linux.dev Link: https://lore.kernel.org/linux-kernel/20241010081802.290893-1-chenridong@huaweicloud.com/ [1] Link: https://lore.kernel.org/lkml/46037a37-4cf6-448e-a94b-30a4d16e8814@linux.dev/ [2] Link: https://lore.kernel.org/linux-mm/20260911121341.178028-1-alex@ghiti.fr/ [4] Signed-off-by: Ridong Chen Signed-off-by: Andrew Morton Suggested-by: Barry Song Link: https://lore.kernel.org/lkml/CAGsJ_4zwP3_+EYY5Ug9EJ+yD1UdxsBSGr25u8s1K3u_i7LH3Zg@mail.gmail.com/ [3] Reviewed-by: Barry Song Reviewed-by: Baolin Wang Cc: Johannes Weiner Cc: David Hildenbrand Cc: Michal Hocko Cc: Qi Zheng Cc: Shakeel Butt Cc: Lorenzo Stoakes Cc: Kairui Song Cc: Axel Rasmussen Cc: Yuanchu Xie Cc: Wei Xu Cc: Baoquan He --- mm/vmscan.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 01a1a47417fb..37467004336b 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1971,7 +1971,7 @@ static bool too_many_isolated(struct pglist_data *pgdat, int file, * * Note: The caller must not hold any lruvec lock. */ -static unsigned int move_folios_to_lru(struct list_head *list) +static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate) { int nr_pages, nr_moved = 0; struct lruvec *lruvec = NULL; @@ -2018,7 +2018,19 @@ static unsigned int move_folios_to_lru(struct list_head *list) continue; } - lruvec_add_folio(lruvec, folio); + /* + * Put clean, unreferenced and unpinned folios that may have + * missed folio_rotate_reclaimable() at the tail to avoid + * cold/hot inversion. + */ + if (do_rotate && !folio_test_active(folio) && !folio_mapped(folio) && + !folio_test_dirty(folio) && !folio_test_writeback(folio) && + !folio_test_referenced(folio) && + folio_ref_count(folio) == folio_expected_ref_count(folio)) + lruvec_add_folio_tail(lruvec, folio); + else + lruvec_add_folio(lruvec, folio); + nr_pages = folio_nr_pages(folio); nr_moved += nr_pages; if (folio_test_active(folio)) @@ -2135,7 +2147,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false, lruvec_memcg(lruvec)); - move_folios_to_lru(&folio_list); + move_folios_to_lru(&folio_list, true); mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), stat.nr_demoted); @@ -2246,8 +2258,8 @@ static void shrink_active_list(unsigned long nr_to_scan, /* * Move folios back to the lru list. */ - nr_activate = move_folios_to_lru(&l_active); - nr_deactivate = move_folios_to_lru(&l_inactive); + nr_activate = move_folios_to_lru(&l_active, false); + nr_deactivate = move_folios_to_lru(&l_inactive, false); count_vm_events(PGDEACTIVATE, nr_deactivate); count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); @@ -5114,7 +5126,7 @@ retry: folio_set_active(folio); } - move_folios_to_lru(&list); + move_folios_to_lru(&list, false); walk = current->reclaim_state->mm_walk; if (walk && walk->batched) { From 2967f3ccdb5fae956f1b085370ec292bab5c7e25 Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Sun, 11 Oct 2026 12:59:56 +0800 Subject: [PATCH 539/539] mm: shmem: assign a lock class to the unused huge shrinklist The unused huge shrinklist shares the default list_lru lock class with the dentry LRU. Although they use distinct lock instances, lockdep conflates their dependencies and reports a circular locking dependency: WARNING: possible circular locking dependency detected Give the unused huge shrinklist a dedicated lock class to fix this false-positive lockdep warning. Link: https://lore.kernel.org/20261011045956.30277-1-qi.zheng@linux.dev Fixes: b9b1861d4f7a ("mm: shmem: make unused huge shrinker memcg aware") Signed-off-by: Qi Zheng Signed-off-by: Andrew Morton Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202610091403.d1d8080b-lkp@intel.com Cc: Hugh Dickins Cc: Baolin Wang Cc: Usama Arif --- mm/shmem.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mm/shmem.c b/mm/shmem.c index 07b2855dfb7b..9f0965b8b59c 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -5160,6 +5160,9 @@ static const struct dentry_operations shmem_ci_dentry_ops = { static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + static struct lock_class_key shmem_unused_huge_lock_key; +#endif struct shmem_options *ctx = fc->fs_private; struct inode *inode; struct shmem_sb_info *sbinfo; @@ -5238,7 +5241,8 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) goto failed; #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (list_lru_init_memcg(&sbinfo->shrinklist, sb->s_shrink)) + if (list_lru_init_memcg_key(&sbinfo->shrinklist, sb->s_shrink, + &shmem_unused_huge_lock_key)) goto failed; #endif