Merge tag 'mm-hotfixes-stable-2026-11-17-19-55' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

Pull MM fixes from Andrew Morton:

 - "maple_tree fixes for 7.3" (Liam R. Howlett):

   Fix an RCU reader NULL dereference when the tree is emptied during a
   walk, including /proc/PID/maps racing with munmap(). Add tests and
   make debug WARN and BUG checks report failures properly

 - "mm/kasan: Detect double-free involving kfree_rcu_sheaf()" (Kuniyuki
   Iwashima):

   Detect double-free when an object enters an RCU sheaf. KASAN could
   previously miss it until the object was reused, leading to a later
   free of a live object. Add KUnit tests for the different free orders

 - "mm: Fix page-table teardown during memory hot-remove" (Muchun Song):

   Balance page-table constructors and destructors when hot-remove frees
   tables on x86-64, RISC-V and arm64. This fixes NR_PAGETABLE
   accounting for tables with and without constructors

 - "mm/damon: fix watermark state, kunit pid leak and selftest fake
   pass" (SJ Park):

   Recompute watermark wait state when DAMON schemes are removed, so
   kdamond resumes monitoring. Fix a pid reference leak in KUnit and a
   selftest that reports a skipped run as a pass

 - An XArray iterator could step backwards when a large page-cache entry
   was split, sending filemap_map_pages() outside the PTE page it
   locked. The XArray correction and a filemap bound check prevent
   out-of-bounds accesses and use-after-free. Another XArray fix
   prevents lockless walkers from following a parent pointer into a
   freed node

 - Userfaultfd now validates the whole write-protection range before
   changing any VMA, avoiding partial unprotect without waking blocked
   faults. Failed unmaps now abort queued UFFD_EVENT_UNMAPs; previously
   they could report a nonexistent unmap, leak the userfaultfd context
   and leave later operations returning -EAGAIN. A fork fix clears stale
   uffd PTE bits if the new context is released before its event is
   read

 - Moving the hugetlbfs inode trace event before RCU freeing avoids a
   use-after-free of its superblock. Shmem swapin no longer retries
   forever on reused swap slots containing a mismatched large swap-cache
   folio. Swapoff also avoids a cluster-list loop that can soft-lock
   reclaim and leave swapoff hung

 - Weighted interleave recalculation now uses one node set throughout,
   avoiding a divide-by-zero during memory hotplug. Hwpoison holds the
   folio lock when inspecting its mapping and no longer leaks a folio
   reference; it also updates poison statistics before notifying
   rasdaemon through a trace event

 - Cross-node migration now moves NR_FILE_THPS and NR_KERNEL_FILE_PAGES
   with page-cache folios, preventing per-node and memcg-node accounting
   from retaining stale counts or underflowing after the folios are
   freed

* tag 'mm-hotfixes-stable-2026-11-17-19-55' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (27 commits)
  xarray: fix index jumping backwards in xas_find()
  userfaultfd: validate the whole range before changing write protection
  maple_tree: update internal warning and bug_on code
  maple_tree: test emptying tree while RCU reading
  maple_tree: fix rcu reader NULL dereference after the tree is emptied
  mm/vma: read anon_vma carefully in validate_mm()
  hugetlbfs: move hugetlbfs_free_inode tracepoint to ->destroy_inode()
  selftests/damon/sysfs_memcg_path_leak: define ksft_skip
  mm/damon/tests/sysfs-kunit: fix leaked pid reference in add_targets test
  mm/damon/core: reset watermark wait state on each iteration
  mm/filemap: stop fault-around when the walk steps back
  mm: memory-failure: update statistics before emitting the trace event
  mm/vma: undo userfaultfd unmap preparation when unmapping fails
  userfaultfd: clear uffd state in child PTEs if uffd is released during fork
  mm/memory-failure: fix folio refcount leak and min_order_for_split() locking
  mm/shmem: fix swapin livelock on a mismatched large swap cache folio
  mm/mempolicy: use one node set for interleave weights
  xarray: clear the new head's parent before freeing the old head
  arm64/mm: fix destructor for unconstructed hotplug page tables
  riscv/mm: fix missing destructor for hotplug PUD tables
  ...
This commit is contained in:
Linus Torvalds
2026-11-18 09:01:16 -08:00
32 changed files with 859 additions and 209 deletions
+1 -2
View File
@@ -1495,8 +1495,7 @@ static void free_hotplug_page_range(struct page *page, size_t size,
static void free_hotplug_pgtable_page(struct page *page)
{
pagetable_dtor(page_ptdesc(page));
free_hotplug_page_range(page, PAGE_SIZE, NULL);
pagetable_free(page_ptdesc(page));
}
static bool pgtable_range_aligned(unsigned long start, unsigned long end,
+13 -19
View File
@@ -1501,10 +1501,17 @@ struct execmem_info __init *execmem_arch_setup(void)
#endif /* CONFIG_EXECMEM */
#ifdef CONFIG_MEMORY_HOTPLUG
static void __meminit free_pgtable_page(struct page *page)
{
if (PageReserved(page))
free_reserved_page(page);
else
pagetable_free(page_ptdesc(page));
}
static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
{
struct page *page = pmd_page(*pmd);
struct ptdesc *ptdesc = page_ptdesc(page);
pte_t *pte;
int i;
@@ -1514,18 +1521,13 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
return;
}
pagetable_dtor(ptdesc);
if (PageReserved(page))
free_reserved_page(page);
else
pagetable_free(ptdesc);
free_pgtable_page(page);
pmd_clear(pmd);
}
static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud, bool is_vmemmap)
static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)
{
struct page *page = pud_page(*pud);
struct ptdesc *ptdesc = page_ptdesc(page);
pmd_t *pmd;
int i;
@@ -1535,12 +1537,7 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud, bool is_vmemm
return;
}
if (!is_vmemmap)
pagetable_dtor(ptdesc);
if (PageReserved(page))
free_reserved_page(page);
else
pagetable_free(ptdesc);
free_pgtable_page(page);
pud_clear(pud);
}
@@ -1556,10 +1553,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d)
return;
}
if (PageReserved(page))
free_reserved_page(page);
else
__free_pages(page, 0);
free_pgtable_page(page);
p4d_clear(p4d);
}
@@ -1660,7 +1654,7 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un
remove_pmd_mapping(pmd_base, addr, next, is_vmemmap, altmap);
if (pgtable_l4_enabled())
free_pmd_table(pmd_base, pudp, is_vmemmap);
free_pmd_table(pmd_base, pudp);
}
}
+1 -1
View File
@@ -1187,12 +1187,12 @@ static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
static void hugetlbfs_free_inode(struct inode *inode)
{
trace_hugetlbfs_free_inode(inode);
kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
}
static void hugetlbfs_destroy_inode(struct inode *inode)
{
trace_hugetlbfs_destroy_inode(inode);
hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
}
+11 -4
View File
@@ -180,10 +180,12 @@ static __always_inline void * __must_check kasan_init_slab_obj(
}
bool __kasan_slab_pre_free(struct kmem_cache *s, void *object,
unsigned long ip);
unsigned long ip, bool after_rcu_delay);
/**
* kasan_slab_pre_free - Check whether freeing a slab object is safe.
* @s: Cache the object belongs to.
* @object: Object to be freed.
* @after_rcu_delay: Whether the free follows an RCU grace period.
*
* This function checks whether freeing the given object is safe. It may
* check for double-free and invalid-free bugs and report them.
@@ -193,10 +195,12 @@ bool __kasan_slab_pre_free(struct kmem_cache *s, void *object,
* @Return true if freeing the object is unsafe; false otherwise.
*/
static __always_inline bool kasan_slab_pre_free(struct kmem_cache *s,
void *object)
void *object,
bool after_rcu_delay)
{
if (kasan_enabled())
return __kasan_slab_pre_free(s, object, _RET_IP_);
return __kasan_slab_pre_free(s, object, _RET_IP_,
after_rcu_delay);
return false;
}
@@ -424,7 +428,8 @@ static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
return (void *)object;
}
static inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object)
static inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object,
bool after_rcu_delay)
{
return false;
}
@@ -495,6 +500,7 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
void kasan_cache_shrink(struct kmem_cache *cache);
void kasan_cache_shutdown(struct kmem_cache *cache);
void kasan_record_aux_stack(void *ptr);
void kasan_mark_rcu_free_pending(void *ptr);
#else /* CONFIG_KASAN_GENERIC */
@@ -512,6 +518,7 @@ static inline void kasan_cache_create(struct kmem_cache *cache,
static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
static inline void kasan_record_aux_stack(void *ptr) {}
static inline void kasan_mark_rcu_free_pending(void *ptr) {}
#endif /* CONFIG_KASAN_GENERIC */
+24 -24
View File
@@ -684,7 +684,7 @@ void mt_cache_shrink(void);
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
BUG_ON(1); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
@@ -700,7 +700,7 @@ void mt_cache_shrink(void);
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
BUG_ON(1); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
@@ -717,7 +717,7 @@ void mt_cache_shrink(void);
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
BUG_ON(1); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
@@ -726,16 +726,16 @@ void mt_cache_shrink(void);
#define MT_WARN_ON(__tree, __x) ({ \
int ret = !!(__x); \
atomic_inc(&maple_tree_tests_run); \
if (ret) { \
pr_info("WARN at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
if (!ret) { \
atomic_inc(&maple_tree_tests_passed); \
} else if (__ONCE_LITE_IF(ret)) { \
pr_info("WARN at %s:%d\n", \
__func__, __LINE__); \
mt_dump(__tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
WARN_ON(1); \
} \
unlikely(ret); \
})
@@ -743,17 +743,17 @@ void mt_cache_shrink(void);
#define MAS_WARN_ON(__mas, __x) ({ \
int ret = !!(__x); \
atomic_inc(&maple_tree_tests_run); \
if (ret) { \
pr_info("WARN at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
if (!ret) { \
atomic_inc(&maple_tree_tests_passed); \
} else if (__ONCE_LITE_IF(ret)) { \
pr_info("WARN at %s:%d\n", \
__func__, __LINE__); \
mas_dump(__mas); \
mt_dump((__mas)->tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
WARN_ON(1); \
} \
unlikely(ret); \
})
@@ -761,18 +761,18 @@ void mt_cache_shrink(void);
#define MAS_WR_WARN_ON(__wrmas, __x) ({ \
int ret = !!(__x); \
atomic_inc(&maple_tree_tests_run); \
if (ret) { \
pr_info("WARN at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
if (!ret) { \
atomic_inc(&maple_tree_tests_passed); \
} else if (__ONCE_LITE_IF(ret)) { \
pr_info("WARN at %s:%d\n", \
__func__, __LINE__); \
mas_wr_dump(__wrmas); \
mas_dump((__wrmas)->mas); \
mt_dump((__wrmas)->mas->tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
WARN_ON(1); \
} \
unlikely(ret); \
})
@@ -780,9 +780,9 @@ void mt_cache_shrink(void);
#define MT_BUG_ON(__tree, __x) BUG_ON(__x)
#define MAS_BUG_ON(__mas, __x) BUG_ON(__x)
#define MAS_WR_BUG_ON(__mas, __x) BUG_ON(__x)
#define MT_WARN_ON(__tree, __x) WARN_ON(__x)
#define MAS_WARN_ON(__mas, __x) WARN_ON(__x)
#define MAS_WR_WARN_ON(__mas, __x) WARN_ON(__x)
#define MT_WARN_ON(__tree, __x) WARN_ON_ONCE(__x)
#define MAS_WARN_ON(__mas, __x) WARN_ON_ONCE(__x)
#define MAS_WR_WARN_ON(__mas, __x) WARN_ON_ONCE(__x)
#endif /* CONFIG_DEBUG_MAPLE_TREE */
/**
+20 -19
View File
@@ -4032,23 +4032,6 @@ static inline void pagetable_free_kernel(struct ptdesc *pt)
__pagetable_free(pt);
}
#endif
/**
* pagetable_free - Free pagetables
* @pt: The page table descriptor
*
* pagetable_free frees the memory of all page tables described by a page
* table descriptor and the memory for the descriptor itself.
*/
static inline void pagetable_free(struct ptdesc *pt)
{
if (ptdesc_test_kernel(pt)) {
ptdesc_clear_kernel(pt);
pagetable_free_kernel(pt);
} else {
__pagetable_free(pt);
}
}
#if defined(CONFIG_SPLIT_PTE_PTLOCKS)
#if ALLOC_SPLIT_PTLOCKS
void __init ptlock_cache_init(void);
@@ -4141,6 +4124,26 @@ static inline void pagetable_dtor(struct ptdesc *ptdesc)
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
/**
* pagetable_free - Free pagetables
* @pt: The page table descriptor
*
* pagetable_free frees the memory of all page tables described by a page
* table descriptor and the memory for the descriptor itself.
*/
static inline void pagetable_free(struct ptdesc *pt)
{
if (PageTable(ptdesc_page(pt)))
pagetable_dtor(pt);
if (ptdesc_test_kernel(pt)) {
ptdesc_clear_kernel(pt);
pagetable_free_kernel(pt);
} else {
__pagetable_free(pt);
}
}
static inline void pagetable_dtor_free(struct ptdesc *ptdesc)
{
pagetable_dtor(ptdesc);
@@ -4428,9 +4431,7 @@ anon_rmap_tree_iter_first(struct anon_vma *anon_vma,
struct anon_vma_chain *
anon_rmap_tree_iter_next(struct anon_vma_chain *avc,
pgoff_t pgoff_start, pgoff_t pgoff_last);
#ifdef CONFIG_DEBUG_VM_RB
void anon_rmap_tree_verify(struct anon_vma_chain *avc);
#endif
#define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \
for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \
+5
View File
@@ -277,6 +277,7 @@ extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
unsigned long start, unsigned long end, struct list_head *uf);
extern void userfaultfd_unmap_complete(struct mm_struct *mm,
struct list_head *uf);
void userfaultfd_unmap_abort(struct list_head *uf);
extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma);
extern bool userfaultfd_wp_async(struct vm_area_struct *vma);
extern bool userfaultfd_rwp_async(struct vm_area_struct *vma);
@@ -443,6 +444,10 @@ static inline void userfaultfd_unmap_complete(struct mm_struct *mm,
{
}
static inline void userfaultfd_unmap_abort(struct list_head *uf)
{
}
static inline bool uffd_disable_fault_around(struct vm_area_struct *vma)
{
return false;
+1 -1
View File
@@ -72,7 +72,7 @@ DEFINE_EVENT(hugetlbfs__inode, hugetlbfs_evict_inode,
TP_ARGS(inode)
);
DEFINE_EVENT(hugetlbfs__inode, hugetlbfs_free_inode,
DEFINE_EVENT(hugetlbfs__inode, hugetlbfs_destroy_inode,
TP_PROTO(struct inode *inode),
+93 -27
View File
@@ -3891,14 +3891,64 @@ retry:
goto retry;
}
static __always_inline bool mas_rewalk_if_dead(struct ma_state *mas,
struct maple_node *node, const unsigned long index)
/*
* mas_next_no_node() - Finish a next walk that restarted into a tree without
* nodes.
* @mas: The inactive maple state (ma_none or ma_root)
*
* The arrival here is either because there is a root value or nothing. Either
* way, the next entry is nothing. So set the limits and overwrite the status.
*
* Return: NULL
*/
static inline void *mas_next_no_node(struct ma_state *mas)
{
if (unlikely(ma_dead_node(node))) {
mas_rewalk(mas, index);
return true;
if (mas_is_ptr(mas))
mas->index = 1;
else
mas->index = 0;
mas->last = ULONG_MAX;
mas->status = ma_none;
return NULL;
}
/*
* mas_prev_no_node() - Finish a prev walk that restarted into a tree without
* nodes.
* @mas: The inactive maple state (ma_none or ma_root)
* @index: The start of the range the walk left
* @min: The lowest index to return
*
* The arrival here is either because there is a root value or nothing. With
* nothing, leave the limits mas_walk() sets for an empty tree. A root value
* sits at 0: stop on it as mas_prev_setup() does, unless the walk was at 0.
*
* Return: The root entry if @index and @min allow it, NULL otherwise. A node
* if the root changed again since the walk; the caller restarts.
*/
static inline void *mas_prev_no_node(struct ma_state *mas, unsigned long index,
unsigned long min)
{
void *entry;
mas->index = 0;
if (mas_is_none(mas)) {
mas->last = ULONG_MAX;
return NULL;
}
return false;
mas->last = 0;
if (!index || min)
return NULL;
entry = mas_root(mas);
if (!entry) {
mas->status = ma_none;
mas->last = ULONG_MAX;
}
return entry;
}
/*
@@ -4006,14 +4056,14 @@ retry:
node = mas_mn(mas);
type = mte_node_type(mas->node);
pivots = ma_pivots(node, type);
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (unlikely(ma_dead_node(node)))
goto dead;
if (mas->min <= min) {
pivot = mas_safe_min(mas, pivots, mas->offset);
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (unlikely(ma_dead_node(node)))
goto dead;
if (pivot <= min)
goto underflow;
@@ -4028,10 +4078,8 @@ again:
if (mas->index <= min)
goto underflow;
if (mas_prev_node(mas, min)) {
mas_rewalk(mas, save_point);
goto retry;
}
if (mas_prev_node(mas, min))
goto dead;
if (WARN_ON_ONCE(mas_is_underflow(mas)))
return NULL;
@@ -4045,8 +4093,8 @@ again:
slots = ma_slots(node, type);
entry = mas_slot(mas, slots, mas->offset);
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (unlikely(ma_dead_node(node)))
goto dead;
if (likely(entry))
return entry;
@@ -4063,6 +4111,17 @@ again:
underflow:
mas->status = ma_underflow;
return NULL;
dead:
mas_rewalk(mas, save_point);
if (likely(mas_is_active(mas)))
goto retry;
entry = mas_prev_no_node(mas, save_point, min);
if (unlikely(xa_is_node(entry))) /* The root changed again */
goto dead;
return entry;
}
/*
@@ -4176,8 +4235,8 @@ retry:
node = mas_mn(mas);
type = mte_node_type(mas->node);
pivots = ma_pivots(node, type);
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (unlikely(ma_dead_node(node)))
goto dead;
if (mas->max >= max) {
if (likely(mas->offset < mas->end))
@@ -4185,8 +4244,8 @@ retry:
else
pivot = mas->max;
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (unlikely(ma_dead_node(node)))
goto dead;
if (pivot >= max) { /* Was at the limit, next will extend beyond */
mas->status = ma_overflow;
@@ -4208,10 +4267,8 @@ again:
return NULL;
}
if (mas_next_node(mas, node, max)) {
mas_rewalk(mas, save_point);
goto retry;
}
if (mas_next_node(mas, node, max))
goto dead;
if (WARN_ON_ONCE(mas_is_overflow(mas)))
return NULL;
@@ -4226,8 +4283,8 @@ again:
slots = ma_slots(node, type);
entry = mt_slot(mas->tree, slots, mas->offset);
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (unlikely(ma_dead_node(node)))
goto dead;
if (entry)
return entry;
@@ -4244,6 +4301,13 @@ again:
}
return entry;
dead:
mas_rewalk(mas, save_point);
if (likely(mas_is_active(mas)))
goto retry;
return mas_next_no_node(mas);
}
/*
@@ -5450,7 +5514,9 @@ void *mas_find(struct ma_state *mas, unsigned long max)
/* Retries on dead nodes handled by mas_next_slot */
entry = mas_next_slot(mas, max, false);
/* Ignore overflow */
mas->status = ma_active;
if (mas_is_overflow(mas))
mas->status = ma_active;
return entry;
}
EXPORT_SYMBOL_GPL(mas_find);
+165
View File
@@ -3740,6 +3740,170 @@ static noinline void __init check_range64_in_rcu(struct maple_tree *mt)
rcu_barrier();
}
/*
* A reader that restarts after its leaf died may find a tree without nodes:
* emptied, or emptied and then given a single entry at 0. Each case starts
* from a tree with nodes, built by check_seq(). The stores run while the
* reader holds rcu_read_lock, so they use GFP_ATOMIC.
*/
static noinline void __init check_restart_no_node(struct maple_tree *mt)
{
void *ptr = xa_mk_value(1234);
void *entry;
MA_STATE(mas, mt, 0, 0);
MT_BUG_ON(mt, !mtree_empty(mt));
mt_set_non_kernel(99999);
/* find: the tree is emptied */
mt_init_flags(mt, MT_FLAGS_USE_RCU);
check_seq(mt, 100, false);
rcu_read_lock();
mas_set(&mas, 0);
entry = mas_find(&mas, ULONG_MAX);
MT_BUG_ON(mt, entry != xa_mk_value(0));
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
entry = mas_find(&mas, ULONG_MAX);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_none);
MT_BUG_ON(mt, mas.index != 0);
MT_BUG_ON(mt, mas.last != ULONG_MAX);
entry = mas_find(&mas, ULONG_MAX);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_none);
rcu_read_unlock();
mtree_destroy(mt);
rcu_barrier();
/* next: the tree is emptied */
mt_init_flags(mt, MT_FLAGS_USE_RCU);
check_seq(mt, 100, false);
rcu_read_lock();
mas_set(&mas, 0);
entry = mas_find(&mas, ULONG_MAX);
MT_BUG_ON(mt, entry != xa_mk_value(0));
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
entry = mas_next(&mas, ULONG_MAX);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_none);
MT_BUG_ON(mt, mas.index != 0);
MT_BUG_ON(mt, mas.last != ULONG_MAX);
rcu_read_unlock();
mtree_destroy(mt);
rcu_barrier();
/* find: the tree is emptied, then holds one entry at 0 */
mt_init_flags(mt, MT_FLAGS_USE_RCU);
check_seq(mt, 100, false);
rcu_read_lock();
mas_set(&mas, 0);
entry = mas_find(&mas, ULONG_MAX);
MT_BUG_ON(mt, entry != xa_mk_value(0));
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
entry = mas_find(&mas, ULONG_MAX);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_none);
MT_BUG_ON(mt, mas.index != 1);
MT_BUG_ON(mt, mas.last != ULONG_MAX);
rcu_read_unlock();
mtree_destroy(mt);
rcu_barrier();
/* find_rev: the tree is emptied */
mt_init_flags(mt, MT_FLAGS_USE_RCU);
check_seq(mt, 100, false);
rcu_read_lock();
mas_set(&mas, ULONG_MAX);
entry = mas_find_rev(&mas, 0);
MT_BUG_ON(mt, entry != xa_mk_value(100));
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
entry = mas_find_rev(&mas, 0);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_none);
MT_BUG_ON(mt, mas.index != 0);
MT_BUG_ON(mt, mas.last != ULONG_MAX);
entry = mas_find_rev(&mas, 0);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_none);
rcu_read_unlock();
mtree_destroy(mt);
rcu_barrier();
/* prev: the tree is emptied, then holds one entry at 0 */
mt_init_flags(mt, MT_FLAGS_USE_RCU);
check_seq(mt, 100, false);
rcu_read_lock();
mas_set(&mas, ULONG_MAX);
entry = mas_find_rev(&mas, 0);
MT_BUG_ON(mt, entry != xa_mk_value(100));
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
entry = mas_prev(&mas, 0);
MT_BUG_ON(mt, entry != ptr);
MT_BUG_ON(mt, mas.status != ma_root);
MT_BUG_ON(mt, mas.index != 0);
MT_BUG_ON(mt, mas.last != 0);
rcu_read_unlock();
mtree_destroy(mt);
rcu_barrier();
/* find_rev: the entry at 0 is below the minimum */
mt_init_flags(mt, MT_FLAGS_USE_RCU);
check_seq(mt, 100, false);
rcu_read_lock();
mas_set(&mas, ULONG_MAX);
entry = mas_find_rev(&mas, 1);
MT_BUG_ON(mt, entry != xa_mk_value(100));
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
entry = mas_find_rev(&mas, 1);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_root);
MT_BUG_ON(mt, mas.index != 0);
MT_BUG_ON(mt, mas.last != 0);
rcu_read_unlock();
mtree_destroy(mt);
rcu_barrier();
/* find_rev: the reader is at 0 when the tree gets one entry at 0 */
mt_init_flags(mt, MT_FLAGS_USE_RCU);
check_seq(mt, 100, false);
rcu_read_lock();
mas_set(&mas, 0);
entry = mas_find_rev(&mas, 0);
MT_BUG_ON(mt, entry != xa_mk_value(0));
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
entry = mas_find_rev(&mas, 0);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_root);
MT_BUG_ON(mt, mas.index != 0);
MT_BUG_ON(mt, mas.last != 0);
rcu_read_unlock();
mtree_destroy(mt);
rcu_barrier();
/* find_range_rev: the reader is at 0 when the tree gets one entry at 0 */
mt_init_flags(mt, MT_FLAGS_USE_RCU);
check_seq(mt, 100, false);
rcu_read_lock();
mas_set(&mas, 0);
entry = mas_find_range_rev(&mas, 0);
MT_BUG_ON(mt, entry != xa_mk_value(0));
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
entry = mas_find_range_rev(&mas, 0);
MT_BUG_ON(mt, entry != NULL);
MT_BUG_ON(mt, mas.status != ma_root);
MT_BUG_ON(mt, mas.index != 0);
MT_BUG_ON(mt, mas.last != 0);
rcu_read_unlock();
mtree_destroy(mt);
rcu_barrier();
mt_set_non_kernel(0);
}
static DEFINE_MTREE(tree);
static int __init maple_tree_seed(void)
{
@@ -4031,6 +4195,7 @@ static int __init maple_tree_seed(void)
check_range64_in_rcu(&tree);
check_restart_no_node(&tree);
#if defined(BENCH)
skip:
+70
View File
@@ -1247,6 +1247,75 @@ static noinline void check_multi_find_3(struct xarray *xa)
}
}
static noinline void check_multi_find_4(struct xarray *xa)
{
#ifdef CONFIG_XARRAY_MULTI
unsigned int order = XA_CHUNK_SHIFT + 1;
unsigned long next = 1UL << order;
unsigned long start = next - 1;
XA_STATE(xas, xa, start);
XA_STATE_ORDER(split, xa, 0, 0);
void *entry;
unsigned long i;
/* Multi-index entry in two slots of a non-leaf node. */
xa_store_order(xa, 0, order, xa_mk_index(0), GFP_KERNEL);
XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL);
rcu_read_lock();
entry = xas_find(&xas, ULONG_MAX);
XA_BUG_ON(xa, entry != xa_mk_index(0));
XA_BUG_ON(xa, xas.xa_index != start);
entry = xas_find(&xas, ULONG_MAX);
XA_BUG_ON(xa, entry != xa_mk_index(next));
XA_BUG_ON(xa, xas.xa_index != next);
entry = xas_find(&xas, ULONG_MAX);
XA_BUG_ON(xa, entry != NULL);
rcu_read_unlock();
xa_erase_index(xa, next);
xa_erase_index(xa, 0);
XA_BUG_ON(xa, !xa_empty(xa));
/* Split the multi-index entry after a lookup begins inside it. */
xa_store_order(xa, 0, order, xa_mk_index(0), GFP_KERNEL);
XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL);
xas_set(&xas, start);
rcu_read_lock();
entry = xas_find(&xas, ULONG_MAX);
XA_BUG_ON(xa, entry != xa_mk_index(0));
XA_BUG_ON(xa, xas.xa_index != start);
rcu_read_unlock();
xas_split_alloc(&split, xa_mk_index(0), order, GFP_KERNEL);
if (xas_error(&split)) {
XA_BUG_ON(xa, true);
goto out;
}
xas_lock(&split);
xas_split(&split, xa_mk_index(0), order);
for (i = 0; i < next; i++)
__xa_store(xa, i, xa_mk_index(i), 0);
xas_unlock(&split);
rcu_read_lock();
entry = xas_find(&xas, ULONG_MAX);
XA_BUG_ON(xa, entry != xa_mk_index(next));
XA_BUG_ON(xa, xas.xa_index != next);
entry = xas_find(&xas, ULONG_MAX);
XA_BUG_ON(xa, entry != NULL);
rcu_read_unlock();
out:
xa_destroy(xa);
XA_BUG_ON(xa, !xa_empty(xa));
#endif
}
static noinline void check_find_1(struct xarray *xa)
{
unsigned long i, j, k;
@@ -1370,6 +1439,7 @@ static noinline void check_find(struct xarray *xa)
check_multi_find_1(xa, i);
check_multi_find_2(xa);
check_multi_find_3(xa);
check_multi_find_4(xa);
}
/* See find_swap_entry() in mm/shmem.c */
+7 -4
View File
@@ -471,11 +471,12 @@ static void xas_shrink(struct xa_state *xas)
if (!xa_is_node(entry))
RCU_INIT_POINTER(node->slots[0], XA_RETRY_ENTRY);
xas_update(xas, node);
if (xa_is_node(entry))
xa_to_node(entry)->parent = NULL;
xa_node_free(node);
if (!xa_is_node(entry))
break;
node = xa_to_node(entry);
node->parent = NULL;
}
}
@@ -1409,9 +1410,11 @@ void *xas_find(struct xa_state *xas, unsigned long max)
entry = xas_load(xas);
if (entry || xas_not_node(xas->xa_node))
return entry;
} else if (!xas->xa_node->shift &&
xas->xa_offset != (xas->xa_index & XA_CHUNK_MASK)) {
xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1;
} else if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node)) {
if (!xas->xa_node->shift)
xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1;
else
xas->xa_offset = get_offset(xas->xa_index, xas->xa_node);
}
xas_next_offset(xas);
+3 -2
View File
@@ -3975,10 +3975,11 @@ static int kdamond_wait_activation(struct damon_ctx *ctx)
{
struct damos *s;
unsigned long wait_time;
unsigned long min_wait_time = 0;
bool init_wait_time = false;
while (!kdamond_need_stop(ctx)) {
unsigned long min_wait_time = 0;
bool init_wait_time = false;
damon_for_each_scheme(s, ctx) {
wait_time = damos_wmark_wait_us(s);
if (!init_wait_time || wait_time < min_wait_time) {
+9
View File
@@ -79,6 +79,15 @@ static void damon_sysfs_test_add_targets(struct kunit *test)
kunit_skip(test, "ctx alloc fail");
}
if (damon_select_ops(ctx, DAMON_OPS_VADDR)) {
damon_destroy_ctx(ctx);
kfree(sysfs_targets->targets_arr);
kfree(sysfs_targets);
kfree(sysfs_target->regions);
kfree(sysfs_target);
kunit_skip(test, "vaddr ops not registered");
}
damon_sysfs_add_targets(ctx, sysfs_targets);
KUNIT_EXPECT_EQ(test, 1u, nr_damon_targets(ctx));
+7
View File
@@ -3976,6 +3976,13 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf,
unsigned long end;
vm_fault_t map_ret;
/* A lower index could move vmf->pte out of its page table. */
if (unlikely(xas.xa_index < last_pgoff)) {
folio_unlock(folio);
folio_put(folio);
break;
}
addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
vmf->pte += xas.xa_index - last_pgoff;
last_pgoff = xas.xa_index;
+4
View File
@@ -4593,10 +4593,14 @@ int folio_split(struct folio *folio, unsigned int new_order,
* If a file-backed folio is truncated, 0 will be returned. Any subsequent
* split attempt should get -EBUSY from split checking code.
*
* Context: @folio must be locked.
*
* Return: @folio's minimum order for split
*/
unsigned int min_order_for_split(struct folio *folio)
{
VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
if (folio_test_anon(folio))
return 0;
+8 -5
View File
@@ -68,6 +68,7 @@ void kasan_set_track(struct kasan_track *track, depot_stack_handle_t stack)
track->timestamp = ts_nsec >> 9;
#endif /* CONFIG_KASAN_EXTRA_INFO */
track->pid = current->pid;
track->rcu_free = 0;
track->stack = stack;
}
@@ -231,7 +232,7 @@ void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
/* Returns true when freeing the object is not safe. */
static bool check_slab_allocation(struct kmem_cache *cache, void *object,
unsigned long ip)
unsigned long ip, bool after_rcu_delay)
{
void *tagged_object = object;
@@ -242,7 +243,8 @@ static bool check_slab_allocation(struct kmem_cache *cache, void *object,
return true;
}
if (!kasan_byte_accessible(tagged_object)) {
if (!kasan_byte_accessible(tagged_object) ||
kasan_check_rcu_free(cache, object, after_rcu_delay)) {
kasan_report_invalid_free(tagged_object, ip, KASAN_REPORT_DOUBLE_FREE);
return true;
}
@@ -259,17 +261,18 @@ static inline void poison_slab_object(struct kmem_cache *cache, void *object,
kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),
KASAN_SLAB_FREE, init);
kasan_clear_rcu_free(cache, object);
if (kasan_stack_collection_enabled())
kasan_save_free_info(cache, tagged_object);
}
bool __kasan_slab_pre_free(struct kmem_cache *cache, void *object,
unsigned long ip)
unsigned long ip, bool after_rcu_delay)
{
if (is_kfence_address(object))
return false;
return check_slab_allocation(cache, object, ip);
return check_slab_allocation(cache, object, ip, after_rcu_delay);
}
bool __kasan_slab_free(struct kmem_cache *cache, void *object, bool init,
@@ -544,7 +547,7 @@ bool __kasan_mempool_poison_object(void *ptr, unsigned long ip)
slab = page_slab(page);
if (check_slab_allocation(slab->slab_cache, ptr, ip))
if (check_slab_allocation(slab->slab_cache, ptr, ip, false))
return false;
poison_slab_object(slab->slab_cache, ptr, false);
+16 -1
View File
@@ -533,10 +533,13 @@ size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object)
}
/*
* Records an auxiliary stack trace for the object at @addr and, if @rcu_free
* is set, marks the object as pending an RCU-delayed free.
*
* This function avoids dynamic memory allocations and thus can be called from
* contexts that do not allow allocating memory.
*/
void kasan_record_aux_stack(void *addr)
static void __kasan_record_aux_stack(void *addr, bool rcu_free)
{
struct slab *slab = kasan_addr_to_slab(addr);
struct kmem_cache *cache;
@@ -552,10 +555,22 @@ void kasan_record_aux_stack(void *addr)
if (!alloc_meta)
return;
if (rcu_free)
alloc_meta->alloc_track.rcu_free = 1;
alloc_meta->aux_stack[1] = alloc_meta->aux_stack[0];
alloc_meta->aux_stack[0] = kasan_save_stack(0, 0);
}
void kasan_record_aux_stack(void *addr)
{
__kasan_record_aux_stack(addr, false);
}
void kasan_mark_rcu_free_pending(void *addr)
{
__kasan_record_aux_stack(addr, true);
}
void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
{
struct kasan_alloc_meta *alloc_meta;
+33 -1
View File
@@ -191,7 +191,10 @@ static inline bool kasan_requires_meta(void)
#define KASAN_STACK_DEPTH 64
struct kasan_track {
u32 pid;
#define KASAN_PID_BITS const_ilog2(PID_MAX_LIMIT)
u32 pid:KASAN_PID_BITS;
u32 rcu_free:1;
u32 unused:32 - (KASAN_PID_BITS + 1);
depot_stack_handle_t stack;
#ifdef CONFIG_KASAN_EXTRA_INFO
u64 cpu:20;
@@ -390,8 +393,37 @@ struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
const void *object);
void kasan_init_object_meta(struct kmem_cache *cache, const void *object);
static inline bool kasan_check_rcu_free(struct kmem_cache *cache,
const void *object,
bool after_rcu_delay)
{
struct kasan_alloc_meta *alloc_meta;
alloc_meta = kasan_get_alloc_meta(cache, object);
if (!alloc_meta)
return false;
return alloc_meta->alloc_track.rcu_free != after_rcu_delay;
}
static inline void kasan_clear_rcu_free(struct kmem_cache *cache,
const void *object)
{
struct kasan_alloc_meta *alloc_meta;
alloc_meta = kasan_get_alloc_meta(cache, object);
if (alloc_meta)
alloc_meta->alloc_track.rcu_free = 0;
}
#else
static inline void kasan_init_object_meta(struct kmem_cache *cache, const void *object) { }
static inline bool kasan_check_rcu_free(struct kmem_cache *cache, const void *object,
bool after_rcu_delay)
{
return false;
}
static inline void kasan_clear_rcu_free(struct kmem_cache *cache, const void *object) { }
#endif
depot_stack_handle_t kasan_save_stack(gfp_t flags, depot_flags_t depot_flags);
+139
View File
@@ -30,6 +30,7 @@
#include <asm/page.h>
#include "kasan.h"
#include "../slab.h"
#define OOB_TAG_OFF (IS_ENABLED(CONFIG_KASAN_GENERIC) ? 0 : KASAN_GRANULE_SIZE)
@@ -1070,6 +1071,139 @@ static void kmem_cache_double_free(struct kunit *test)
kmem_cache_destroy(cache);
}
#ifndef MODULE
static void __kmem_cache_double_free_rcu(struct kunit *test, int mode)
{
struct kasan_rcu_info *p, *victim = NULL, **objs;
struct kmem_cache *cache;
int i, n = 0;
KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC);
KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_PREEMPT_RT);
cache = kmem_cache_create("test_cache", sizeof(*p), 0, SLAB_NO_MERGE,
NULL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache);
if (!cache->sheaf_capacity) {
kmem_cache_destroy(cache);
kunit_skip(test, "Test requires sheaves");
}
objs = kunit_kcalloc(test, cache->sheaf_capacity * 2, sizeof(*objs),
GFP_KERNEL);
if (!objs) {
kunit_err(test, "Allocation failed: %s\n", __func__);
goto out;
}
migrate_disable();
p = kmem_cache_alloc(cache, GFP_KERNEL);
if (!p) {
migrate_enable();
kunit_err(test, "Allocation failed: %s\n", __func__);
goto out;
}
switch (mode) {
case 1:
kmem_cache_free(cache, p);
/* p is already in quarantine. */
KUNIT_EXPECT_KASAN_FAIL(test, kfree_rcu(p, rcu));
migrate_enable();
break;
case 2:
/* The actual free order is inverse. */
kfree_rcu(p, rcu);
KUNIT_EXPECT_KASAN_FAIL(test, kmem_cache_free(cache, p));
migrate_enable();
break;
case 3:
kfree_rcu(p, rcu);
for (i = 0; i < cache->sheaf_capacity - 1; i++) {
objs[i] = kmem_cache_alloc(cache, GFP_KERNEL);
kfree_rcu(objs[i], rcu);
}
KUNIT_EXPECT_KASAN_FAIL(test, kfree_rcu(p, rcu));
migrate_enable();
/*
* Move the 1st full sheaf to quarantine, keeping the
* 2nd in pcs->rcu_free.
*/
rcu_barrier();
break;
}
/*
* Move p to quarantine if not poisoned yet (case 2,
* or fallback in case 3).
*/
if (kasan_byte_accessible(p))
kvfree_rcu_barrier_on_cache(cache);
/*
* Evict p from quarantine and reallocate it as a live victim object
* before the RCU sheaf is flushed.
*/
kasan_quarantine_remove_cache(cache);
for (i = 0; i < cache->sheaf_capacity * 2; i++) {
objs[n] = kmem_cache_alloc(cache, GFP_KERNEL);
if (!objs[n])
break;
if (objs[n] == p) {
victim = objs[n];
break;
}
n++;
}
for (i = 0; i < n; i++)
kmem_cache_free(cache, objs[i]);
KUNIT_EXPECT_PTR_EQ(test, victim, p);
if (victim != p)
goto out;
WRITE_ONCE(victim->i, 42);
/*
* If kfree_rcu() failed to reject the double-free above,
* flushing the RCU sheaf will silently poison the live
* victim and cause a false-positive UAF on victim->i.
*/
kvfree_rcu_barrier_on_cache(cache);
KUNIT_EXPECT_EQ(test, READ_ONCE(victim->i), 42);
if (!READ_ONCE(test_status.report_found))
kmem_cache_free(cache, victim);
out:
kmem_cache_destroy(cache);
}
static void kmem_cache_double_free_rcu_1(struct kunit *test)
{
/* kmem_cache_free() -> kfree_rcu() */
__kmem_cache_double_free_rcu(test, 1);
}
static void kmem_cache_double_free_rcu_2(struct kunit *test)
{
/* kfree_rcu() -> kmem_cache_free() */
__kmem_cache_double_free_rcu(test, 2);
}
static void kmem_cache_double_free_rcu_3(struct kunit *test)
{
/* kfree_rcu() -> kfree_rcu() */
__kmem_cache_double_free_rcu(test, 3);
}
#endif
static void kmem_cache_invalid_free(struct kunit *test)
{
char *p;
@@ -2239,6 +2373,11 @@ static struct kunit_case kasan_kunit_test_cases[] = {
KUNIT_CASE(kfree_via_phys),
KUNIT_CASE(kmem_cache_oob),
KUNIT_CASE(kmem_cache_double_free),
#ifndef MODULE
KUNIT_CASE(kmem_cache_double_free_rcu_1),
KUNIT_CASE(kmem_cache_double_free_rcu_2),
KUNIT_CASE(kmem_cache_double_free_rcu_3),
#endif
KUNIT_CASE(kmem_cache_invalid_free),
KUNIT_CASE(kmem_cache_rcu_uaf),
KUNIT_CASE(kmem_cache_rcu_reuse),
+24 -45
View File
@@ -1320,13 +1320,13 @@ static bool panic_on_unrecoverable_mf(enum mf_action_page_type type,
static int action_result(unsigned long pfn, enum mf_action_page_type type,
enum mf_result result)
{
trace_memory_failure_event(pfn, type, result);
if (type != MF_MSG_ALREADY_POISONED && type != MF_MSG_PFN_MAP) {
num_poisoned_pages_inc(pfn);
update_per_node_mf_stats(pfn, result);
}
trace_memory_failure_event(pfn, type, result);
pr_err("%#lx: recovery action for %s: %s\n",
pfn, action_page_types[type], action_name[result]);
@@ -1744,26 +1744,6 @@ static int identify_page_state(unsigned long pfn, struct page *p,
return page_action(ps, p, pfn);
}
/*
* When 'release' is 'false', it means that if thp split has failed,
* there is still more to do, hence the page refcount we took earlier
* is still needed.
*/
static int try_to_split_thp_page(struct page *page, unsigned int new_order,
bool release)
{
int ret;
lock_page(page);
ret = split_huge_page_to_order(page, new_order);
unlock_page(page);
if (ret && release)
put_page(page);
return ret;
}
static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn,
struct address_space *mapping, pgoff_t index, int flags)
{
@@ -2548,9 +2528,6 @@ try_again:
folio_unlock(folio);
if (folio_test_large(folio)) {
const int new_order = min_order_for_split(folio);
int err;
/*
* The flag must be set after the refcount is bumped
* otherwise it may race with THP split.
@@ -2565,24 +2542,25 @@ try_again:
* page is a valid handlable page.
*/
folio_set_has_hwpoisoned(folio);
err = try_to_split_thp_page(p, new_order, /* release= */ false);
folio_lock(folio);
split_huge_page_to_order(p, min_order_for_split(folio));
folio = page_folio(p);
folio_unlock(folio);
/*
* If splitting a folio to order-0 fails, kill the process.
* Split the folio regardless to minimize unusable pages.
* Because the memory failure code cannot handle large
* folios, this split is always treated as if it failed.
*/
if (err || new_order) {
/* get folio again in case the original one is split */
folio = page_folio(p);
if (folio_test_large(folio)) {
res = -EHWPOISON;
kill_procs_now(p, pfn, flags, folio);
put_page(p);
folio_put(folio);
action_result(pfn, MF_MSG_UNSPLIT_THP, MF_FAILED);
goto unlock_mutex;
}
VM_BUG_ON_PAGE(!page_count(p), p);
folio = page_folio(p);
}
/*
@@ -2900,25 +2878,26 @@ static int soft_offline_in_use_page(struct page *page)
.reason = MR_MEMORY_FAILURE,
};
folio_lock(folio);
if (!huge && folio_test_large(folio)) {
const int new_order = min_order_for_split(folio);
/*
* If new_order (target split order) is not 0, do not split the
* folio at all to retain the still accessible large folio.
* NOTE: if minimizing the number of soft offline pages is
* preferred, split it to non-zero new_order like it is done in
* memory_failure().
* If we cannot split to order-0, do not split the folio at all
* to retain the still accessible large folio.
*/
if (new_order || try_to_split_thp_page(page, /* new_order= */ 0,
/* release= */ true)) {
pr_info("%#lx: thp split failed\n", pfn);
return -EBUSY;
}
if (!min_order_for_split(folio))
ret = split_huge_page_to_order(page, 0) ? -EBUSY : 0;
else
ret = -EBUSY;
folio = page_folio(page);
if (ret) {
folio_unlock(folio);
folio_put(folio);
pr_info("%#lx: thp split failed\n", pfn);
return ret;
}
}
folio_lock(folio);
if (!huge)
folio_wait_writeback(folio);
if (PageHWPoison(page)) {
+10 -3
View File
@@ -207,15 +207,22 @@ static u8 get_il_weight(int node)
*/
static void reduce_interleave_weights(unsigned int *bw, u8 *new_iw)
{
nodemask_t nodes = NODE_MASK_NONE;
u64 sum_bw = 0;
unsigned int cast_sum_bw, scaling_factor = 1, iw_gcd = 0;
int nid;
for_each_node_state(nid, N_MEMORY)
/*
* N_MEMORY can change while wi_state_lock is held. Remember the
* nodes contributing to sum_bw so all passes use the same set.
*/
for_each_node_state(nid, N_MEMORY) {
node_set(nid, nodes);
sum_bw += bw[nid];
}
/* Scale bandwidths to whole numbers in the range [1, weightiness] */
for_each_node_state(nid, N_MEMORY) {
for_each_node_mask(nid, nodes) {
/*
* Try not to perform 64-bit division.
* If sum_bw < scaling_factor, then sum_bw < U32_MAX.
@@ -234,7 +241,7 @@ static void reduce_interleave_weights(unsigned int *bw, u8 *new_iw)
}
/* 1:2 is strictly better than 16:32. Reduce by the weights' GCD. */
for_each_node_state(nid, N_MEMORY)
for_each_node_mask(nid, nodes)
new_iw[nid] /= iw_gcd;
}
+10
View File
@@ -711,6 +711,16 @@ static int __folio_migrate_mapping(struct address_space *mapping,
mod_lruvec_state(old_lruvec, NR_SHMEM_THPS, -nr);
mod_lruvec_state(new_lruvec, NR_SHMEM_THPS, nr);
}
} else if (!folio_test_swapbacked(folio) &&
folio_test_pmd_mappable(folio)) {
mod_lruvec_state(old_lruvec, NR_FILE_THPS, -nr);
mod_lruvec_state(new_lruvec, NR_FILE_THPS, nr);
}
if (test_bit(AS_KERNEL_FILE, &mapping->flags)) {
__mod_node_page_state(oldzone->zone_pgdat,
NR_KERNEL_FILE_PAGES, -nr);
__mod_node_page_state(newzone->zone_pgdat,
NR_KERNEL_FILE_PAGES, nr);
}
#ifdef CONFIG_SWAP
if (folio_test_swapcache(folio)) {
+14 -2
View File
@@ -2612,11 +2612,14 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
* and swap cache folios are never partially freed.
*/
folio_lock(folio);
if (!folio_matches_swap_entry(folio, swap) ||
shmem_confirm_swap(mapping, index, swap) < 0) {
if (!folio_matches_swap_entry(folio, swap)) {
error = -EEXIST;
goto unlock;
}
if (shmem_confirm_swap(mapping, index, swap) < 0) {
error = -EEXIST;
goto failed;
}
if (!folio_test_uptodate(folio)) {
error = -EIO;
goto failed;
@@ -2657,6 +2660,15 @@ failed:
error = -EEXIST;
if (error == -EIO)
shmem_set_folio_swapin_error(inode, index, folio, swap);
/*
* A large folio that does not fit the mapping, e.g. because its swap
* entries were freed and reused, would be found again on every retry.
* Drop it from the swap cache so the retry can swap in a fitting one.
*/
if (error == -EEXIST && folio && folio_test_large(folio) &&
!folio_test_anon(folio) && !folio_test_dirty(folio) &&
!folio_test_writeback(folio))
swap_cache_del_folio(folio);
unlock:
if (folio)
folio_unlock(folio);
+10 -3
View File
@@ -2722,7 +2722,7 @@ bool slab_free_hook(struct kmem_cache *s, void *x, bool init,
* Give KASAN a chance to notice an invalid free operation before we
* modify the object.
*/
if (kasan_slab_pre_free(s, x))
if (kasan_slab_pre_free(s, x, after_rcu_delay))
return false;
#ifdef CONFIG_SLUB_RCU_DEBUG
@@ -2738,7 +2738,7 @@ bool slab_free_hook(struct kmem_cache *s, void *x, bool init,
* We have to do this manually because the rcu_head is
* not located inside the object.
*/
kasan_record_aux_stack(x);
kasan_mark_rcu_free_pending(x);
delayed_free->object = x;
call_rcu(&delayed_free->head, slab_free_after_rcu_debug);
@@ -6298,6 +6298,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
}
do_free:
if (unlikely(kasan_slab_pre_free(s, obj, false))) {
local_unlock(&s->cpu_sheaves->lock);
goto out;
}
kasan_mark_rcu_free_pending(obj);
rcu_sheaf = pcs->rcu_free;
@@ -6338,6 +6344,7 @@ do_free:
local_unlock(&s->cpu_sheaves->lock);
stat(s, FREE_RCU_SHEAF);
out:
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
lock_map_release(&kfree_rcu_sheaf_map);
return true;
@@ -6971,7 +6978,7 @@ void kfree_nolock(const void *object)
* -> kfree_nolock() -> kasan_report_invalid_free() on the same CPU
* is double buggy and deserves to deadlock.
*/
if (kasan_slab_pre_free(s, x))
if (kasan_slab_pre_free(s, x, false))
return;
/*
* memcg, kasan_slab_pre_free are done for 'x'.
+5
View File
@@ -635,6 +635,10 @@ static struct swap_cluster_info *isolate_lock_cluster(
u8 flags = CLUSTER_FLAG_NONE;
spin_lock(&si->lock);
if (unlikely(!(si->flags & SWP_WRITEOK)))
goto out;
list_for_each_entry(ci, list, list) {
if (!spin_trylock(&ci->lock))
continue;
@@ -650,6 +654,7 @@ static struct swap_cluster_info *isolate_lock_cluster(
found = ci;
break;
}
out:
spin_unlock(&si->lock);
/* Cluster's table is freed when and only when it's on the free list. */
+62 -25
View File
@@ -1109,21 +1109,26 @@ static int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
if (atomic_read(&ctx->mmap_changing))
goto out_unlock;
/*
* Validate every spanned VMA before changing any of them, so an
* invalid range is rejected without changing part of it.
*/
err = -ENOENT;
for_each_vma_range(vmi, dst_vma, end) {
if (!userfaultfd_wp(dst_vma)) {
err = -ENOENT;
break;
}
if (!userfaultfd_wp(dst_vma))
goto out_unlock;
if (vma_is_hugetlb(dst_vma)) {
err = -EINVAL;
page_mask = vma_kernel_pagesize(dst_vma) - 1;
if ((start & page_mask) || (len & page_mask))
break;
if ((start & page_mask) || (len & page_mask)) {
err = -EINVAL;
goto out_unlock;
}
}
}
vma_iter_set(&vmi, start);
for_each_vma_range(vmi, dst_vma, end) {
_start = max(dst_vma->vm_start, start);
_end = min(dst_vma->vm_end, end);
@@ -2196,6 +2201,32 @@ static void userfaultfd_reset_ctx(struct vm_area_struct *vma)
userfaultfd_set_ctx(vma, NULL, 0);
}
/*
* Clear uffd state from a WP or RWP range: the uffd bit, WP markers and
* RWP PAGE_NONE entries. Call this before dropping the registration, as it
* needs the VMA flags to know what to resolve.
*/
static void userfaultfd_clear_ptes(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
unsigned int mm_cp_flags = 0;
struct mmu_gather tlb;
if (!userfaultfd_protected(vma))
return;
if (userfaultfd_wp(vma))
mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE;
if (userfaultfd_rwp(vma))
mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE;
if (vma_wants_manual_pte_write_upgrade(vma))
mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
tlb_gather_mmu(&tlb, vma->vm_mm);
change_protection(&tlb, vma, start, end, mm_cp_flags);
tlb_finish_mmu(&tlb);
}
static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
@@ -2215,22 +2246,7 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi,
if (start == vma->vm_start && end == vma->vm_end)
give_up_on_oom = true;
/* Clear the uffd bit and/or restore protnone PTEs */
if (userfaultfd_protected(vma)) {
unsigned int mm_cp_flags = 0;
struct mmu_gather tlb;
if (userfaultfd_wp(vma))
mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE;
if (userfaultfd_rwp(vma))
mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE;
if (vma_wants_manual_pte_write_upgrade(vma))
mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
tlb_gather_mmu(&tlb, vma->vm_mm);
change_protection(&tlb, vma, start, end, mm_cp_flags);
tlb_finish_mmu(&tlb);
}
userfaultfd_clear_ptes(vma, start, end);
ret = vma_modify_flags_uffd(vmi, prev, vma, start, end,
&new_vma_flags, NULL_VM_UFFD_CTX,
@@ -2339,8 +2355,10 @@ static void userfaultfd_release_new(struct userfaultfd_ctx *ctx)
/* the various vma->vm_userfaultfd_ctx still points to it */
mmap_write_lock(mm);
for_each_vma(vmi, vma) {
if (vma->vm_userfaultfd_ctx.ctx == ctx)
if (vma->vm_userfaultfd_ctx.ctx == ctx) {
userfaultfd_clear_ptes(vma, vma->vm_start, vma->vm_end);
userfaultfd_reset_ctx(vma);
}
}
mmap_write_unlock(mm);
}
@@ -3281,6 +3299,25 @@ void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
}
}
void userfaultfd_unmap_abort(struct list_head *uf)
{
struct userfaultfd_unmap_ctx *unmap_ctx, *n;
if (!uf)
return;
list_for_each_entry_safe(unmap_ctx, n, uf, list) {
struct userfaultfd_ctx *ctx = unmap_ctx->ctx;
atomic_dec(&ctx->mmap_changing);
VM_WARN_ON_ONCE(atomic_read(&ctx->mmap_changing) < 0);
userfaultfd_ctx_put(ctx);
list_del(&unmap_ctx->list);
kfree(unmap_ctx);
}
}
static int userfaultfd_release(struct inode *inode, struct file *file)
{
struct userfaultfd_ctx *ctx = file->private_data;
+45 -17
View File
@@ -676,6 +676,26 @@ static int dup_anon_vma(struct vm_area_struct *dst,
}
#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
static void validate_mm_anon_rmap(struct vm_area_struct *vma)
{
struct anon_vma *anon_vma;
struct anon_vma_chain *avc;
if (!IS_ENABLED(CONFIG_DEBUG_VM_RB))
return;
vma_assert_stabilised(vma);
anon_vma = READ_ONCE(vma->anon_vma);
if (!anon_vma)
return;
anon_vma_lock_read(anon_vma);
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
anon_rmap_tree_verify(avc);
anon_vma_unlock_read(anon_vma);
}
void validate_mm(struct mm_struct *mm)
{
int bug = 0;
@@ -685,10 +705,6 @@ void validate_mm(struct mm_struct *mm)
mt_validate(&mm->mm_mt);
for_each_vma(vmi, vma) {
#ifdef CONFIG_DEBUG_VM_RB
struct anon_vma *anon_vma = vma->anon_vma;
struct anon_vma_chain *avc;
#endif
unsigned long vmi_start, vmi_end;
bool warn = 0;
@@ -709,14 +725,8 @@ void validate_mm(struct mm_struct *mm)
vma_iter_dump_tree(&vmi);
}
#ifdef CONFIG_DEBUG_VM_RB
if (vma_has_anon_rmap(vma)) {
anon_vma_lock_read(anon_vma);
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
anon_rmap_tree_verify(avc);
anon_vma_unlock_read(anon_vma);
}
#endif
validate_mm_anon_rmap(vma);
/* Check for a infinite loop */
if (++i > mm->map_count + 10) {
i = -1;
@@ -1686,9 +1696,18 @@ int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
mt_on_stack(mt_detach);
struct vma_munmap_struct vms;
struct list_head *uf_unmaps;
LIST_HEAD(uf_new_unmaps);
int error;
init_vma_munmap(&vms, vmi, vma, start, end, uf, unlock);
/*
* @uf may already hold events of earlier successful unmaps, e.g. in
* mremap().
* Collect new events on a separate list so they can be dropped if this
* unmap fails, and splice them into @uf on success.
*/
uf_unmaps = uf ? &uf_new_unmaps : NULL;
init_vma_munmap(&vms, vmi, vma, start, end, uf_unmaps, unlock);
error = vms_gather_munmap_vmas(&vms, &mas_detach);
if (error)
goto gather_failed;
@@ -1698,12 +1717,15 @@ int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
goto clear_tree_failed;
/* Point of no return */
if (uf)
list_splice_tail(&uf_new_unmaps, uf);
vms_complete_munmap_vmas(&vms, &mas_detach);
return 0;
clear_tree_failed:
reattach_vmas(&mas_detach);
gather_failed:
userfaultfd_unmap_abort(&uf_new_unmaps);
validate_mm(mm);
return error;
}
@@ -2592,7 +2614,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
if (error) {
/* On error VMAs will already have been reattached. */
vms->nr_pages = 0;
return error;
goto abort_unmaps;
}
map->next = vms->next;
@@ -2602,8 +2624,10 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
}
/* Check against address space limit. */
if (!may_expand_vm(map->mm, &map->vma_flags, map->pglen - vms->nr_pages))
return -ENOMEM;
if (!may_expand_vm(map->mm, &map->vma_flags, map->pglen - vms->nr_pages)) {
error = -ENOMEM;
goto abort_unmaps;
}
/* Private writable mapping: check memory availability. */
if (accountable_mapping(map)) {
@@ -2613,7 +2637,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
if (uncharged) {
error = security_vm_enough_memory_mm(map->mm, uncharged);
if (error)
return error;
goto abort_unmaps;
}
/* Our new mapping owns the entire charge now. */
@@ -2632,6 +2656,10 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
set_desc_from_map(desc, map);
return 0;
abort_unmaps:
userfaultfd_unmap_abort(uf);
return error;
}
static bool map_same_file(struct mmap_state *map)
+9 -4
View File
@@ -36502,22 +36502,27 @@ static void regression_tests(void)
test_spanning_store_regression();
}
void maple_tree_tests(void)
int maple_tree_tests(void)
{
int ret;
#if !defined(BENCH)
regression_tests();
farmer_tests();
#endif
maple_tree_seed();
ret = maple_tree_seed();
maple_tree_harvest();
return ret;
}
int __weak main(void)
{
int ret;
maple_tree_init();
maple_tree_tests();
ret = maple_tree_tests();
rcu_barrier();
if (nr_allocated)
printf("nr_allocated = %d\n", nr_allocated);
return 0;
return ret ? EXIT_FAILURE : EXIT_SUCCESS;
}
@@ -1,6 +1,9 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
# Kselftest framework requirement - SKIP code is 4.
ksft_skip=4
if [ $EUID -ne 0 ]
then
echo "Run as root"
+33
View File
@@ -8,6 +8,7 @@
#include "shared.h"
#include <stdlib.h>
#include <time.h>
#include <assert.h>
#include "linux/init.h"
void maple_rcu_cb(struct rcu_head *head);
@@ -20,5 +21,37 @@ do { \
call_rcu(&((_p_struct)->_memb), rcu_cb); \
} while(0);
/* Make WARN() fatal */
#undef WARN
#undef WARN_ON
#undef WARN_ONCE
#undef WARN_ON_ONCE
#define WARN(condition, format...) ({ \
int __ret_warn_on = !!(condition); \
if (unlikely(__ret_warn_on)) { \
__WARN_printf(format); \
assert(0); \
} \
unlikely(__ret_warn_on); \
})
#define WARN_ON(condition) \
WARN(condition, "assertion failed at %s:%d\n", __FILE__, __LINE__)
#define WARN_ONCE(condition, format...) WARN(condition, format)
#define WARN_ON_ONCE(condition) WARN_ON(condition)
/* Once state for the debug macros */
#ifndef __ONCE_LITE_IF
#define __ONCE_LITE_IF(condition) \
({ \
static bool __already_done; \
bool __ret_once = false; \
\
if ((condition) && !__already_done) { \
__already_done = true; \
__ret_once = true; \
} \
__ret_once; \
})
#endif
#endif /* __MAPLE_SHARED_H__ */
+4
View File
@@ -148,6 +148,10 @@ static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
return 0;
}
static inline void userfaultfd_unmap_abort(struct list_head *uf)
{
}
static inline void mmap_write_downgrade(struct mm_struct *mm)
{
}