mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
Merge tag 'mm-hotfixes-stable-2026-11-17-19-55' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull MM fixes from Andrew Morton: - "maple_tree fixes for 7.3" (Liam R. Howlett): Fix an RCU reader NULL dereference when the tree is emptied during a walk, including /proc/PID/maps racing with munmap(). Add tests and make debug WARN and BUG checks report failures properly - "mm/kasan: Detect double-free involving kfree_rcu_sheaf()" (Kuniyuki Iwashima): Detect double-free when an object enters an RCU sheaf. KASAN could previously miss it until the object was reused, leading to a later free of a live object. Add KUnit tests for the different free orders - "mm: Fix page-table teardown during memory hot-remove" (Muchun Song): Balance page-table constructors and destructors when hot-remove frees tables on x86-64, RISC-V and arm64. This fixes NR_PAGETABLE accounting for tables with and without constructors - "mm/damon: fix watermark state, kunit pid leak and selftest fake pass" (SJ Park): Recompute watermark wait state when DAMON schemes are removed, so kdamond resumes monitoring. Fix a pid reference leak in KUnit and a selftest that reports a skipped run as a pass - An XArray iterator could step backwards when a large page-cache entry was split, sending filemap_map_pages() outside the PTE page it locked. The XArray correction and a filemap bound check prevent out-of-bounds accesses and use-after-free. Another XArray fix prevents lockless walkers from following a parent pointer into a freed node - Userfaultfd now validates the whole write-protection range before changing any VMA, avoiding partial unprotect without waking blocked faults. Failed unmaps now abort queued UFFD_EVENT_UNMAPs; previously they could report a nonexistent unmap, leak the userfaultfd context and leave later operations returning -EAGAIN. A fork fix clears stale uffd PTE bits if the new context is released before its event is read - Moving the hugetlbfs inode trace event before RCU freeing avoids a use-after-free of its superblock. Shmem swapin no longer retries forever on reused swap slots containing a mismatched large swap-cache folio. Swapoff also avoids a cluster-list loop that can soft-lock reclaim and leave swapoff hung - Weighted interleave recalculation now uses one node set throughout, avoiding a divide-by-zero during memory hotplug. Hwpoison holds the folio lock when inspecting its mapping and no longer leaks a folio reference; it also updates poison statistics before notifying rasdaemon through a trace event - Cross-node migration now moves NR_FILE_THPS and NR_KERNEL_FILE_PAGES with page-cache folios, preventing per-node and memcg-node accounting from retaining stale counts or underflowing after the folios are freed * tag 'mm-hotfixes-stable-2026-11-17-19-55' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (27 commits) xarray: fix index jumping backwards in xas_find() userfaultfd: validate the whole range before changing write protection maple_tree: update internal warning and bug_on code maple_tree: test emptying tree while RCU reading maple_tree: fix rcu reader NULL dereference after the tree is emptied mm/vma: read anon_vma carefully in validate_mm() hugetlbfs: move hugetlbfs_free_inode tracepoint to ->destroy_inode() selftests/damon/sysfs_memcg_path_leak: define ksft_skip mm/damon/tests/sysfs-kunit: fix leaked pid reference in add_targets test mm/damon/core: reset watermark wait state on each iteration mm/filemap: stop fault-around when the walk steps back mm: memory-failure: update statistics before emitting the trace event mm/vma: undo userfaultfd unmap preparation when unmapping fails userfaultfd: clear uffd state in child PTEs if uffd is released during fork mm/memory-failure: fix folio refcount leak and min_order_for_split() locking mm/shmem: fix swapin livelock on a mismatched large swap cache folio mm/mempolicy: use one node set for interleave weights xarray: clear the new head's parent before freeing the old head arm64/mm: fix destructor for unconstructed hotplug page tables riscv/mm: fix missing destructor for hotplug PUD tables ...
This commit is contained in:
+1
-2
@@ -1495,8 +1495,7 @@ static void free_hotplug_page_range(struct page *page, size_t size,
|
||||
|
||||
static void free_hotplug_pgtable_page(struct page *page)
|
||||
{
|
||||
pagetable_dtor(page_ptdesc(page));
|
||||
free_hotplug_page_range(page, PAGE_SIZE, NULL);
|
||||
pagetable_free(page_ptdesc(page));
|
||||
}
|
||||
|
||||
static bool pgtable_range_aligned(unsigned long start, unsigned long end,
|
||||
|
||||
+13
-19
@@ -1501,10 +1501,17 @@ struct execmem_info __init *execmem_arch_setup(void)
|
||||
#endif /* CONFIG_EXECMEM */
|
||||
|
||||
#ifdef CONFIG_MEMORY_HOTPLUG
|
||||
static void __meminit free_pgtable_page(struct page *page)
|
||||
{
|
||||
if (PageReserved(page))
|
||||
free_reserved_page(page);
|
||||
else
|
||||
pagetable_free(page_ptdesc(page));
|
||||
}
|
||||
|
||||
static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
|
||||
{
|
||||
struct page *page = pmd_page(*pmd);
|
||||
struct ptdesc *ptdesc = page_ptdesc(page);
|
||||
pte_t *pte;
|
||||
int i;
|
||||
|
||||
@@ -1514,18 +1521,13 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
|
||||
return;
|
||||
}
|
||||
|
||||
pagetable_dtor(ptdesc);
|
||||
if (PageReserved(page))
|
||||
free_reserved_page(page);
|
||||
else
|
||||
pagetable_free(ptdesc);
|
||||
free_pgtable_page(page);
|
||||
pmd_clear(pmd);
|
||||
}
|
||||
|
||||
static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud, bool is_vmemmap)
|
||||
static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)
|
||||
{
|
||||
struct page *page = pud_page(*pud);
|
||||
struct ptdesc *ptdesc = page_ptdesc(page);
|
||||
pmd_t *pmd;
|
||||
int i;
|
||||
|
||||
@@ -1535,12 +1537,7 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud, bool is_vmemm
|
||||
return;
|
||||
}
|
||||
|
||||
if (!is_vmemmap)
|
||||
pagetable_dtor(ptdesc);
|
||||
if (PageReserved(page))
|
||||
free_reserved_page(page);
|
||||
else
|
||||
pagetable_free(ptdesc);
|
||||
free_pgtable_page(page);
|
||||
pud_clear(pud);
|
||||
}
|
||||
|
||||
@@ -1556,10 +1553,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d)
|
||||
return;
|
||||
}
|
||||
|
||||
if (PageReserved(page))
|
||||
free_reserved_page(page);
|
||||
else
|
||||
__free_pages(page, 0);
|
||||
free_pgtable_page(page);
|
||||
p4d_clear(p4d);
|
||||
}
|
||||
|
||||
@@ -1660,7 +1654,7 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un
|
||||
remove_pmd_mapping(pmd_base, addr, next, is_vmemmap, altmap);
|
||||
|
||||
if (pgtable_l4_enabled())
|
||||
free_pmd_table(pmd_base, pudp, is_vmemmap);
|
||||
free_pmd_table(pmd_base, pudp);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1187,12 +1187,12 @@ static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
|
||||
|
||||
static void hugetlbfs_free_inode(struct inode *inode)
|
||||
{
|
||||
trace_hugetlbfs_free_inode(inode);
|
||||
kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
|
||||
}
|
||||
|
||||
static void hugetlbfs_destroy_inode(struct inode *inode)
|
||||
{
|
||||
trace_hugetlbfs_destroy_inode(inode);
|
||||
hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
|
||||
}
|
||||
|
||||
|
||||
+11
-4
@@ -180,10 +180,12 @@ static __always_inline void * __must_check kasan_init_slab_obj(
|
||||
}
|
||||
|
||||
bool __kasan_slab_pre_free(struct kmem_cache *s, void *object,
|
||||
unsigned long ip);
|
||||
unsigned long ip, bool after_rcu_delay);
|
||||
/**
|
||||
* kasan_slab_pre_free - Check whether freeing a slab object is safe.
|
||||
* @s: Cache the object belongs to.
|
||||
* @object: Object to be freed.
|
||||
* @after_rcu_delay: Whether the free follows an RCU grace period.
|
||||
*
|
||||
* This function checks whether freeing the given object is safe. It may
|
||||
* check for double-free and invalid-free bugs and report them.
|
||||
@@ -193,10 +195,12 @@ bool __kasan_slab_pre_free(struct kmem_cache *s, void *object,
|
||||
* @Return true if freeing the object is unsafe; false otherwise.
|
||||
*/
|
||||
static __always_inline bool kasan_slab_pre_free(struct kmem_cache *s,
|
||||
void *object)
|
||||
void *object,
|
||||
bool after_rcu_delay)
|
||||
{
|
||||
if (kasan_enabled())
|
||||
return __kasan_slab_pre_free(s, object, _RET_IP_);
|
||||
return __kasan_slab_pre_free(s, object, _RET_IP_,
|
||||
after_rcu_delay);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -424,7 +428,8 @@ static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
|
||||
return (void *)object;
|
||||
}
|
||||
|
||||
static inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object)
|
||||
static inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object,
|
||||
bool after_rcu_delay)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -495,6 +500,7 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
|
||||
void kasan_cache_shrink(struct kmem_cache *cache);
|
||||
void kasan_cache_shutdown(struct kmem_cache *cache);
|
||||
void kasan_record_aux_stack(void *ptr);
|
||||
void kasan_mark_rcu_free_pending(void *ptr);
|
||||
|
||||
#else /* CONFIG_KASAN_GENERIC */
|
||||
|
||||
@@ -512,6 +518,7 @@ static inline void kasan_cache_create(struct kmem_cache *cache,
|
||||
static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
|
||||
static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
|
||||
static inline void kasan_record_aux_stack(void *ptr) {}
|
||||
static inline void kasan_mark_rcu_free_pending(void *ptr) {}
|
||||
|
||||
#endif /* CONFIG_KASAN_GENERIC */
|
||||
|
||||
|
||||
+24
-24
@@ -684,7 +684,7 @@ void mt_cache_shrink(void);
|
||||
pr_info("Pass: %u Run:%u\n", \
|
||||
atomic_read(&maple_tree_tests_passed), \
|
||||
atomic_read(&maple_tree_tests_run)); \
|
||||
dump_stack(); \
|
||||
BUG_ON(1); \
|
||||
} else { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
} \
|
||||
@@ -700,7 +700,7 @@ void mt_cache_shrink(void);
|
||||
pr_info("Pass: %u Run:%u\n", \
|
||||
atomic_read(&maple_tree_tests_passed), \
|
||||
atomic_read(&maple_tree_tests_run)); \
|
||||
dump_stack(); \
|
||||
BUG_ON(1); \
|
||||
} else { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
} \
|
||||
@@ -717,7 +717,7 @@ void mt_cache_shrink(void);
|
||||
pr_info("Pass: %u Run:%u\n", \
|
||||
atomic_read(&maple_tree_tests_passed), \
|
||||
atomic_read(&maple_tree_tests_run)); \
|
||||
dump_stack(); \
|
||||
BUG_ON(1); \
|
||||
} else { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
} \
|
||||
@@ -726,16 +726,16 @@ void mt_cache_shrink(void);
|
||||
#define MT_WARN_ON(__tree, __x) ({ \
|
||||
int ret = !!(__x); \
|
||||
atomic_inc(&maple_tree_tests_run); \
|
||||
if (ret) { \
|
||||
pr_info("WARN at %s:%d (%u)\n", \
|
||||
__func__, __LINE__, __x); \
|
||||
if (!ret) { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
} else if (__ONCE_LITE_IF(ret)) { \
|
||||
pr_info("WARN at %s:%d\n", \
|
||||
__func__, __LINE__); \
|
||||
mt_dump(__tree, mt_dump_hex); \
|
||||
pr_info("Pass: %u Run:%u\n", \
|
||||
atomic_read(&maple_tree_tests_passed), \
|
||||
atomic_read(&maple_tree_tests_run)); \
|
||||
dump_stack(); \
|
||||
} else { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
WARN_ON(1); \
|
||||
} \
|
||||
unlikely(ret); \
|
||||
})
|
||||
@@ -743,17 +743,17 @@ void mt_cache_shrink(void);
|
||||
#define MAS_WARN_ON(__mas, __x) ({ \
|
||||
int ret = !!(__x); \
|
||||
atomic_inc(&maple_tree_tests_run); \
|
||||
if (ret) { \
|
||||
pr_info("WARN at %s:%d (%u)\n", \
|
||||
__func__, __LINE__, __x); \
|
||||
if (!ret) { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
} else if (__ONCE_LITE_IF(ret)) { \
|
||||
pr_info("WARN at %s:%d\n", \
|
||||
__func__, __LINE__); \
|
||||
mas_dump(__mas); \
|
||||
mt_dump((__mas)->tree, mt_dump_hex); \
|
||||
pr_info("Pass: %u Run:%u\n", \
|
||||
atomic_read(&maple_tree_tests_passed), \
|
||||
atomic_read(&maple_tree_tests_run)); \
|
||||
dump_stack(); \
|
||||
} else { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
WARN_ON(1); \
|
||||
} \
|
||||
unlikely(ret); \
|
||||
})
|
||||
@@ -761,18 +761,18 @@ void mt_cache_shrink(void);
|
||||
#define MAS_WR_WARN_ON(__wrmas, __x) ({ \
|
||||
int ret = !!(__x); \
|
||||
atomic_inc(&maple_tree_tests_run); \
|
||||
if (ret) { \
|
||||
pr_info("WARN at %s:%d (%u)\n", \
|
||||
__func__, __LINE__, __x); \
|
||||
if (!ret) { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
} else if (__ONCE_LITE_IF(ret)) { \
|
||||
pr_info("WARN at %s:%d\n", \
|
||||
__func__, __LINE__); \
|
||||
mas_wr_dump(__wrmas); \
|
||||
mas_dump((__wrmas)->mas); \
|
||||
mt_dump((__wrmas)->mas->tree, mt_dump_hex); \
|
||||
pr_info("Pass: %u Run:%u\n", \
|
||||
atomic_read(&maple_tree_tests_passed), \
|
||||
atomic_read(&maple_tree_tests_run)); \
|
||||
dump_stack(); \
|
||||
} else { \
|
||||
atomic_inc(&maple_tree_tests_passed); \
|
||||
WARN_ON(1); \
|
||||
} \
|
||||
unlikely(ret); \
|
||||
})
|
||||
@@ -780,9 +780,9 @@ void mt_cache_shrink(void);
|
||||
#define MT_BUG_ON(__tree, __x) BUG_ON(__x)
|
||||
#define MAS_BUG_ON(__mas, __x) BUG_ON(__x)
|
||||
#define MAS_WR_BUG_ON(__mas, __x) BUG_ON(__x)
|
||||
#define MT_WARN_ON(__tree, __x) WARN_ON(__x)
|
||||
#define MAS_WARN_ON(__mas, __x) WARN_ON(__x)
|
||||
#define MAS_WR_WARN_ON(__mas, __x) WARN_ON(__x)
|
||||
#define MT_WARN_ON(__tree, __x) WARN_ON_ONCE(__x)
|
||||
#define MAS_WARN_ON(__mas, __x) WARN_ON_ONCE(__x)
|
||||
#define MAS_WR_WARN_ON(__mas, __x) WARN_ON_ONCE(__x)
|
||||
#endif /* CONFIG_DEBUG_MAPLE_TREE */
|
||||
|
||||
/**
|
||||
|
||||
+20
-19
@@ -4032,23 +4032,6 @@ static inline void pagetable_free_kernel(struct ptdesc *pt)
|
||||
__pagetable_free(pt);
|
||||
}
|
||||
#endif
|
||||
/**
|
||||
* pagetable_free - Free pagetables
|
||||
* @pt: The page table descriptor
|
||||
*
|
||||
* pagetable_free frees the memory of all page tables described by a page
|
||||
* table descriptor and the memory for the descriptor itself.
|
||||
*/
|
||||
static inline void pagetable_free(struct ptdesc *pt)
|
||||
{
|
||||
if (ptdesc_test_kernel(pt)) {
|
||||
ptdesc_clear_kernel(pt);
|
||||
pagetable_free_kernel(pt);
|
||||
} else {
|
||||
__pagetable_free(pt);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(CONFIG_SPLIT_PTE_PTLOCKS)
|
||||
#if ALLOC_SPLIT_PTLOCKS
|
||||
void __init ptlock_cache_init(void);
|
||||
@@ -4141,6 +4124,26 @@ static inline void pagetable_dtor(struct ptdesc *ptdesc)
|
||||
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
|
||||
}
|
||||
|
||||
/**
|
||||
* pagetable_free - Free pagetables
|
||||
* @pt: The page table descriptor
|
||||
*
|
||||
* pagetable_free frees the memory of all page tables described by a page
|
||||
* table descriptor and the memory for the descriptor itself.
|
||||
*/
|
||||
static inline void pagetable_free(struct ptdesc *pt)
|
||||
{
|
||||
if (PageTable(ptdesc_page(pt)))
|
||||
pagetable_dtor(pt);
|
||||
|
||||
if (ptdesc_test_kernel(pt)) {
|
||||
ptdesc_clear_kernel(pt);
|
||||
pagetable_free_kernel(pt);
|
||||
} else {
|
||||
__pagetable_free(pt);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void pagetable_dtor_free(struct ptdesc *ptdesc)
|
||||
{
|
||||
pagetable_dtor(ptdesc);
|
||||
@@ -4428,9 +4431,7 @@ anon_rmap_tree_iter_first(struct anon_vma *anon_vma,
|
||||
struct anon_vma_chain *
|
||||
anon_rmap_tree_iter_next(struct anon_vma_chain *avc,
|
||||
pgoff_t pgoff_start, pgoff_t pgoff_last);
|
||||
#ifdef CONFIG_DEBUG_VM_RB
|
||||
void anon_rmap_tree_verify(struct anon_vma_chain *avc);
|
||||
#endif
|
||||
|
||||
#define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \
|
||||
for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \
|
||||
|
||||
@@ -277,6 +277,7 @@ extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end, struct list_head *uf);
|
||||
extern void userfaultfd_unmap_complete(struct mm_struct *mm,
|
||||
struct list_head *uf);
|
||||
void userfaultfd_unmap_abort(struct list_head *uf);
|
||||
extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma);
|
||||
extern bool userfaultfd_wp_async(struct vm_area_struct *vma);
|
||||
extern bool userfaultfd_rwp_async(struct vm_area_struct *vma);
|
||||
@@ -443,6 +444,10 @@ static inline void userfaultfd_unmap_complete(struct mm_struct *mm,
|
||||
{
|
||||
}
|
||||
|
||||
static inline void userfaultfd_unmap_abort(struct list_head *uf)
|
||||
{
|
||||
}
|
||||
|
||||
static inline bool uffd_disable_fault_around(struct vm_area_struct *vma)
|
||||
{
|
||||
return false;
|
||||
|
||||
@@ -72,7 +72,7 @@ DEFINE_EVENT(hugetlbfs__inode, hugetlbfs_evict_inode,
|
||||
TP_ARGS(inode)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(hugetlbfs__inode, hugetlbfs_free_inode,
|
||||
DEFINE_EVENT(hugetlbfs__inode, hugetlbfs_destroy_inode,
|
||||
|
||||
TP_PROTO(struct inode *inode),
|
||||
|
||||
|
||||
+93
-27
@@ -3891,14 +3891,64 @@ retry:
|
||||
goto retry;
|
||||
}
|
||||
|
||||
static __always_inline bool mas_rewalk_if_dead(struct ma_state *mas,
|
||||
struct maple_node *node, const unsigned long index)
|
||||
/*
|
||||
* mas_next_no_node() - Finish a next walk that restarted into a tree without
|
||||
* nodes.
|
||||
* @mas: The inactive maple state (ma_none or ma_root)
|
||||
*
|
||||
* The arrival here is either because there is a root value or nothing. Either
|
||||
* way, the next entry is nothing. So set the limits and overwrite the status.
|
||||
*
|
||||
* Return: NULL
|
||||
*/
|
||||
static inline void *mas_next_no_node(struct ma_state *mas)
|
||||
{
|
||||
if (unlikely(ma_dead_node(node))) {
|
||||
mas_rewalk(mas, index);
|
||||
return true;
|
||||
if (mas_is_ptr(mas))
|
||||
mas->index = 1;
|
||||
else
|
||||
mas->index = 0;
|
||||
|
||||
mas->last = ULONG_MAX;
|
||||
mas->status = ma_none;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* mas_prev_no_node() - Finish a prev walk that restarted into a tree without
|
||||
* nodes.
|
||||
* @mas: The inactive maple state (ma_none or ma_root)
|
||||
* @index: The start of the range the walk left
|
||||
* @min: The lowest index to return
|
||||
*
|
||||
* The arrival here is either because there is a root value or nothing. With
|
||||
* nothing, leave the limits mas_walk() sets for an empty tree. A root value
|
||||
* sits at 0: stop on it as mas_prev_setup() does, unless the walk was at 0.
|
||||
*
|
||||
* Return: The root entry if @index and @min allow it, NULL otherwise. A node
|
||||
* if the root changed again since the walk; the caller restarts.
|
||||
*/
|
||||
static inline void *mas_prev_no_node(struct ma_state *mas, unsigned long index,
|
||||
unsigned long min)
|
||||
{
|
||||
void *entry;
|
||||
|
||||
mas->index = 0;
|
||||
if (mas_is_none(mas)) {
|
||||
mas->last = ULONG_MAX;
|
||||
return NULL;
|
||||
}
|
||||
return false;
|
||||
|
||||
mas->last = 0;
|
||||
if (!index || min)
|
||||
return NULL;
|
||||
|
||||
entry = mas_root(mas);
|
||||
if (!entry) {
|
||||
mas->status = ma_none;
|
||||
mas->last = ULONG_MAX;
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -4006,14 +4056,14 @@ retry:
|
||||
node = mas_mn(mas);
|
||||
type = mte_node_type(mas->node);
|
||||
pivots = ma_pivots(node, type);
|
||||
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
|
||||
goto retry;
|
||||
if (unlikely(ma_dead_node(node)))
|
||||
goto dead;
|
||||
|
||||
if (mas->min <= min) {
|
||||
pivot = mas_safe_min(mas, pivots, mas->offset);
|
||||
|
||||
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
|
||||
goto retry;
|
||||
if (unlikely(ma_dead_node(node)))
|
||||
goto dead;
|
||||
|
||||
if (pivot <= min)
|
||||
goto underflow;
|
||||
@@ -4028,10 +4078,8 @@ again:
|
||||
if (mas->index <= min)
|
||||
goto underflow;
|
||||
|
||||
if (mas_prev_node(mas, min)) {
|
||||
mas_rewalk(mas, save_point);
|
||||
goto retry;
|
||||
}
|
||||
if (mas_prev_node(mas, min))
|
||||
goto dead;
|
||||
|
||||
if (WARN_ON_ONCE(mas_is_underflow(mas)))
|
||||
return NULL;
|
||||
@@ -4045,8 +4093,8 @@ again:
|
||||
|
||||
slots = ma_slots(node, type);
|
||||
entry = mas_slot(mas, slots, mas->offset);
|
||||
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
|
||||
goto retry;
|
||||
if (unlikely(ma_dead_node(node)))
|
||||
goto dead;
|
||||
|
||||
if (likely(entry))
|
||||
return entry;
|
||||
@@ -4063,6 +4111,17 @@ again:
|
||||
underflow:
|
||||
mas->status = ma_underflow;
|
||||
return NULL;
|
||||
|
||||
dead:
|
||||
mas_rewalk(mas, save_point);
|
||||
if (likely(mas_is_active(mas)))
|
||||
goto retry;
|
||||
|
||||
entry = mas_prev_no_node(mas, save_point, min);
|
||||
if (unlikely(xa_is_node(entry))) /* The root changed again */
|
||||
goto dead;
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -4176,8 +4235,8 @@ retry:
|
||||
node = mas_mn(mas);
|
||||
type = mte_node_type(mas->node);
|
||||
pivots = ma_pivots(node, type);
|
||||
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
|
||||
goto retry;
|
||||
if (unlikely(ma_dead_node(node)))
|
||||
goto dead;
|
||||
|
||||
if (mas->max >= max) {
|
||||
if (likely(mas->offset < mas->end))
|
||||
@@ -4185,8 +4244,8 @@ retry:
|
||||
else
|
||||
pivot = mas->max;
|
||||
|
||||
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
|
||||
goto retry;
|
||||
if (unlikely(ma_dead_node(node)))
|
||||
goto dead;
|
||||
|
||||
if (pivot >= max) { /* Was at the limit, next will extend beyond */
|
||||
mas->status = ma_overflow;
|
||||
@@ -4208,10 +4267,8 @@ again:
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (mas_next_node(mas, node, max)) {
|
||||
mas_rewalk(mas, save_point);
|
||||
goto retry;
|
||||
}
|
||||
if (mas_next_node(mas, node, max))
|
||||
goto dead;
|
||||
|
||||
if (WARN_ON_ONCE(mas_is_overflow(mas)))
|
||||
return NULL;
|
||||
@@ -4226,8 +4283,8 @@ again:
|
||||
|
||||
slots = ma_slots(node, type);
|
||||
entry = mt_slot(mas->tree, slots, mas->offset);
|
||||
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
|
||||
goto retry;
|
||||
if (unlikely(ma_dead_node(node)))
|
||||
goto dead;
|
||||
|
||||
if (entry)
|
||||
return entry;
|
||||
@@ -4244,6 +4301,13 @@ again:
|
||||
}
|
||||
|
||||
return entry;
|
||||
|
||||
dead:
|
||||
mas_rewalk(mas, save_point);
|
||||
if (likely(mas_is_active(mas)))
|
||||
goto retry;
|
||||
|
||||
return mas_next_no_node(mas);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -5450,7 +5514,9 @@ void *mas_find(struct ma_state *mas, unsigned long max)
|
||||
/* Retries on dead nodes handled by mas_next_slot */
|
||||
entry = mas_next_slot(mas, max, false);
|
||||
/* Ignore overflow */
|
||||
mas->status = ma_active;
|
||||
if (mas_is_overflow(mas))
|
||||
mas->status = ma_active;
|
||||
|
||||
return entry;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(mas_find);
|
||||
|
||||
@@ -3740,6 +3740,170 @@ static noinline void __init check_range64_in_rcu(struct maple_tree *mt)
|
||||
rcu_barrier();
|
||||
}
|
||||
|
||||
/*
|
||||
* A reader that restarts after its leaf died may find a tree without nodes:
|
||||
* emptied, or emptied and then given a single entry at 0. Each case starts
|
||||
* from a tree with nodes, built by check_seq(). The stores run while the
|
||||
* reader holds rcu_read_lock, so they use GFP_ATOMIC.
|
||||
*/
|
||||
static noinline void __init check_restart_no_node(struct maple_tree *mt)
|
||||
{
|
||||
void *ptr = xa_mk_value(1234);
|
||||
void *entry;
|
||||
MA_STATE(mas, mt, 0, 0);
|
||||
|
||||
MT_BUG_ON(mt, !mtree_empty(mt));
|
||||
mt_set_non_kernel(99999);
|
||||
|
||||
/* find: the tree is emptied */
|
||||
mt_init_flags(mt, MT_FLAGS_USE_RCU);
|
||||
check_seq(mt, 100, false);
|
||||
rcu_read_lock();
|
||||
mas_set(&mas, 0);
|
||||
entry = mas_find(&mas, ULONG_MAX);
|
||||
MT_BUG_ON(mt, entry != xa_mk_value(0));
|
||||
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
|
||||
entry = mas_find(&mas, ULONG_MAX);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_none);
|
||||
MT_BUG_ON(mt, mas.index != 0);
|
||||
MT_BUG_ON(mt, mas.last != ULONG_MAX);
|
||||
entry = mas_find(&mas, ULONG_MAX);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_none);
|
||||
rcu_read_unlock();
|
||||
mtree_destroy(mt);
|
||||
rcu_barrier();
|
||||
|
||||
/* next: the tree is emptied */
|
||||
mt_init_flags(mt, MT_FLAGS_USE_RCU);
|
||||
check_seq(mt, 100, false);
|
||||
rcu_read_lock();
|
||||
mas_set(&mas, 0);
|
||||
entry = mas_find(&mas, ULONG_MAX);
|
||||
MT_BUG_ON(mt, entry != xa_mk_value(0));
|
||||
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
|
||||
entry = mas_next(&mas, ULONG_MAX);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_none);
|
||||
MT_BUG_ON(mt, mas.index != 0);
|
||||
MT_BUG_ON(mt, mas.last != ULONG_MAX);
|
||||
rcu_read_unlock();
|
||||
mtree_destroy(mt);
|
||||
rcu_barrier();
|
||||
|
||||
/* find: the tree is emptied, then holds one entry at 0 */
|
||||
mt_init_flags(mt, MT_FLAGS_USE_RCU);
|
||||
check_seq(mt, 100, false);
|
||||
rcu_read_lock();
|
||||
mas_set(&mas, 0);
|
||||
entry = mas_find(&mas, ULONG_MAX);
|
||||
MT_BUG_ON(mt, entry != xa_mk_value(0));
|
||||
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
|
||||
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
|
||||
entry = mas_find(&mas, ULONG_MAX);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_none);
|
||||
MT_BUG_ON(mt, mas.index != 1);
|
||||
MT_BUG_ON(mt, mas.last != ULONG_MAX);
|
||||
rcu_read_unlock();
|
||||
mtree_destroy(mt);
|
||||
rcu_barrier();
|
||||
|
||||
/* find_rev: the tree is emptied */
|
||||
mt_init_flags(mt, MT_FLAGS_USE_RCU);
|
||||
check_seq(mt, 100, false);
|
||||
rcu_read_lock();
|
||||
mas_set(&mas, ULONG_MAX);
|
||||
entry = mas_find_rev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != xa_mk_value(100));
|
||||
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
|
||||
entry = mas_find_rev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_none);
|
||||
MT_BUG_ON(mt, mas.index != 0);
|
||||
MT_BUG_ON(mt, mas.last != ULONG_MAX);
|
||||
entry = mas_find_rev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_none);
|
||||
rcu_read_unlock();
|
||||
mtree_destroy(mt);
|
||||
rcu_barrier();
|
||||
|
||||
/* prev: the tree is emptied, then holds one entry at 0 */
|
||||
mt_init_flags(mt, MT_FLAGS_USE_RCU);
|
||||
check_seq(mt, 100, false);
|
||||
rcu_read_lock();
|
||||
mas_set(&mas, ULONG_MAX);
|
||||
entry = mas_find_rev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != xa_mk_value(100));
|
||||
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
|
||||
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
|
||||
entry = mas_prev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != ptr);
|
||||
MT_BUG_ON(mt, mas.status != ma_root);
|
||||
MT_BUG_ON(mt, mas.index != 0);
|
||||
MT_BUG_ON(mt, mas.last != 0);
|
||||
rcu_read_unlock();
|
||||
mtree_destroy(mt);
|
||||
rcu_barrier();
|
||||
|
||||
/* find_rev: the entry at 0 is below the minimum */
|
||||
mt_init_flags(mt, MT_FLAGS_USE_RCU);
|
||||
check_seq(mt, 100, false);
|
||||
rcu_read_lock();
|
||||
mas_set(&mas, ULONG_MAX);
|
||||
entry = mas_find_rev(&mas, 1);
|
||||
MT_BUG_ON(mt, entry != xa_mk_value(100));
|
||||
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
|
||||
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
|
||||
entry = mas_find_rev(&mas, 1);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_root);
|
||||
MT_BUG_ON(mt, mas.index != 0);
|
||||
MT_BUG_ON(mt, mas.last != 0);
|
||||
rcu_read_unlock();
|
||||
mtree_destroy(mt);
|
||||
rcu_barrier();
|
||||
|
||||
/* find_rev: the reader is at 0 when the tree gets one entry at 0 */
|
||||
mt_init_flags(mt, MT_FLAGS_USE_RCU);
|
||||
check_seq(mt, 100, false);
|
||||
rcu_read_lock();
|
||||
mas_set(&mas, 0);
|
||||
entry = mas_find_rev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != xa_mk_value(0));
|
||||
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
|
||||
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
|
||||
entry = mas_find_rev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_root);
|
||||
MT_BUG_ON(mt, mas.index != 0);
|
||||
MT_BUG_ON(mt, mas.last != 0);
|
||||
rcu_read_unlock();
|
||||
mtree_destroy(mt);
|
||||
rcu_barrier();
|
||||
|
||||
/* find_range_rev: the reader is at 0 when the tree gets one entry at 0 */
|
||||
mt_init_flags(mt, MT_FLAGS_USE_RCU);
|
||||
check_seq(mt, 100, false);
|
||||
rcu_read_lock();
|
||||
mas_set(&mas, 0);
|
||||
entry = mas_find_range_rev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != xa_mk_value(0));
|
||||
MT_BUG_ON(mt, mtree_store_range(mt, 0, ULONG_MAX, NULL, GFP_ATOMIC));
|
||||
MT_BUG_ON(mt, mtree_store(mt, 0, ptr, GFP_ATOMIC));
|
||||
entry = mas_find_range_rev(&mas, 0);
|
||||
MT_BUG_ON(mt, entry != NULL);
|
||||
MT_BUG_ON(mt, mas.status != ma_root);
|
||||
MT_BUG_ON(mt, mas.index != 0);
|
||||
MT_BUG_ON(mt, mas.last != 0);
|
||||
rcu_read_unlock();
|
||||
mtree_destroy(mt);
|
||||
rcu_barrier();
|
||||
mt_set_non_kernel(0);
|
||||
}
|
||||
|
||||
static DEFINE_MTREE(tree);
|
||||
static int __init maple_tree_seed(void)
|
||||
{
|
||||
@@ -4031,6 +4195,7 @@ static int __init maple_tree_seed(void)
|
||||
|
||||
check_range64_in_rcu(&tree);
|
||||
|
||||
check_restart_no_node(&tree);
|
||||
|
||||
#if defined(BENCH)
|
||||
skip:
|
||||
|
||||
@@ -1247,6 +1247,75 @@ static noinline void check_multi_find_3(struct xarray *xa)
|
||||
}
|
||||
}
|
||||
|
||||
static noinline void check_multi_find_4(struct xarray *xa)
|
||||
{
|
||||
#ifdef CONFIG_XARRAY_MULTI
|
||||
unsigned int order = XA_CHUNK_SHIFT + 1;
|
||||
unsigned long next = 1UL << order;
|
||||
unsigned long start = next - 1;
|
||||
XA_STATE(xas, xa, start);
|
||||
XA_STATE_ORDER(split, xa, 0, 0);
|
||||
void *entry;
|
||||
unsigned long i;
|
||||
|
||||
/* Multi-index entry in two slots of a non-leaf node. */
|
||||
xa_store_order(xa, 0, order, xa_mk_index(0), GFP_KERNEL);
|
||||
XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL);
|
||||
|
||||
rcu_read_lock();
|
||||
entry = xas_find(&xas, ULONG_MAX);
|
||||
XA_BUG_ON(xa, entry != xa_mk_index(0));
|
||||
XA_BUG_ON(xa, xas.xa_index != start);
|
||||
|
||||
entry = xas_find(&xas, ULONG_MAX);
|
||||
XA_BUG_ON(xa, entry != xa_mk_index(next));
|
||||
XA_BUG_ON(xa, xas.xa_index != next);
|
||||
|
||||
entry = xas_find(&xas, ULONG_MAX);
|
||||
XA_BUG_ON(xa, entry != NULL);
|
||||
rcu_read_unlock();
|
||||
|
||||
xa_erase_index(xa, next);
|
||||
xa_erase_index(xa, 0);
|
||||
XA_BUG_ON(xa, !xa_empty(xa));
|
||||
|
||||
/* Split the multi-index entry after a lookup begins inside it. */
|
||||
xa_store_order(xa, 0, order, xa_mk_index(0), GFP_KERNEL);
|
||||
XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL);
|
||||
|
||||
xas_set(&xas, start);
|
||||
rcu_read_lock();
|
||||
entry = xas_find(&xas, ULONG_MAX);
|
||||
XA_BUG_ON(xa, entry != xa_mk_index(0));
|
||||
XA_BUG_ON(xa, xas.xa_index != start);
|
||||
rcu_read_unlock();
|
||||
|
||||
xas_split_alloc(&split, xa_mk_index(0), order, GFP_KERNEL);
|
||||
if (xas_error(&split)) {
|
||||
XA_BUG_ON(xa, true);
|
||||
goto out;
|
||||
}
|
||||
xas_lock(&split);
|
||||
xas_split(&split, xa_mk_index(0), order);
|
||||
for (i = 0; i < next; i++)
|
||||
__xa_store(xa, i, xa_mk_index(i), 0);
|
||||
xas_unlock(&split);
|
||||
|
||||
rcu_read_lock();
|
||||
entry = xas_find(&xas, ULONG_MAX);
|
||||
XA_BUG_ON(xa, entry != xa_mk_index(next));
|
||||
XA_BUG_ON(xa, xas.xa_index != next);
|
||||
|
||||
entry = xas_find(&xas, ULONG_MAX);
|
||||
XA_BUG_ON(xa, entry != NULL);
|
||||
rcu_read_unlock();
|
||||
|
||||
out:
|
||||
xa_destroy(xa);
|
||||
XA_BUG_ON(xa, !xa_empty(xa));
|
||||
#endif
|
||||
}
|
||||
|
||||
static noinline void check_find_1(struct xarray *xa)
|
||||
{
|
||||
unsigned long i, j, k;
|
||||
@@ -1370,6 +1439,7 @@ static noinline void check_find(struct xarray *xa)
|
||||
check_multi_find_1(xa, i);
|
||||
check_multi_find_2(xa);
|
||||
check_multi_find_3(xa);
|
||||
check_multi_find_4(xa);
|
||||
}
|
||||
|
||||
/* See find_swap_entry() in mm/shmem.c */
|
||||
|
||||
+7
-4
@@ -471,11 +471,12 @@ static void xas_shrink(struct xa_state *xas)
|
||||
if (!xa_is_node(entry))
|
||||
RCU_INIT_POINTER(node->slots[0], XA_RETRY_ENTRY);
|
||||
xas_update(xas, node);
|
||||
if (xa_is_node(entry))
|
||||
xa_to_node(entry)->parent = NULL;
|
||||
xa_node_free(node);
|
||||
if (!xa_is_node(entry))
|
||||
break;
|
||||
node = xa_to_node(entry);
|
||||
node->parent = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1409,9 +1410,11 @@ void *xas_find(struct xa_state *xas, unsigned long max)
|
||||
entry = xas_load(xas);
|
||||
if (entry || xas_not_node(xas->xa_node))
|
||||
return entry;
|
||||
} else if (!xas->xa_node->shift &&
|
||||
xas->xa_offset != (xas->xa_index & XA_CHUNK_MASK)) {
|
||||
xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1;
|
||||
} else if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node)) {
|
||||
if (!xas->xa_node->shift)
|
||||
xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1;
|
||||
else
|
||||
xas->xa_offset = get_offset(xas->xa_index, xas->xa_node);
|
||||
}
|
||||
|
||||
xas_next_offset(xas);
|
||||
|
||||
+3
-2
@@ -3975,10 +3975,11 @@ static int kdamond_wait_activation(struct damon_ctx *ctx)
|
||||
{
|
||||
struct damos *s;
|
||||
unsigned long wait_time;
|
||||
unsigned long min_wait_time = 0;
|
||||
bool init_wait_time = false;
|
||||
|
||||
while (!kdamond_need_stop(ctx)) {
|
||||
unsigned long min_wait_time = 0;
|
||||
bool init_wait_time = false;
|
||||
|
||||
damon_for_each_scheme(s, ctx) {
|
||||
wait_time = damos_wmark_wait_us(s);
|
||||
if (!init_wait_time || wait_time < min_wait_time) {
|
||||
|
||||
@@ -79,6 +79,15 @@ static void damon_sysfs_test_add_targets(struct kunit *test)
|
||||
kunit_skip(test, "ctx alloc fail");
|
||||
}
|
||||
|
||||
if (damon_select_ops(ctx, DAMON_OPS_VADDR)) {
|
||||
damon_destroy_ctx(ctx);
|
||||
kfree(sysfs_targets->targets_arr);
|
||||
kfree(sysfs_targets);
|
||||
kfree(sysfs_target->regions);
|
||||
kfree(sysfs_target);
|
||||
kunit_skip(test, "vaddr ops not registered");
|
||||
}
|
||||
|
||||
damon_sysfs_add_targets(ctx, sysfs_targets);
|
||||
KUNIT_EXPECT_EQ(test, 1u, nr_damon_targets(ctx));
|
||||
|
||||
|
||||
@@ -3976,6 +3976,13 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf,
|
||||
unsigned long end;
|
||||
vm_fault_t map_ret;
|
||||
|
||||
/* A lower index could move vmf->pte out of its page table. */
|
||||
if (unlikely(xas.xa_index < last_pgoff)) {
|
||||
folio_unlock(folio);
|
||||
folio_put(folio);
|
||||
break;
|
||||
}
|
||||
|
||||
addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
|
||||
vmf->pte += xas.xa_index - last_pgoff;
|
||||
last_pgoff = xas.xa_index;
|
||||
|
||||
@@ -4593,10 +4593,14 @@ int folio_split(struct folio *folio, unsigned int new_order,
|
||||
* If a file-backed folio is truncated, 0 will be returned. Any subsequent
|
||||
* split attempt should get -EBUSY from split checking code.
|
||||
*
|
||||
* Context: @folio must be locked.
|
||||
*
|
||||
* Return: @folio's minimum order for split
|
||||
*/
|
||||
unsigned int min_order_for_split(struct folio *folio)
|
||||
{
|
||||
VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
|
||||
|
||||
if (folio_test_anon(folio))
|
||||
return 0;
|
||||
|
||||
|
||||
+8
-5
@@ -68,6 +68,7 @@ void kasan_set_track(struct kasan_track *track, depot_stack_handle_t stack)
|
||||
track->timestamp = ts_nsec >> 9;
|
||||
#endif /* CONFIG_KASAN_EXTRA_INFO */
|
||||
track->pid = current->pid;
|
||||
track->rcu_free = 0;
|
||||
track->stack = stack;
|
||||
}
|
||||
|
||||
@@ -231,7 +232,7 @@ void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
|
||||
|
||||
/* Returns true when freeing the object is not safe. */
|
||||
static bool check_slab_allocation(struct kmem_cache *cache, void *object,
|
||||
unsigned long ip)
|
||||
unsigned long ip, bool after_rcu_delay)
|
||||
{
|
||||
void *tagged_object = object;
|
||||
|
||||
@@ -242,7 +243,8 @@ static bool check_slab_allocation(struct kmem_cache *cache, void *object,
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!kasan_byte_accessible(tagged_object)) {
|
||||
if (!kasan_byte_accessible(tagged_object) ||
|
||||
kasan_check_rcu_free(cache, object, after_rcu_delay)) {
|
||||
kasan_report_invalid_free(tagged_object, ip, KASAN_REPORT_DOUBLE_FREE);
|
||||
return true;
|
||||
}
|
||||
@@ -259,17 +261,18 @@ static inline void poison_slab_object(struct kmem_cache *cache, void *object,
|
||||
|
||||
kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),
|
||||
KASAN_SLAB_FREE, init);
|
||||
kasan_clear_rcu_free(cache, object);
|
||||
|
||||
if (kasan_stack_collection_enabled())
|
||||
kasan_save_free_info(cache, tagged_object);
|
||||
}
|
||||
|
||||
bool __kasan_slab_pre_free(struct kmem_cache *cache, void *object,
|
||||
unsigned long ip)
|
||||
unsigned long ip, bool after_rcu_delay)
|
||||
{
|
||||
if (is_kfence_address(object))
|
||||
return false;
|
||||
return check_slab_allocation(cache, object, ip);
|
||||
return check_slab_allocation(cache, object, ip, after_rcu_delay);
|
||||
}
|
||||
|
||||
bool __kasan_slab_free(struct kmem_cache *cache, void *object, bool init,
|
||||
@@ -544,7 +547,7 @@ bool __kasan_mempool_poison_object(void *ptr, unsigned long ip)
|
||||
|
||||
slab = page_slab(page);
|
||||
|
||||
if (check_slab_allocation(slab->slab_cache, ptr, ip))
|
||||
if (check_slab_allocation(slab->slab_cache, ptr, ip, false))
|
||||
return false;
|
||||
|
||||
poison_slab_object(slab->slab_cache, ptr, false);
|
||||
|
||||
+16
-1
@@ -533,10 +533,13 @@ size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object)
|
||||
}
|
||||
|
||||
/*
|
||||
* Records an auxiliary stack trace for the object at @addr and, if @rcu_free
|
||||
* is set, marks the object as pending an RCU-delayed free.
|
||||
*
|
||||
* This function avoids dynamic memory allocations and thus can be called from
|
||||
* contexts that do not allow allocating memory.
|
||||
*/
|
||||
void kasan_record_aux_stack(void *addr)
|
||||
static void __kasan_record_aux_stack(void *addr, bool rcu_free)
|
||||
{
|
||||
struct slab *slab = kasan_addr_to_slab(addr);
|
||||
struct kmem_cache *cache;
|
||||
@@ -552,10 +555,22 @@ void kasan_record_aux_stack(void *addr)
|
||||
if (!alloc_meta)
|
||||
return;
|
||||
|
||||
if (rcu_free)
|
||||
alloc_meta->alloc_track.rcu_free = 1;
|
||||
alloc_meta->aux_stack[1] = alloc_meta->aux_stack[0];
|
||||
alloc_meta->aux_stack[0] = kasan_save_stack(0, 0);
|
||||
}
|
||||
|
||||
void kasan_record_aux_stack(void *addr)
|
||||
{
|
||||
__kasan_record_aux_stack(addr, false);
|
||||
}
|
||||
|
||||
void kasan_mark_rcu_free_pending(void *addr)
|
||||
{
|
||||
__kasan_record_aux_stack(addr, true);
|
||||
}
|
||||
|
||||
void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
|
||||
{
|
||||
struct kasan_alloc_meta *alloc_meta;
|
||||
|
||||
+33
-1
@@ -191,7 +191,10 @@ static inline bool kasan_requires_meta(void)
|
||||
#define KASAN_STACK_DEPTH 64
|
||||
|
||||
struct kasan_track {
|
||||
u32 pid;
|
||||
#define KASAN_PID_BITS const_ilog2(PID_MAX_LIMIT)
|
||||
u32 pid:KASAN_PID_BITS;
|
||||
u32 rcu_free:1;
|
||||
u32 unused:32 - (KASAN_PID_BITS + 1);
|
||||
depot_stack_handle_t stack;
|
||||
#ifdef CONFIG_KASAN_EXTRA_INFO
|
||||
u64 cpu:20;
|
||||
@@ -390,8 +393,37 @@ struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
|
||||
struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
|
||||
const void *object);
|
||||
void kasan_init_object_meta(struct kmem_cache *cache, const void *object);
|
||||
|
||||
static inline bool kasan_check_rcu_free(struct kmem_cache *cache,
|
||||
const void *object,
|
||||
bool after_rcu_delay)
|
||||
{
|
||||
struct kasan_alloc_meta *alloc_meta;
|
||||
|
||||
alloc_meta = kasan_get_alloc_meta(cache, object);
|
||||
if (!alloc_meta)
|
||||
return false;
|
||||
|
||||
return alloc_meta->alloc_track.rcu_free != after_rcu_delay;
|
||||
}
|
||||
|
||||
static inline void kasan_clear_rcu_free(struct kmem_cache *cache,
|
||||
const void *object)
|
||||
{
|
||||
struct kasan_alloc_meta *alloc_meta;
|
||||
|
||||
alloc_meta = kasan_get_alloc_meta(cache, object);
|
||||
if (alloc_meta)
|
||||
alloc_meta->alloc_track.rcu_free = 0;
|
||||
}
|
||||
#else
|
||||
static inline void kasan_init_object_meta(struct kmem_cache *cache, const void *object) { }
|
||||
static inline bool kasan_check_rcu_free(struct kmem_cache *cache, const void *object,
|
||||
bool after_rcu_delay)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
static inline void kasan_clear_rcu_free(struct kmem_cache *cache, const void *object) { }
|
||||
#endif
|
||||
|
||||
depot_stack_handle_t kasan_save_stack(gfp_t flags, depot_flags_t depot_flags);
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
#include <asm/page.h>
|
||||
|
||||
#include "kasan.h"
|
||||
#include "../slab.h"
|
||||
|
||||
#define OOB_TAG_OFF (IS_ENABLED(CONFIG_KASAN_GENERIC) ? 0 : KASAN_GRANULE_SIZE)
|
||||
|
||||
@@ -1070,6 +1071,139 @@ static void kmem_cache_double_free(struct kunit *test)
|
||||
kmem_cache_destroy(cache);
|
||||
}
|
||||
|
||||
#ifndef MODULE
|
||||
static void __kmem_cache_double_free_rcu(struct kunit *test, int mode)
|
||||
{
|
||||
struct kasan_rcu_info *p, *victim = NULL, **objs;
|
||||
struct kmem_cache *cache;
|
||||
int i, n = 0;
|
||||
|
||||
KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC);
|
||||
KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_PREEMPT_RT);
|
||||
|
||||
cache = kmem_cache_create("test_cache", sizeof(*p), 0, SLAB_NO_MERGE,
|
||||
NULL);
|
||||
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache);
|
||||
|
||||
if (!cache->sheaf_capacity) {
|
||||
kmem_cache_destroy(cache);
|
||||
kunit_skip(test, "Test requires sheaves");
|
||||
}
|
||||
|
||||
objs = kunit_kcalloc(test, cache->sheaf_capacity * 2, sizeof(*objs),
|
||||
GFP_KERNEL);
|
||||
if (!objs) {
|
||||
kunit_err(test, "Allocation failed: %s\n", __func__);
|
||||
goto out;
|
||||
}
|
||||
|
||||
migrate_disable();
|
||||
|
||||
p = kmem_cache_alloc(cache, GFP_KERNEL);
|
||||
if (!p) {
|
||||
migrate_enable();
|
||||
kunit_err(test, "Allocation failed: %s\n", __func__);
|
||||
goto out;
|
||||
}
|
||||
|
||||
switch (mode) {
|
||||
case 1:
|
||||
kmem_cache_free(cache, p);
|
||||
/* p is already in quarantine. */
|
||||
KUNIT_EXPECT_KASAN_FAIL(test, kfree_rcu(p, rcu));
|
||||
migrate_enable();
|
||||
break;
|
||||
case 2:
|
||||
/* The actual free order is inverse. */
|
||||
kfree_rcu(p, rcu);
|
||||
KUNIT_EXPECT_KASAN_FAIL(test, kmem_cache_free(cache, p));
|
||||
migrate_enable();
|
||||
break;
|
||||
case 3:
|
||||
kfree_rcu(p, rcu);
|
||||
|
||||
for (i = 0; i < cache->sheaf_capacity - 1; i++) {
|
||||
objs[i] = kmem_cache_alloc(cache, GFP_KERNEL);
|
||||
kfree_rcu(objs[i], rcu);
|
||||
}
|
||||
|
||||
KUNIT_EXPECT_KASAN_FAIL(test, kfree_rcu(p, rcu));
|
||||
migrate_enable();
|
||||
|
||||
/*
|
||||
* Move the 1st full sheaf to quarantine, keeping the
|
||||
* 2nd in pcs->rcu_free.
|
||||
*/
|
||||
rcu_barrier();
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* Move p to quarantine if not poisoned yet (case 2,
|
||||
* or fallback in case 3).
|
||||
*/
|
||||
if (kasan_byte_accessible(p))
|
||||
kvfree_rcu_barrier_on_cache(cache);
|
||||
|
||||
/*
|
||||
* Evict p from quarantine and reallocate it as a live victim object
|
||||
* before the RCU sheaf is flushed.
|
||||
*/
|
||||
kasan_quarantine_remove_cache(cache);
|
||||
|
||||
for (i = 0; i < cache->sheaf_capacity * 2; i++) {
|
||||
objs[n] = kmem_cache_alloc(cache, GFP_KERNEL);
|
||||
if (!objs[n])
|
||||
break;
|
||||
if (objs[n] == p) {
|
||||
victim = objs[n];
|
||||
break;
|
||||
}
|
||||
n++;
|
||||
}
|
||||
|
||||
for (i = 0; i < n; i++)
|
||||
kmem_cache_free(cache, objs[i]);
|
||||
|
||||
KUNIT_EXPECT_PTR_EQ(test, victim, p);
|
||||
if (victim != p)
|
||||
goto out;
|
||||
|
||||
WRITE_ONCE(victim->i, 42);
|
||||
|
||||
/*
|
||||
* If kfree_rcu() failed to reject the double-free above,
|
||||
* flushing the RCU sheaf will silently poison the live
|
||||
* victim and cause a false-positive UAF on victim->i.
|
||||
*/
|
||||
kvfree_rcu_barrier_on_cache(cache);
|
||||
KUNIT_EXPECT_EQ(test, READ_ONCE(victim->i), 42);
|
||||
|
||||
if (!READ_ONCE(test_status.report_found))
|
||||
kmem_cache_free(cache, victim);
|
||||
out:
|
||||
kmem_cache_destroy(cache);
|
||||
}
|
||||
|
||||
static void kmem_cache_double_free_rcu_1(struct kunit *test)
|
||||
{
|
||||
/* kmem_cache_free() -> kfree_rcu() */
|
||||
__kmem_cache_double_free_rcu(test, 1);
|
||||
}
|
||||
|
||||
static void kmem_cache_double_free_rcu_2(struct kunit *test)
|
||||
{
|
||||
/* kfree_rcu() -> kmem_cache_free() */
|
||||
__kmem_cache_double_free_rcu(test, 2);
|
||||
}
|
||||
|
||||
static void kmem_cache_double_free_rcu_3(struct kunit *test)
|
||||
{
|
||||
/* kfree_rcu() -> kfree_rcu() */
|
||||
__kmem_cache_double_free_rcu(test, 3);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void kmem_cache_invalid_free(struct kunit *test)
|
||||
{
|
||||
char *p;
|
||||
@@ -2239,6 +2373,11 @@ static struct kunit_case kasan_kunit_test_cases[] = {
|
||||
KUNIT_CASE(kfree_via_phys),
|
||||
KUNIT_CASE(kmem_cache_oob),
|
||||
KUNIT_CASE(kmem_cache_double_free),
|
||||
#ifndef MODULE
|
||||
KUNIT_CASE(kmem_cache_double_free_rcu_1),
|
||||
KUNIT_CASE(kmem_cache_double_free_rcu_2),
|
||||
KUNIT_CASE(kmem_cache_double_free_rcu_3),
|
||||
#endif
|
||||
KUNIT_CASE(kmem_cache_invalid_free),
|
||||
KUNIT_CASE(kmem_cache_rcu_uaf),
|
||||
KUNIT_CASE(kmem_cache_rcu_reuse),
|
||||
|
||||
+24
-45
@@ -1320,13 +1320,13 @@ static bool panic_on_unrecoverable_mf(enum mf_action_page_type type,
|
||||
static int action_result(unsigned long pfn, enum mf_action_page_type type,
|
||||
enum mf_result result)
|
||||
{
|
||||
trace_memory_failure_event(pfn, type, result);
|
||||
|
||||
if (type != MF_MSG_ALREADY_POISONED && type != MF_MSG_PFN_MAP) {
|
||||
num_poisoned_pages_inc(pfn);
|
||||
update_per_node_mf_stats(pfn, result);
|
||||
}
|
||||
|
||||
trace_memory_failure_event(pfn, type, result);
|
||||
|
||||
pr_err("%#lx: recovery action for %s: %s\n",
|
||||
pfn, action_page_types[type], action_name[result]);
|
||||
|
||||
@@ -1744,26 +1744,6 @@ static int identify_page_state(unsigned long pfn, struct page *p,
|
||||
return page_action(ps, p, pfn);
|
||||
}
|
||||
|
||||
/*
|
||||
* When 'release' is 'false', it means that if thp split has failed,
|
||||
* there is still more to do, hence the page refcount we took earlier
|
||||
* is still needed.
|
||||
*/
|
||||
static int try_to_split_thp_page(struct page *page, unsigned int new_order,
|
||||
bool release)
|
||||
{
|
||||
int ret;
|
||||
|
||||
lock_page(page);
|
||||
ret = split_huge_page_to_order(page, new_order);
|
||||
unlock_page(page);
|
||||
|
||||
if (ret && release)
|
||||
put_page(page);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn,
|
||||
struct address_space *mapping, pgoff_t index, int flags)
|
||||
{
|
||||
@@ -2548,9 +2528,6 @@ try_again:
|
||||
folio_unlock(folio);
|
||||
|
||||
if (folio_test_large(folio)) {
|
||||
const int new_order = min_order_for_split(folio);
|
||||
int err;
|
||||
|
||||
/*
|
||||
* The flag must be set after the refcount is bumped
|
||||
* otherwise it may race with THP split.
|
||||
@@ -2565,24 +2542,25 @@ try_again:
|
||||
* page is a valid handlable page.
|
||||
*/
|
||||
folio_set_has_hwpoisoned(folio);
|
||||
err = try_to_split_thp_page(p, new_order, /* release= */ false);
|
||||
|
||||
folio_lock(folio);
|
||||
split_huge_page_to_order(p, min_order_for_split(folio));
|
||||
folio = page_folio(p);
|
||||
folio_unlock(folio);
|
||||
|
||||
/*
|
||||
* If splitting a folio to order-0 fails, kill the process.
|
||||
* Split the folio regardless to minimize unusable pages.
|
||||
* Because the memory failure code cannot handle large
|
||||
* folios, this split is always treated as if it failed.
|
||||
*/
|
||||
if (err || new_order) {
|
||||
/* get folio again in case the original one is split */
|
||||
folio = page_folio(p);
|
||||
if (folio_test_large(folio)) {
|
||||
res = -EHWPOISON;
|
||||
kill_procs_now(p, pfn, flags, folio);
|
||||
put_page(p);
|
||||
folio_put(folio);
|
||||
action_result(pfn, MF_MSG_UNSPLIT_THP, MF_FAILED);
|
||||
goto unlock_mutex;
|
||||
}
|
||||
VM_BUG_ON_PAGE(!page_count(p), p);
|
||||
folio = page_folio(p);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -2900,25 +2878,26 @@ static int soft_offline_in_use_page(struct page *page)
|
||||
.reason = MR_MEMORY_FAILURE,
|
||||
};
|
||||
|
||||
folio_lock(folio);
|
||||
if (!huge && folio_test_large(folio)) {
|
||||
const int new_order = min_order_for_split(folio);
|
||||
|
||||
/*
|
||||
* If new_order (target split order) is not 0, do not split the
|
||||
* folio at all to retain the still accessible large folio.
|
||||
* NOTE: if minimizing the number of soft offline pages is
|
||||
* preferred, split it to non-zero new_order like it is done in
|
||||
* memory_failure().
|
||||
* If we cannot split to order-0, do not split the folio at all
|
||||
* to retain the still accessible large folio.
|
||||
*/
|
||||
if (new_order || try_to_split_thp_page(page, /* new_order= */ 0,
|
||||
/* release= */ true)) {
|
||||
pr_info("%#lx: thp split failed\n", pfn);
|
||||
return -EBUSY;
|
||||
}
|
||||
if (!min_order_for_split(folio))
|
||||
ret = split_huge_page_to_order(page, 0) ? -EBUSY : 0;
|
||||
else
|
||||
ret = -EBUSY;
|
||||
folio = page_folio(page);
|
||||
|
||||
if (ret) {
|
||||
folio_unlock(folio);
|
||||
folio_put(folio);
|
||||
pr_info("%#lx: thp split failed\n", pfn);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
folio_lock(folio);
|
||||
if (!huge)
|
||||
folio_wait_writeback(folio);
|
||||
if (PageHWPoison(page)) {
|
||||
|
||||
+10
-3
@@ -207,15 +207,22 @@ static u8 get_il_weight(int node)
|
||||
*/
|
||||
static void reduce_interleave_weights(unsigned int *bw, u8 *new_iw)
|
||||
{
|
||||
nodemask_t nodes = NODE_MASK_NONE;
|
||||
u64 sum_bw = 0;
|
||||
unsigned int cast_sum_bw, scaling_factor = 1, iw_gcd = 0;
|
||||
int nid;
|
||||
|
||||
for_each_node_state(nid, N_MEMORY)
|
||||
/*
|
||||
* N_MEMORY can change while wi_state_lock is held. Remember the
|
||||
* nodes contributing to sum_bw so all passes use the same set.
|
||||
*/
|
||||
for_each_node_state(nid, N_MEMORY) {
|
||||
node_set(nid, nodes);
|
||||
sum_bw += bw[nid];
|
||||
}
|
||||
|
||||
/* Scale bandwidths to whole numbers in the range [1, weightiness] */
|
||||
for_each_node_state(nid, N_MEMORY) {
|
||||
for_each_node_mask(nid, nodes) {
|
||||
/*
|
||||
* Try not to perform 64-bit division.
|
||||
* If sum_bw < scaling_factor, then sum_bw < U32_MAX.
|
||||
@@ -234,7 +241,7 @@ static void reduce_interleave_weights(unsigned int *bw, u8 *new_iw)
|
||||
}
|
||||
|
||||
/* 1:2 is strictly better than 16:32. Reduce by the weights' GCD. */
|
||||
for_each_node_state(nid, N_MEMORY)
|
||||
for_each_node_mask(nid, nodes)
|
||||
new_iw[nid] /= iw_gcd;
|
||||
}
|
||||
|
||||
|
||||
@@ -711,6 +711,16 @@ static int __folio_migrate_mapping(struct address_space *mapping,
|
||||
mod_lruvec_state(old_lruvec, NR_SHMEM_THPS, -nr);
|
||||
mod_lruvec_state(new_lruvec, NR_SHMEM_THPS, nr);
|
||||
}
|
||||
} else if (!folio_test_swapbacked(folio) &&
|
||||
folio_test_pmd_mappable(folio)) {
|
||||
mod_lruvec_state(old_lruvec, NR_FILE_THPS, -nr);
|
||||
mod_lruvec_state(new_lruvec, NR_FILE_THPS, nr);
|
||||
}
|
||||
if (test_bit(AS_KERNEL_FILE, &mapping->flags)) {
|
||||
__mod_node_page_state(oldzone->zone_pgdat,
|
||||
NR_KERNEL_FILE_PAGES, -nr);
|
||||
__mod_node_page_state(newzone->zone_pgdat,
|
||||
NR_KERNEL_FILE_PAGES, nr);
|
||||
}
|
||||
#ifdef CONFIG_SWAP
|
||||
if (folio_test_swapcache(folio)) {
|
||||
|
||||
+14
-2
@@ -2612,11 +2612,14 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
|
||||
* and swap cache folios are never partially freed.
|
||||
*/
|
||||
folio_lock(folio);
|
||||
if (!folio_matches_swap_entry(folio, swap) ||
|
||||
shmem_confirm_swap(mapping, index, swap) < 0) {
|
||||
if (!folio_matches_swap_entry(folio, swap)) {
|
||||
error = -EEXIST;
|
||||
goto unlock;
|
||||
}
|
||||
if (shmem_confirm_swap(mapping, index, swap) < 0) {
|
||||
error = -EEXIST;
|
||||
goto failed;
|
||||
}
|
||||
if (!folio_test_uptodate(folio)) {
|
||||
error = -EIO;
|
||||
goto failed;
|
||||
@@ -2657,6 +2660,15 @@ failed:
|
||||
error = -EEXIST;
|
||||
if (error == -EIO)
|
||||
shmem_set_folio_swapin_error(inode, index, folio, swap);
|
||||
/*
|
||||
* A large folio that does not fit the mapping, e.g. because its swap
|
||||
* entries were freed and reused, would be found again on every retry.
|
||||
* Drop it from the swap cache so the retry can swap in a fitting one.
|
||||
*/
|
||||
if (error == -EEXIST && folio && folio_test_large(folio) &&
|
||||
!folio_test_anon(folio) && !folio_test_dirty(folio) &&
|
||||
!folio_test_writeback(folio))
|
||||
swap_cache_del_folio(folio);
|
||||
unlock:
|
||||
if (folio)
|
||||
folio_unlock(folio);
|
||||
|
||||
@@ -2722,7 +2722,7 @@ bool slab_free_hook(struct kmem_cache *s, void *x, bool init,
|
||||
* Give KASAN a chance to notice an invalid free operation before we
|
||||
* modify the object.
|
||||
*/
|
||||
if (kasan_slab_pre_free(s, x))
|
||||
if (kasan_slab_pre_free(s, x, after_rcu_delay))
|
||||
return false;
|
||||
|
||||
#ifdef CONFIG_SLUB_RCU_DEBUG
|
||||
@@ -2738,7 +2738,7 @@ bool slab_free_hook(struct kmem_cache *s, void *x, bool init,
|
||||
* We have to do this manually because the rcu_head is
|
||||
* not located inside the object.
|
||||
*/
|
||||
kasan_record_aux_stack(x);
|
||||
kasan_mark_rcu_free_pending(x);
|
||||
|
||||
delayed_free->object = x;
|
||||
call_rcu(&delayed_free->head, slab_free_after_rcu_debug);
|
||||
@@ -6298,6 +6298,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
|
||||
}
|
||||
|
||||
do_free:
|
||||
if (unlikely(kasan_slab_pre_free(s, obj, false))) {
|
||||
local_unlock(&s->cpu_sheaves->lock);
|
||||
goto out;
|
||||
}
|
||||
|
||||
kasan_mark_rcu_free_pending(obj);
|
||||
|
||||
rcu_sheaf = pcs->rcu_free;
|
||||
|
||||
@@ -6338,6 +6344,7 @@ do_free:
|
||||
local_unlock(&s->cpu_sheaves->lock);
|
||||
|
||||
stat(s, FREE_RCU_SHEAF);
|
||||
out:
|
||||
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
|
||||
lock_map_release(&kfree_rcu_sheaf_map);
|
||||
return true;
|
||||
@@ -6971,7 +6978,7 @@ void kfree_nolock(const void *object)
|
||||
* -> kfree_nolock() -> kasan_report_invalid_free() on the same CPU
|
||||
* is double buggy and deserves to deadlock.
|
||||
*/
|
||||
if (kasan_slab_pre_free(s, x))
|
||||
if (kasan_slab_pre_free(s, x, false))
|
||||
return;
|
||||
/*
|
||||
* memcg, kasan_slab_pre_free are done for 'x'.
|
||||
|
||||
@@ -635,6 +635,10 @@ static struct swap_cluster_info *isolate_lock_cluster(
|
||||
u8 flags = CLUSTER_FLAG_NONE;
|
||||
|
||||
spin_lock(&si->lock);
|
||||
|
||||
if (unlikely(!(si->flags & SWP_WRITEOK)))
|
||||
goto out;
|
||||
|
||||
list_for_each_entry(ci, list, list) {
|
||||
if (!spin_trylock(&ci->lock))
|
||||
continue;
|
||||
@@ -650,6 +654,7 @@ static struct swap_cluster_info *isolate_lock_cluster(
|
||||
found = ci;
|
||||
break;
|
||||
}
|
||||
out:
|
||||
spin_unlock(&si->lock);
|
||||
|
||||
/* Cluster's table is freed when and only when it's on the free list. */
|
||||
|
||||
+62
-25
@@ -1109,21 +1109,26 @@ static int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
|
||||
if (atomic_read(&ctx->mmap_changing))
|
||||
goto out_unlock;
|
||||
|
||||
/*
|
||||
* Validate every spanned VMA before changing any of them, so an
|
||||
* invalid range is rejected without changing part of it.
|
||||
*/
|
||||
err = -ENOENT;
|
||||
for_each_vma_range(vmi, dst_vma, end) {
|
||||
|
||||
if (!userfaultfd_wp(dst_vma)) {
|
||||
err = -ENOENT;
|
||||
break;
|
||||
}
|
||||
if (!userfaultfd_wp(dst_vma))
|
||||
goto out_unlock;
|
||||
|
||||
if (vma_is_hugetlb(dst_vma)) {
|
||||
err = -EINVAL;
|
||||
page_mask = vma_kernel_pagesize(dst_vma) - 1;
|
||||
if ((start & page_mask) || (len & page_mask))
|
||||
break;
|
||||
if ((start & page_mask) || (len & page_mask)) {
|
||||
err = -EINVAL;
|
||||
goto out_unlock;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
vma_iter_set(&vmi, start);
|
||||
for_each_vma_range(vmi, dst_vma, end) {
|
||||
_start = max(dst_vma->vm_start, start);
|
||||
_end = min(dst_vma->vm_end, end);
|
||||
|
||||
@@ -2196,6 +2201,32 @@ static void userfaultfd_reset_ctx(struct vm_area_struct *vma)
|
||||
userfaultfd_set_ctx(vma, NULL, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Clear uffd state from a WP or RWP range: the uffd bit, WP markers and
|
||||
* RWP PAGE_NONE entries. Call this before dropping the registration, as it
|
||||
* needs the VMA flags to know what to resolve.
|
||||
*/
|
||||
static void userfaultfd_clear_ptes(struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end)
|
||||
{
|
||||
unsigned int mm_cp_flags = 0;
|
||||
struct mmu_gather tlb;
|
||||
|
||||
if (!userfaultfd_protected(vma))
|
||||
return;
|
||||
|
||||
if (userfaultfd_wp(vma))
|
||||
mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE;
|
||||
if (userfaultfd_rwp(vma))
|
||||
mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE;
|
||||
if (vma_wants_manual_pte_write_upgrade(vma))
|
||||
mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
|
||||
|
||||
tlb_gather_mmu(&tlb, vma->vm_mm);
|
||||
change_protection(&tlb, vma, start, end, mm_cp_flags);
|
||||
tlb_finish_mmu(&tlb);
|
||||
}
|
||||
|
||||
static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi,
|
||||
struct vm_area_struct *prev,
|
||||
struct vm_area_struct *vma,
|
||||
@@ -2215,22 +2246,7 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi,
|
||||
if (start == vma->vm_start && end == vma->vm_end)
|
||||
give_up_on_oom = true;
|
||||
|
||||
/* Clear the uffd bit and/or restore protnone PTEs */
|
||||
if (userfaultfd_protected(vma)) {
|
||||
unsigned int mm_cp_flags = 0;
|
||||
struct mmu_gather tlb;
|
||||
|
||||
if (userfaultfd_wp(vma))
|
||||
mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE;
|
||||
if (userfaultfd_rwp(vma))
|
||||
mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE;
|
||||
if (vma_wants_manual_pte_write_upgrade(vma))
|
||||
mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
|
||||
|
||||
tlb_gather_mmu(&tlb, vma->vm_mm);
|
||||
change_protection(&tlb, vma, start, end, mm_cp_flags);
|
||||
tlb_finish_mmu(&tlb);
|
||||
}
|
||||
userfaultfd_clear_ptes(vma, start, end);
|
||||
|
||||
ret = vma_modify_flags_uffd(vmi, prev, vma, start, end,
|
||||
&new_vma_flags, NULL_VM_UFFD_CTX,
|
||||
@@ -2339,8 +2355,10 @@ static void userfaultfd_release_new(struct userfaultfd_ctx *ctx)
|
||||
/* the various vma->vm_userfaultfd_ctx still points to it */
|
||||
mmap_write_lock(mm);
|
||||
for_each_vma(vmi, vma) {
|
||||
if (vma->vm_userfaultfd_ctx.ctx == ctx)
|
||||
if (vma->vm_userfaultfd_ctx.ctx == ctx) {
|
||||
userfaultfd_clear_ptes(vma, vma->vm_start, vma->vm_end);
|
||||
userfaultfd_reset_ctx(vma);
|
||||
}
|
||||
}
|
||||
mmap_write_unlock(mm);
|
||||
}
|
||||
@@ -3281,6 +3299,25 @@ void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
|
||||
}
|
||||
}
|
||||
|
||||
void userfaultfd_unmap_abort(struct list_head *uf)
|
||||
{
|
||||
struct userfaultfd_unmap_ctx *unmap_ctx, *n;
|
||||
|
||||
if (!uf)
|
||||
return;
|
||||
|
||||
list_for_each_entry_safe(unmap_ctx, n, uf, list) {
|
||||
struct userfaultfd_ctx *ctx = unmap_ctx->ctx;
|
||||
|
||||
atomic_dec(&ctx->mmap_changing);
|
||||
VM_WARN_ON_ONCE(atomic_read(&ctx->mmap_changing) < 0);
|
||||
userfaultfd_ctx_put(ctx);
|
||||
|
||||
list_del(&unmap_ctx->list);
|
||||
kfree(unmap_ctx);
|
||||
}
|
||||
}
|
||||
|
||||
static int userfaultfd_release(struct inode *inode, struct file *file)
|
||||
{
|
||||
struct userfaultfd_ctx *ctx = file->private_data;
|
||||
|
||||
@@ -676,6 +676,26 @@ static int dup_anon_vma(struct vm_area_struct *dst,
|
||||
}
|
||||
|
||||
#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
|
||||
static void validate_mm_anon_rmap(struct vm_area_struct *vma)
|
||||
{
|
||||
struct anon_vma *anon_vma;
|
||||
struct anon_vma_chain *avc;
|
||||
|
||||
if (!IS_ENABLED(CONFIG_DEBUG_VM_RB))
|
||||
return;
|
||||
|
||||
vma_assert_stabilised(vma);
|
||||
|
||||
anon_vma = READ_ONCE(vma->anon_vma);
|
||||
if (!anon_vma)
|
||||
return;
|
||||
|
||||
anon_vma_lock_read(anon_vma);
|
||||
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
|
||||
anon_rmap_tree_verify(avc);
|
||||
anon_vma_unlock_read(anon_vma);
|
||||
}
|
||||
|
||||
void validate_mm(struct mm_struct *mm)
|
||||
{
|
||||
int bug = 0;
|
||||
@@ -685,10 +705,6 @@ void validate_mm(struct mm_struct *mm)
|
||||
|
||||
mt_validate(&mm->mm_mt);
|
||||
for_each_vma(vmi, vma) {
|
||||
#ifdef CONFIG_DEBUG_VM_RB
|
||||
struct anon_vma *anon_vma = vma->anon_vma;
|
||||
struct anon_vma_chain *avc;
|
||||
#endif
|
||||
unsigned long vmi_start, vmi_end;
|
||||
bool warn = 0;
|
||||
|
||||
@@ -709,14 +725,8 @@ void validate_mm(struct mm_struct *mm)
|
||||
vma_iter_dump_tree(&vmi);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_DEBUG_VM_RB
|
||||
if (vma_has_anon_rmap(vma)) {
|
||||
anon_vma_lock_read(anon_vma);
|
||||
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
|
||||
anon_rmap_tree_verify(avc);
|
||||
anon_vma_unlock_read(anon_vma);
|
||||
}
|
||||
#endif
|
||||
validate_mm_anon_rmap(vma);
|
||||
|
||||
/* Check for a infinite loop */
|
||||
if (++i > mm->map_count + 10) {
|
||||
i = -1;
|
||||
@@ -1686,9 +1696,18 @@ int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
|
||||
mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
|
||||
mt_on_stack(mt_detach);
|
||||
struct vma_munmap_struct vms;
|
||||
struct list_head *uf_unmaps;
|
||||
LIST_HEAD(uf_new_unmaps);
|
||||
int error;
|
||||
|
||||
init_vma_munmap(&vms, vmi, vma, start, end, uf, unlock);
|
||||
/*
|
||||
* @uf may already hold events of earlier successful unmaps, e.g. in
|
||||
* mremap().
|
||||
* Collect new events on a separate list so they can be dropped if this
|
||||
* unmap fails, and splice them into @uf on success.
|
||||
*/
|
||||
uf_unmaps = uf ? &uf_new_unmaps : NULL;
|
||||
init_vma_munmap(&vms, vmi, vma, start, end, uf_unmaps, unlock);
|
||||
error = vms_gather_munmap_vmas(&vms, &mas_detach);
|
||||
if (error)
|
||||
goto gather_failed;
|
||||
@@ -1698,12 +1717,15 @@ int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
|
||||
goto clear_tree_failed;
|
||||
|
||||
/* Point of no return */
|
||||
if (uf)
|
||||
list_splice_tail(&uf_new_unmaps, uf);
|
||||
vms_complete_munmap_vmas(&vms, &mas_detach);
|
||||
return 0;
|
||||
|
||||
clear_tree_failed:
|
||||
reattach_vmas(&mas_detach);
|
||||
gather_failed:
|
||||
userfaultfd_unmap_abort(&uf_new_unmaps);
|
||||
validate_mm(mm);
|
||||
return error;
|
||||
}
|
||||
@@ -2592,7 +2614,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
|
||||
if (error) {
|
||||
/* On error VMAs will already have been reattached. */
|
||||
vms->nr_pages = 0;
|
||||
return error;
|
||||
goto abort_unmaps;
|
||||
}
|
||||
|
||||
map->next = vms->next;
|
||||
@@ -2602,8 +2624,10 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
|
||||
}
|
||||
|
||||
/* Check against address space limit. */
|
||||
if (!may_expand_vm(map->mm, &map->vma_flags, map->pglen - vms->nr_pages))
|
||||
return -ENOMEM;
|
||||
if (!may_expand_vm(map->mm, &map->vma_flags, map->pglen - vms->nr_pages)) {
|
||||
error = -ENOMEM;
|
||||
goto abort_unmaps;
|
||||
}
|
||||
|
||||
/* Private writable mapping: check memory availability. */
|
||||
if (accountable_mapping(map)) {
|
||||
@@ -2613,7 +2637,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
|
||||
if (uncharged) {
|
||||
error = security_vm_enough_memory_mm(map->mm, uncharged);
|
||||
if (error)
|
||||
return error;
|
||||
goto abort_unmaps;
|
||||
}
|
||||
|
||||
/* Our new mapping owns the entire charge now. */
|
||||
@@ -2632,6 +2656,10 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
|
||||
|
||||
set_desc_from_map(desc, map);
|
||||
return 0;
|
||||
|
||||
abort_unmaps:
|
||||
userfaultfd_unmap_abort(uf);
|
||||
return error;
|
||||
}
|
||||
|
||||
static bool map_same_file(struct mmap_state *map)
|
||||
|
||||
@@ -36502,22 +36502,27 @@ static void regression_tests(void)
|
||||
test_spanning_store_regression();
|
||||
}
|
||||
|
||||
void maple_tree_tests(void)
|
||||
int maple_tree_tests(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
#if !defined(BENCH)
|
||||
regression_tests();
|
||||
farmer_tests();
|
||||
#endif
|
||||
maple_tree_seed();
|
||||
ret = maple_tree_seed();
|
||||
maple_tree_harvest();
|
||||
return ret;
|
||||
}
|
||||
|
||||
int __weak main(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
maple_tree_init();
|
||||
maple_tree_tests();
|
||||
ret = maple_tree_tests();
|
||||
rcu_barrier();
|
||||
if (nr_allocated)
|
||||
printf("nr_allocated = %d\n", nr_allocated);
|
||||
return 0;
|
||||
return ret ? EXIT_FAILURE : EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
# Kselftest framework requirement - SKIP code is 4.
|
||||
ksft_skip=4
|
||||
|
||||
if [ $EUID -ne 0 ]
|
||||
then
|
||||
echo "Run as root"
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "shared.h"
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
#include <assert.h>
|
||||
#include "linux/init.h"
|
||||
|
||||
void maple_rcu_cb(struct rcu_head *head);
|
||||
@@ -20,5 +21,37 @@ do { \
|
||||
call_rcu(&((_p_struct)->_memb), rcu_cb); \
|
||||
} while(0);
|
||||
|
||||
/* Make WARN() fatal */
|
||||
#undef WARN
|
||||
#undef WARN_ON
|
||||
#undef WARN_ONCE
|
||||
#undef WARN_ON_ONCE
|
||||
#define WARN(condition, format...) ({ \
|
||||
int __ret_warn_on = !!(condition); \
|
||||
if (unlikely(__ret_warn_on)) { \
|
||||
__WARN_printf(format); \
|
||||
assert(0); \
|
||||
} \
|
||||
unlikely(__ret_warn_on); \
|
||||
})
|
||||
#define WARN_ON(condition) \
|
||||
WARN(condition, "assertion failed at %s:%d\n", __FILE__, __LINE__)
|
||||
#define WARN_ONCE(condition, format...) WARN(condition, format)
|
||||
#define WARN_ON_ONCE(condition) WARN_ON(condition)
|
||||
|
||||
/* Once state for the debug macros */
|
||||
#ifndef __ONCE_LITE_IF
|
||||
#define __ONCE_LITE_IF(condition) \
|
||||
({ \
|
||||
static bool __already_done; \
|
||||
bool __ret_once = false; \
|
||||
\
|
||||
if ((condition) && !__already_done) { \
|
||||
__already_done = true; \
|
||||
__ret_once = true; \
|
||||
} \
|
||||
__ret_once; \
|
||||
})
|
||||
#endif
|
||||
|
||||
#endif /* __MAPLE_SHARED_H__ */
|
||||
|
||||
@@ -148,6 +148,10 @@ static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void userfaultfd_unmap_abort(struct list_head *uf)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void mmap_write_downgrade(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user