Merge tag 'mm-core-7.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core

Pull core memory management fixes from David Hildenbrand:
 "The main changes are:

   - Prevent folio refcount overflows with pagecache folios, mostly
     relevant for large folios today. The networking changes for this
     will go through the net tree.

   - Fix out-of-bounds page table accesses with folded PMDs in 3
     scenarios.

   - Fix write-sealing of memfd to detect GUP pins with concurrent
     (un)mapping more reliably, preventing writes through GUP pins after
     write-sealing.

   - Fix lost wakeups when splitting large folios.

   - Fix exposing uninitialized data through MADV_COLLAPSE to user
     space.

  The refcount changes also include a related iommufd fix for properly
  cleaning up when pinning memfd folios fails, which is now easier to
  trigger"

* tag 'mm-core-7.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core:
  mm/gup: limit pagecache folio refcount when grabbing references
  iommufd: fix cleanup when adding memfd folio pins fails
  mm/mmu_gather: flush pending references early before the folio limit
  mm: limit pagecache folio refcount when splicing into pipes
  mm: limit pagecache folio refcount when mapping pages
  mm/huge_memory: prevent losing PG_waiters in __split_folio_to_order()
  mm/rmap: fix PTE table overruns during batched reference checks
  mm/rmap: fix PTE table overruns during batched unmap
  mm/userfaultfd: fix PTE table overruns in UFFDIO_MOVE
  mm/memfd: handle GUP pins more reliably when write-sealing
  mm/khugepaged: don't map folios that are not uptodate in MADV_COLLAPSE
This commit is contained in:
Linus Torvalds
2026-11-17 11:51:40 -08:00
19 changed files with 337 additions and 75 deletions
+11 -4
View File
@@ -781,6 +781,7 @@ struct pfn_reader_user {
struct file *file;
struct folio **ufolios;
size_t ufolios_len;
unsigned long ufolios_count;
unsigned long ufolios_offset;
struct folio **ufolios_next;
};
@@ -800,6 +801,7 @@ static void pfn_reader_user_init(struct pfn_reader_user *user,
user->file = (pages->type == IOPT_ADDRESS_FILE) ? pages->file : NULL;
user->ufolios = NULL;
user->ufolios_len = 0;
user->ufolios_count = 0;
user->ufolios_next = NULL;
user->ufolios_offset = 0;
}
@@ -843,6 +845,7 @@ static long pin_memfd_pages(struct pfn_reader_user *user, unsigned long start,
offset >>= PAGE_SHIFT;
user->ufolios_next = user->ufolios;
user->ufolios_count = nfolios;
user->ufolios_offset = offset;
for (i = 0; i < nfolios; i++) {
@@ -859,8 +862,12 @@ static long pin_memfd_pages(struct pfn_reader_user *user, unsigned long start,
} else {
int rc = folio_add_pins(folio, npin - 1);
if (rc)
if (rc) {
unpin_user_pages(user->upages,
upages - user->upages);
unpin_folios(user->ufolios + i, nfolios - i);
return rc;
}
while (npin--)
*upages++ = folio_page(folio, offset++);
@@ -1251,14 +1258,14 @@ static int pfn_reader_next(struct pfn_reader *pfns)
return -EINVAL;
rc = pfn_reader_fill_span(pfns);
pfns->batch_end_index = pfns->batch_start_index +
pfns->batch.total_pfns;
if (rc)
return rc;
if (WARN_ON(!pfns->batch.total_pfns))
return -EINVAL;
pfns->batch_end_index =
pfns->batch_start_index + pfns->batch.total_pfns;
if (pfns->batch_end_index == pfns->span.last_used + 1)
interval_tree_double_span_iter_next(&pfns->span);
@@ -1320,7 +1327,7 @@ static void pfn_reader_release_pins(struct pfn_reader *pfns)
if (!user->file) {
unpin_user_pages(user->upages + start_index, npages);
} else {
long n = user->ufolios_len / sizeof(*user->ufolios);
const unsigned long n = user->ufolios_count;
unpin_folios(user->ufolios_next,
user->ufolios + n - user->ufolios_next);
+3 -1
View File
@@ -1108,7 +1108,9 @@ static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio,
if (cs->nr_segs >= cs->pipe->max_usage)
return -EIO;
folio_get(folio);
if (!folio_get_checked(folio))
return -EFAULT;
err = unlock_request(cs->req);
if (err) {
folio_put(folio);
+2 -2
View File
@@ -75,8 +75,8 @@
* - __tlb_remove_folio_pages_size()
*
* __tlb_remove_folio_pages_size() is the basic primitive that queues pages
* for freeing. It will return a boolean indicating if the queue is (now)
* full and a call to tlb_flush_mmu() is required.
* for freeing. It returns true when a call to tlb_flush_mmu() is required,
* either because the queue is full or a folio is near its refcount limit.
*
* tlb_remove_page() and tlb_remove_page_size() imply the call to
* tlb_flush_mmu() when required and has no return value.
+76 -15
View File
@@ -2249,9 +2249,54 @@ vm_fault_t finish_fault(struct vm_fault *vmf);
* back into memory.
*/
/* 127: arbitrary random number, small enough to assemble well */
#define folio_ref_zero_or_close_to_overflow(folio) \
((unsigned int) folio_ref_count(folio) + 127u <= 127u)
/* Leave separate reserves for administrators and concurrent reference gains. */
#define FOLIO_REF_COUNT_LIMIT (INT_MAX - (1U << 26))
#define FOLIO_REF_COUNT_ADMIN_LIMIT (INT_MAX - (1U << 25))
#define FOLIO_REF_COUNT_FLUSH_LIMIT (INT_MAX - (1U << 27))
bool folio_ref_count_over_limit_slowpath(const struct folio *folio,
unsigned int ref_count);
/**
* __folio_ref_count_over_limit - Check whether to refuse more folio references.
* @folio: The folio.
* @ref_count: The sampled folio reference count.
*
* Check before adding page table mappings or other significant refcount users
* to pagecache folios (!hugetlb), either before acquiring references or
* afterwards with rollback if the operation is refused.
*
* Also required when pinning folios without a dedicated pincount that use
* GUP_PIN_COUNTING_BIAS.
*
* Checking other folios does no harm.
*
* This check is an approximation; we leave some headroom such that concurrent
* increments, batches, temporary references, or other bounded refcount
* consumers are not a problem.
*
* Return: True if the caller should refuse the operation, false otherwise.
*/
static inline bool __folio_ref_count_over_limit(const struct folio *folio,
unsigned int ref_count)
{
if (likely(ref_count < FOLIO_REF_COUNT_LIMIT))
return false;
return folio_ref_count_over_limit_slowpath(folio, ref_count);
}
/**
* folio_ref_count_over_limit - Check whether to refuse more folio references.
* @folio: The folio.
*
* See __folio_ref_count_over_limit().
*
* Return: True if the caller should refuse the operation, false otherwise.
*/
static inline bool folio_ref_count_over_limit(const struct folio *folio)
{
return __folio_ref_count_over_limit(folio, folio_ref_count(folio));
}
/**
* folio_get - Increment the reference count on a folio.
@@ -2263,7 +2308,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf);
*/
static inline void folio_get(struct folio *folio)
{
VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio);
VM_WARN_ON_FOLIO(folio_ref_count(folio) <= 0, folio);
folio_ref_inc(folio);
}
@@ -2277,13 +2322,33 @@ static inline void get_page(struct page *page)
folio_get(folio);
}
/**
* folio_get_checked - Try to acquire a reference to a live folio.
* @folio: The folio.
*
* The caller must already hold a reference to @folio. Warn and fail on a
* non-positive refcount, and refuse additional references at the limit.
*
* Use folio_try_get() for speculative reference acquisition where the
* refcount can legitimately be zero.
*
* Return: True if the refcount was incremented, false otherwise.
*/
static inline __must_check bool folio_get_checked(struct folio *folio)
{
const int ref_count = folio_ref_count(folio);
if (WARN_ON_ONCE(ref_count <= 0))
return false;
if (unlikely(__folio_ref_count_over_limit(folio, ref_count)))
return false;
folio_ref_inc(folio);
return true;
}
static inline __must_check bool try_get_page(struct page *page)
{
page = compound_head(page);
if (WARN_ON_ONCE(page_ref_count(page) <= 0))
return false;
page_ref_inc(page);
return true;
return folio_get_checked(page_folio(page));
}
/**
@@ -2925,12 +2990,8 @@ static inline bool folio_maybe_dma_pinned(struct folio *folio)
return atomic_read(&folio->_pincount) > 0;
/*
* folio_ref_count() is signed. If that refcount overflows, then
* folio_ref_count() returns a negative value, and callers will avoid
* further incrementing the refcount.
*
* Here, for that overflow case, use the sign bit to count a little
* bit higher via unsigned math, and thus still get an accurate result.
* Use unsigned math so even an unexpected negative refcount is
* treated as possibly pinned.
*/
return ((unsigned int)folio_ref_count(folio)) >=
GUP_PIN_COUNTING_BIAS;
+24
View File
@@ -12,6 +12,7 @@
#ifdef CONFIG_MMU
#include <linux/mm_types.h>
#include <linux/minmax.h>
#include <linux/bug.h>
#include <linux/errno.h>
#include <asm-generic/pgtable_uffd.h>
@@ -1518,6 +1519,29 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
})
#endif
/**
* pte_table_clamped_end() - Limit a range to a single PTE table.
* @addr: Start address.
* @end: Exclusive end address.
*
* Only use pmd_addr_end() when traversing page tables, in a pattern like::
*
* next = pmd_addr_end(addr, end);
*
* Higher levels already bound the range when PMDs are folded. Use this
* helper elsewhere to clamp the range regardless of PMD folding.
*
* Return: The exclusive end of the PTE table containing @addr, capped at @end.
*/
static inline unsigned long pte_table_clamped_end(unsigned long addr,
unsigned long end)
{
const unsigned long pmd_end = (addr & PMD_MASK) + PMD_SIZE;
/* Compare inclusive ends to avoid wraparounds. */
return min(pmd_end - 1, end - 1) + 1;
}
/*
* When walking page tables, we usually want to skip any p?d_none entries;
* and any p?d_bad entries - reporting the error before resetting to none.
+18 -10
View File
@@ -3047,10 +3047,11 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
EXPORT_SYMBOL(generic_file_read_iter);
/*
* Splice subpages from a folio into a pipe.
* Splice subpages from a folio into a pipe. Return the number of bytes spliced,
* or -EFAULT if we cannot splice from the folio.
*/
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct folio *folio, loff_t fpos, size_t size)
ssize_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct folio *folio, loff_t fpos, size_t size)
{
struct page *page;
size_t spliced = 0, offset = offset_in_folio(folio, fpos);
@@ -3063,13 +3064,15 @@ size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct pipe_buffer *buf = pipe_head_buf(pipe);
size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced);
if (!folio_get_checked(folio))
return spliced ? spliced : -EFAULT;
*buf = (struct pipe_buffer) {
.ops = &page_cache_pipe_buf_ops,
.page = page,
.offset = offset,
.len = part,
};
folio_get(folio);
pipe->head++;
page++;
spliced += part;
@@ -3094,9 +3097,10 @@ size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
* Return: On success, the number of bytes read will be returned and *@ppos
* will be updated if appropriate; 0 will be returned if there is no more data
* to be read; -EAGAIN will be returned if the pipe had no space, and some
* other negative error code will be returned on error. A short read may occur
* if the pipe has insufficient space, we reach the end of the data or we hit a
* hole.
* other negative error code will be returned on error; -EFAULT will be returned
* if acquiring a folio reference fails and no data has been spliced.
* A short read may occur if the pipe has insufficient space, we reach the
* end of the data, hit a hole or hit a folio refcount limit.
*/
ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
struct pipe_inode_info *pipe,
@@ -3154,7 +3158,7 @@ ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
size_t n;
ssize_t n;
if (folio_pos(folio) >= end_offset)
goto out;
@@ -3170,13 +3174,15 @@ ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
n = min_t(loff_t, len, isize - *ppos);
n = splice_folio_into_pipe(pipe, folio, *ppos, n);
if (!n)
if (n <= 0) {
error = n;
goto out;
}
len -= n;
total_spliced += n;
*ppos += n;
in->f_ra.prev_pos = *ppos;
if (pipe_is_full(pipe))
if (pipe_is_full(pipe) || *ppos < folio_next_pos(folio))
goto out;
}
@@ -3774,6 +3780,8 @@ static struct folio *next_uptodate_folio(struct xa_state *xas,
continue;
if (!folio_try_get(folio))
continue;
if (unlikely(folio_ref_count_over_limit(folio)))
goto skip;
if (folio_test_locked(folio))
goto skip;
/* Has the page moved or been split? */
+26
View File
@@ -107,6 +107,32 @@ void __folio_put(struct folio *folio)
}
EXPORT_SYMBOL(__folio_put);
bool folio_ref_count_over_limit_slowpath(const struct folio *folio,
unsigned int ref_count)
{
if (ref_count >= FOLIO_REF_COUNT_ADMIN_LIMIT)
goto over_limit;
/* Background threads must not consume admin reserves. */
if (current->flags & PF_KTHREAD)
goto over_limit;
/* Neither must interrupt contexts on behalf of the interrupted task. */
if (!in_task())
goto over_limit;
/* Capability checks are not safe with IRQs disabled. */
if (irqs_disabled())
goto over_limit;
/* Use the task's credentials, which filesystem overrides cannot change. */
if (has_capability_noaudit(current, CAP_SYS_RESOURCE))
return false;
over_limit:
pr_warn_ratelimited("folio:%p refcount:%d mapcount:%d mapping:%p index:%#lx reached refcount limit\n",
folio, (int)ref_count, folio_mapcount(folio),
folio_mapping(folio), folio->index);
return true;
}
EXPORT_SYMBOL_GPL(folio_ref_count_over_limit_slowpath);
typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
static void lru_add(struct lruvec *lruvec, struct folio *folio)
+21 -11
View File
@@ -74,10 +74,14 @@ static inline void sanity_check_pinned_pages(struct page **pages,
static inline struct folio *try_get_folio(struct page *page, int refs)
{
struct folio *folio;
int ref_count;
retry:
folio = page_folio(page);
if (WARN_ON_ONCE(folio_ref_count(folio) < 0))
ref_count = folio_ref_count(folio);
if (WARN_ON_ONCE(ref_count < 0))
return NULL;
if (unlikely(__folio_ref_count_over_limit(folio, ref_count)))
return NULL;
if (unlikely(!folio_ref_try_add(folio, refs)))
return NULL;
@@ -140,7 +144,13 @@ static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)
int __must_check try_grab_folio(struct folio *folio, int refs,
unsigned int flags)
{
if (WARN_ON_ONCE(folio_ref_count(folio) <= 0))
const int ref_count = folio_ref_count(folio);
if (WARN_ON_ONCE(ref_count <= 0))
return -ENOMEM;
if ((flags & (FOLL_GET | FOLL_PIN)) &&
unlikely(__folio_ref_count_over_limit(folio, ref_count)))
return -ENOMEM;
if (unlikely(!(flags & FOLL_PCI_P2PDMA) && folio_is_pci_p2pdma(folio)))
@@ -1482,17 +1492,17 @@ next_page:
struct folio *folio = page_folio(page);
/*
* Since we already hold refcount on the
* large folio, this should never fail.
* Grabbing more references can fail once we
* hit folio refcount limits.
*/
if (try_grab_folio(folio, page_increm - 1,
gup_flags)) {
ret = try_grab_folio(folio, page_increm - 1,
gup_flags);
if (ret) {
/*
* Release the 1st page ref if the
* folio is problematic, fail hard.
* Release the folio reference we
* already took.
*/
gup_put_folio(folio, 1, gup_flags);
ret = -EFAULT;
goto out;
}
}
@@ -3538,10 +3548,10 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end,
break;
}
if (try_grab_folio(folio, 1, FOLL_PIN)) {
ret = try_grab_folio(folio, 1, FOLL_PIN);
if (ret) {
folio_unlock(folio);
folio_batch_release(&fbatch);
ret = -EINVAL;
goto err;
}
+58 -6
View File
@@ -1626,6 +1626,7 @@ static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr,
{
struct mm_struct *mm = vma->vm_mm;
pgtable_t pgtable = NULL;
vm_fault_t ret = VM_FAULT_NOPAGE;
spinlock_t *ptl;
pmd_t entry;
@@ -1662,6 +1663,10 @@ static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr,
if (is_huge_zero_folio(fop.folio)) {
entry = pmd_mkspecial(entry);
} else {
if (unlikely(folio_ref_count_over_limit(fop.folio))) {
ret = VM_FAULT_SIGBUS;
goto out_unlock;
}
folio_get(fop.folio);
folio_add_file_rmap_pmd(fop.folio, &fop.folio->page, vma);
add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PMD_NR);
@@ -1688,7 +1693,7 @@ out_unlock:
spin_unlock(ptl);
if (pgtable)
pte_free(mm, pgtable);
return VM_FAULT_NOPAGE;
return ret;
}
/**
@@ -1759,6 +1764,7 @@ static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr,
pud_t *pud, struct folio_or_pfn fop, pgprot_t prot, bool write)
{
struct mm_struct *mm = vma->vm_mm;
vm_fault_t ret = VM_FAULT_NOPAGE;
spinlock_t *ptl;
pud_t entry;
@@ -1784,6 +1790,10 @@ static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr,
if (fop.is_folio) {
entry = folio_mk_pud(fop.folio, vma->vm_page_prot);
if (unlikely(folio_ref_count_over_limit(fop.folio))) {
ret = VM_FAULT_SIGBUS;
goto out_unlock;
}
folio_get(fop.folio);
folio_add_file_rmap_pud(fop.folio, &fop.folio->page, vma);
add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PUD_NR);
@@ -1799,7 +1809,7 @@ static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr,
update_mmu_cache_pud(vma, addr, pud);
out_unlock:
spin_unlock(ptl);
return VM_FAULT_NOPAGE;
return ret;
}
/**
@@ -2095,6 +2105,14 @@ int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
if (page) {
struct folio *folio = page_folio(page);
if (unlikely(folio_ref_count_over_limit(folio))) {
/*
* VM_MIXEDMAP may contain driver-inserted pages with no
* fault handler to reconstruct the mapping.
*/
ret = (dst_vma->vm_flags & VM_MIXEDMAP) ? -ENOMEM : 0;
goto out_unlock;
}
folio_get(folio);
/*
* TODO: once we support anonymous pages, use
@@ -3775,6 +3793,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
* Don't pass it around before clear_compound_head().
*/
struct folio *new_folio = (struct folio *)new_head;
memdesc_flags_t new_flags;
VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head);
@@ -3791,8 +3810,15 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
* unreferenced sub-pages of an anonymous THP: we can simply drop
* PG_anon_exclusive (-> PG_mappedtodisk) for these here.
*/
new_folio->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
new_folio->flags.f |= (folio->flags.f &
new_flags.f = new_folio->flags.f & ~PAGE_FLAGS_CHECK_AT_PREP;
/*
* data_race() is used because folio->flags is only read here,
* but a reference-less writer can still write to it, e.g.,
* PG_waiters update from softleaf_entry_wait_on_locked().
* The flag is not transferred to new_flags as it is
* folio specific, so it is not lost.
*/
new_flags.f |= data_race(folio->flags.f) &
((1L << PG_referenced) |
(1L << PG_swapbacked) |
(1L << PG_swapcache) |
@@ -3810,7 +3836,19 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
#endif
(1L << PG_dirty) |
(1L << PG_dropbehind) |
LRU_GEN_MASK | LRU_REFS_MASK));
LRU_GEN_MASK | LRU_REFS_MASK);
/*
* Set PG_head before smp_wmb() and clear_compound_head().
* After clear_compound_head() marks new_head as a folio of its
* own, new_head->flags must not be modified by plain RMWs.
* A plain RMW can cause a loss of atomic PG_* updates from
* a reference-less writer, e.g., setting PG_waiters via
* softleaf_entry_wait_on_locked().
*/
if (new_order)
new_flags.f |= 1L << PG_head;
WRITE_ONCE(new_folio->flags.f, new_flags.f);
new_folio->mapping = folio->mapping;
new_folio->index = folio->index + i;
@@ -3840,7 +3878,11 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
*/
clear_compound_head(new_head);
if (new_order) {
prep_compound_page(new_head, new_order);
/*
* PG_head is set above, so only set up the remaining
* metadata on head page and tail pages.
*/
__prep_compound_page(new_head, new_order);
folio_set_large_rmappable(new_folio);
}
@@ -5215,6 +5257,16 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct folio *folio
if (!(pvmw->pmd && !pvmw->pte))
return;
if (!folio_test_anon(folio) && unlikely(folio_ref_count_over_limit(folio))) {
/* Leave the file mapping absent so it can be faulted in. */
pmd_clear(pvmw->pmd);
if (arch_needs_pgtable_deposit())
zap_deposited_table(mm, pvmw->pmd);
update_hiwater_rss(mm);
add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR);
return;
}
entry = softleaf_from_pmd(*pvmw->pmd);
folio_get(folio);
pmde = folio_mk_pmd(folio, READ_ONCE(vma->vm_page_prot));
+4 -2
View File
@@ -834,6 +834,8 @@ static inline void prep_compound_tail(struct page *tail,
VM_WARN_ON_ONCE(tail->private);
}
void __prep_compound_page(struct page *page, unsigned int order);
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
/*
@@ -1272,8 +1274,8 @@ struct migration_target_control {
/*
* mm/filemap.c
*/
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct folio *folio, loff_t fpos, size_t size);
ssize_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct folio *folio, loff_t fpos, size_t size);
static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma)
{
+6
View File
@@ -1888,6 +1888,12 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign
goto drop_folio;
}
/* Leave any uninitialized folios to the page fault path. */
if (!folio_test_uptodate(folio)) {
result = SCAN_FAIL;
goto drop_folio;
}
result = find_pmd_or_thp_or_none(mm, haddr, &pmd);
switch (result) {
case SCAN_SUCCEED:
+7
View File
@@ -32,6 +32,13 @@
static bool memfd_folio_has_extra_refs(struct folio *folio)
{
/*
* Use the precise pincount where available to avoid racy refcount
* comparisons.
*/
if (folio_has_pincount(folio) && folio_maybe_dma_pinned(folio))
return true;
return folio_ref_count(folio) != folio_expected_ref_count(folio);
}
+36 -12
View File
@@ -1168,7 +1168,10 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
* consecutive pages of the same folio by copying them as well.
*
* Returns -EAGAIN if one preallocated page is required to copy the next PTE.
* Otherwise, returns the number of copied PTEs (at least 1).
* Returns -ENOMEM if the file folio refcount is too high and the PTE must
* be copied.
* Returns -EHWPOISON if copying a page fails.
* Otherwise, returns the number of processed PTEs (at least 1).
*/
static inline int
copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
@@ -1186,6 +1189,20 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
folio = page_folio(page);
if (!folio_test_anon(folio) && unlikely(folio_ref_count_over_limit(folio))) {
/*
* VM_MIXEDMAP may contain pages inserted by a driver at mmap
* time, with no fault handler to restore a missing PTE.
*/
if (dst_vma->vm_flags & VM_MIXEDMAP)
return -ENOMEM;
if (userfaultfd_pte_rwp(dst_vma, pte))
return -ENOMEM;
/* Preserve uffd-wp protection when faulting the page back in. */
cond_install_uffd_wp_ptes(dst_vma, addr, dst_pte, pte, 1);
return 1;
}
/*
* If we likely have to copy, just don't bother with batching. Make
* sure that the common "small folio" case is as fast as possible
@@ -1367,9 +1384,9 @@ again:
/*
* If we need a pre-allocated page for this pte, drop the
* locks, allocate, and try again.
* If copy failed due to hwpoison in source page, break out.
* Other errors abort the copy after releasing the page table locks.
*/
if (unlikely(ret == -EAGAIN || ret == -EHWPOISON))
if (unlikely(ret < 0))
break;
if (unlikely(prealloc)) {
/*
@@ -1399,7 +1416,7 @@ again:
goto out;
}
entry.val = 0;
} else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) {
} else if (ret == -EBUSY || ret == -ENOMEM || unlikely(ret == -EHWPOISON)) {
goto out;
} else if (ret == -EAGAIN) {
prealloc = folio_prealloc(src_mm, src_vma, addr, false);
@@ -2059,8 +2076,8 @@ retry:
pte_unmap_unlock(start_pte, ptl);
/*
* If we forced a TLB flush (either due to running out of
* batch buffers or because we needed to flush dirty TLB
* If we forced a TLB flush (either to release batched
* references or because we needed to flush dirty TLB
* entries before releasing the ptl), free the batched
* memory too. Come back again if we didn't do everything.
*/
@@ -2474,6 +2491,8 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
if (unlikely(is_zero_folio(folio))) {
pteval = pte_mkspecial(pteval);
} else {
if (unlikely(folio_ref_count_over_limit(folio)))
return -EOVERFLOW;
folio_get(folio);
pteval = mk_pte(page, prot);
if (mkwrite) {
@@ -4959,13 +4978,8 @@ static void folio_pte_range(const struct folio *folio,
{
/* Stay within the page table and VMA boundaries. */
const unsigned long pmd_start = addr & PMD_MASK;
const unsigned long pmd_end = pmd_start + PMD_SIZE;
const unsigned long lower = max(pmd_start, vma->vm_start);
/*
* We cannot use pmd_addr_end(), as it does not clamp reliably. To
* avoid wraparounds, compute using the inclusive end.
*/
const unsigned long upper = min(pmd_end - 1, vma->vm_end - 1) + 1;
const unsigned long upper = pte_table_clamped_end(addr, vma->vm_end);
const unsigned long offset = idx << PAGE_SHIFT;
/*
@@ -5752,6 +5766,9 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa
if (unlikely(folio_test_has_hwpoisoned(folio)))
return ret;
if (unlikely(folio_ref_count_over_limit(folio)))
return VM_FAULT_SIGBUS;
/*
* Archs like ppc64 need additional space to store information
* related to pte entry. Use the preallocated table for that.
@@ -5806,6 +5823,9 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa
* @page: The first page to create a PTE for.
* @nr: The number of PTEs to create.
* @addr: The first address to create a PTE for.
*
* For file mappings, the caller must check folio_ref_count_over_limit()
* before installing PTEs.
*/
void set_pte_range(struct vm_fault *vmf, struct folio *folio,
struct page *page, unsigned int nr, unsigned long addr)
@@ -5972,6 +5992,10 @@ fallback:
goto fallback;
}
if (!is_cow && unlikely(folio_ref_count_over_limit(folio))) {
ret = VM_FAULT_SIGBUS;
goto unlock;
}
folio_ref_add(folio, nr_pages - 1);
set_pte_range(vmf, folio, page, nr_pages, addr);
type = is_cow ? MM_ANONPAGES : mm_counter_file(folio);
+11
View File
@@ -385,6 +385,17 @@ static bool remove_migration_pte(struct folio *folio,
try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx))
continue;
if (!folio_test_anon(folio) && !folio_test_hugetlb(folio) &&
unlikely(folio_ref_count_over_limit(folio))) {
/* File contents remain available for a later fault. */
pte_clear(vma->vm_mm, pvmw.address, pvmw.pte);
cond_install_uffd_wp_ptes(vma, pvmw.address,
pvmw.pte, old_pte, 1);
update_hiwater_rss(vma->vm_mm);
dec_mm_counter(vma->vm_mm, mm_counter_file(folio));
continue;
}
folio_get(folio);
new = folio_page(folio, idx);
pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
+6 -4
View File
@@ -179,10 +179,7 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb,
#endif
batch = tlb->active;
/*
* Add the page and check if we are full. If so
* force a flush.
*/
/* Add the pages before deciding whether we need to flush. */
if (likely(nr_pages == 1)) {
batch->encoded_pages[batch->nr++] = encode_page(page, flags);
} else {
@@ -190,6 +187,11 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb,
batch->encoded_pages[batch->nr++] = encode_page(page, flags);
batch->encoded_pages[batch->nr++] = encode_nr_pages(nr_pages);
}
/* Flush pending references well before new references are refused. */
if (unlikely((unsigned int)folio_ref_count(page_folio(page)) >=
FOLIO_REF_COUNT_FLUSH_LIMIT))
return true;
/*
* Make sure that we can always add another "page" + "nr_pages",
* requiring two entries instead of only a single one.
+15 -2
View File
@@ -681,18 +681,31 @@ static inline bool pcp_allowed_order(unsigned int order)
* This usage means that zero-order pages may not be compound.
*/
void prep_compound_page(struct page *page, unsigned int order)
/*
* Set up the metadata on head page and tail pages.
* Caller must set PG_head before this.
*/
void __prep_compound_page(struct page *page, unsigned int order)
{
int i;
int nr_pages = 1 << order;
__SetPageHead(page);
for (i = 1; i < nr_pages; i++)
prep_compound_tail(page + i, page, order);
prep_compound_head(page, order);
}
/*
* PG_head is set non-atomically. It should be used when no one else can modify
* @page's flags.
*/
void prep_compound_page(struct page *page, unsigned int order)
{
__SetPageHead(page);
__prep_compound_page(page, order);
}
static inline void set_buddy_order(struct page *page, unsigned int order)
{
set_page_private(page, order);
+2 -2
View File
@@ -973,7 +973,7 @@ static bool folio_referenced_one(struct folio *folio,
pteval = ptep_get(pvmw.pte);
if (pvmw.pte && pte_present(pteval) && folio_test_large(folio)) {
const unsigned long end_addr = pmd_addr_end(address, vma->vm_end);
const unsigned long end_addr = pte_table_clamped_end(address, vma->vm_end);
const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT;
nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr);
@@ -1978,7 +1978,7 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
return 1;
/* We may only batch within a single VMA and a single page table. */
end_addr = pmd_addr_end(addr, vma->vm_end);
end_addr = pte_table_clamped_end(addr, vma->vm_end);
max_nr = (end_addr - addr) >> PAGE_SHIFT;
/* We only support lazyfree or file folios batching for now ... */
+5 -2
View File
@@ -3737,7 +3737,8 @@ static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
struct inode *inode = file_inode(in);
struct address_space *mapping = inode->i_mapping;
struct folio *folio = NULL;
size_t total_spliced = 0, used, npages, n, part;
size_t total_spliced = 0, used, npages, part;
ssize_t n;
loff_t isize;
int error = 0;
@@ -3823,8 +3824,10 @@ static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
n = splice_zeropage_into_pipe(pipe, *ppos, part);
}
if (!n)
if (n <= 0) {
error = n;
break;
}
len -= n;
total_spliced += n;
*ppos += n;
+6 -2
View File
@@ -373,6 +373,10 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd,
goto out_unlock;
if (page_in_cache) {
if (unlikely(folio_ref_count_over_limit(folio))) {
ret = -ENOMEM;
goto out_unlock;
}
folio_add_file_rmap_pte(folio, page, dst_vma);
} else {
folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE);
@@ -1266,8 +1270,8 @@ static long move_present_ptes(struct mm_struct *mm,
unsigned long src_start = src_addr;
unsigned long src_end;
len = pmd_addr_end(dst_addr, dst_addr + len) - dst_addr;
src_end = pmd_addr_end(src_addr, src_addr + len);
len = pte_table_clamped_end(dst_addr, dst_addr + len) - dst_addr;
src_end = pte_table_clamped_end(src_addr, src_addr + len);
flush_cache_range(src_vma, src_addr, src_end);
double_pt_lock(dst_ptl, src_ptl);