mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
Merge tag 'mm-core-7.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core
Pull core memory management fixes from David Hildenbrand:
"The main changes are:
- Prevent folio refcount overflows with pagecache folios, mostly
relevant for large folios today. The networking changes for this
will go through the net tree.
- Fix out-of-bounds page table accesses with folded PMDs in 3
scenarios.
- Fix write-sealing of memfd to detect GUP pins with concurrent
(un)mapping more reliably, preventing writes through GUP pins after
write-sealing.
- Fix lost wakeups when splitting large folios.
- Fix exposing uninitialized data through MADV_COLLAPSE to user
space.
The refcount changes also include a related iommufd fix for properly
cleaning up when pinning memfd folios fails, which is now easier to
trigger"
* tag 'mm-core-7.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core:
mm/gup: limit pagecache folio refcount when grabbing references
iommufd: fix cleanup when adding memfd folio pins fails
mm/mmu_gather: flush pending references early before the folio limit
mm: limit pagecache folio refcount when splicing into pipes
mm: limit pagecache folio refcount when mapping pages
mm/huge_memory: prevent losing PG_waiters in __split_folio_to_order()
mm/rmap: fix PTE table overruns during batched reference checks
mm/rmap: fix PTE table overruns during batched unmap
mm/userfaultfd: fix PTE table overruns in UFFDIO_MOVE
mm/memfd: handle GUP pins more reliably when write-sealing
mm/khugepaged: don't map folios that are not uptodate in MADV_COLLAPSE
This commit is contained in:
@@ -781,6 +781,7 @@ struct pfn_reader_user {
|
||||
struct file *file;
|
||||
struct folio **ufolios;
|
||||
size_t ufolios_len;
|
||||
unsigned long ufolios_count;
|
||||
unsigned long ufolios_offset;
|
||||
struct folio **ufolios_next;
|
||||
};
|
||||
@@ -800,6 +801,7 @@ static void pfn_reader_user_init(struct pfn_reader_user *user,
|
||||
user->file = (pages->type == IOPT_ADDRESS_FILE) ? pages->file : NULL;
|
||||
user->ufolios = NULL;
|
||||
user->ufolios_len = 0;
|
||||
user->ufolios_count = 0;
|
||||
user->ufolios_next = NULL;
|
||||
user->ufolios_offset = 0;
|
||||
}
|
||||
@@ -843,6 +845,7 @@ static long pin_memfd_pages(struct pfn_reader_user *user, unsigned long start,
|
||||
|
||||
offset >>= PAGE_SHIFT;
|
||||
user->ufolios_next = user->ufolios;
|
||||
user->ufolios_count = nfolios;
|
||||
user->ufolios_offset = offset;
|
||||
|
||||
for (i = 0; i < nfolios; i++) {
|
||||
@@ -859,8 +862,12 @@ static long pin_memfd_pages(struct pfn_reader_user *user, unsigned long start,
|
||||
} else {
|
||||
int rc = folio_add_pins(folio, npin - 1);
|
||||
|
||||
if (rc)
|
||||
if (rc) {
|
||||
unpin_user_pages(user->upages,
|
||||
upages - user->upages);
|
||||
unpin_folios(user->ufolios + i, nfolios - i);
|
||||
return rc;
|
||||
}
|
||||
|
||||
while (npin--)
|
||||
*upages++ = folio_page(folio, offset++);
|
||||
@@ -1251,14 +1258,14 @@ static int pfn_reader_next(struct pfn_reader *pfns)
|
||||
return -EINVAL;
|
||||
|
||||
rc = pfn_reader_fill_span(pfns);
|
||||
pfns->batch_end_index = pfns->batch_start_index +
|
||||
pfns->batch.total_pfns;
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
if (WARN_ON(!pfns->batch.total_pfns))
|
||||
return -EINVAL;
|
||||
|
||||
pfns->batch_end_index =
|
||||
pfns->batch_start_index + pfns->batch.total_pfns;
|
||||
if (pfns->batch_end_index == pfns->span.last_used + 1)
|
||||
interval_tree_double_span_iter_next(&pfns->span);
|
||||
|
||||
@@ -1320,7 +1327,7 @@ static void pfn_reader_release_pins(struct pfn_reader *pfns)
|
||||
if (!user->file) {
|
||||
unpin_user_pages(user->upages + start_index, npages);
|
||||
} else {
|
||||
long n = user->ufolios_len / sizeof(*user->ufolios);
|
||||
const unsigned long n = user->ufolios_count;
|
||||
|
||||
unpin_folios(user->ufolios_next,
|
||||
user->ufolios + n - user->ufolios_next);
|
||||
|
||||
+3
-1
@@ -1108,7 +1108,9 @@ static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio,
|
||||
if (cs->nr_segs >= cs->pipe->max_usage)
|
||||
return -EIO;
|
||||
|
||||
folio_get(folio);
|
||||
if (!folio_get_checked(folio))
|
||||
return -EFAULT;
|
||||
|
||||
err = unlock_request(cs->req);
|
||||
if (err) {
|
||||
folio_put(folio);
|
||||
|
||||
@@ -75,8 +75,8 @@
|
||||
* - __tlb_remove_folio_pages_size()
|
||||
*
|
||||
* __tlb_remove_folio_pages_size() is the basic primitive that queues pages
|
||||
* for freeing. It will return a boolean indicating if the queue is (now)
|
||||
* full and a call to tlb_flush_mmu() is required.
|
||||
* for freeing. It returns true when a call to tlb_flush_mmu() is required,
|
||||
* either because the queue is full or a folio is near its refcount limit.
|
||||
*
|
||||
* tlb_remove_page() and tlb_remove_page_size() imply the call to
|
||||
* tlb_flush_mmu() when required and has no return value.
|
||||
|
||||
+76
-15
@@ -2249,9 +2249,54 @@ vm_fault_t finish_fault(struct vm_fault *vmf);
|
||||
* back into memory.
|
||||
*/
|
||||
|
||||
/* 127: arbitrary random number, small enough to assemble well */
|
||||
#define folio_ref_zero_or_close_to_overflow(folio) \
|
||||
((unsigned int) folio_ref_count(folio) + 127u <= 127u)
|
||||
/* Leave separate reserves for administrators and concurrent reference gains. */
|
||||
#define FOLIO_REF_COUNT_LIMIT (INT_MAX - (1U << 26))
|
||||
#define FOLIO_REF_COUNT_ADMIN_LIMIT (INT_MAX - (1U << 25))
|
||||
#define FOLIO_REF_COUNT_FLUSH_LIMIT (INT_MAX - (1U << 27))
|
||||
|
||||
bool folio_ref_count_over_limit_slowpath(const struct folio *folio,
|
||||
unsigned int ref_count);
|
||||
|
||||
/**
|
||||
* __folio_ref_count_over_limit - Check whether to refuse more folio references.
|
||||
* @folio: The folio.
|
||||
* @ref_count: The sampled folio reference count.
|
||||
*
|
||||
* Check before adding page table mappings or other significant refcount users
|
||||
* to pagecache folios (!hugetlb), either before acquiring references or
|
||||
* afterwards with rollback if the operation is refused.
|
||||
*
|
||||
* Also required when pinning folios without a dedicated pincount that use
|
||||
* GUP_PIN_COUNTING_BIAS.
|
||||
*
|
||||
* Checking other folios does no harm.
|
||||
*
|
||||
* This check is an approximation; we leave some headroom such that concurrent
|
||||
* increments, batches, temporary references, or other bounded refcount
|
||||
* consumers are not a problem.
|
||||
*
|
||||
* Return: True if the caller should refuse the operation, false otherwise.
|
||||
*/
|
||||
static inline bool __folio_ref_count_over_limit(const struct folio *folio,
|
||||
unsigned int ref_count)
|
||||
{
|
||||
if (likely(ref_count < FOLIO_REF_COUNT_LIMIT))
|
||||
return false;
|
||||
return folio_ref_count_over_limit_slowpath(folio, ref_count);
|
||||
}
|
||||
|
||||
/**
|
||||
* folio_ref_count_over_limit - Check whether to refuse more folio references.
|
||||
* @folio: The folio.
|
||||
*
|
||||
* See __folio_ref_count_over_limit().
|
||||
*
|
||||
* Return: True if the caller should refuse the operation, false otherwise.
|
||||
*/
|
||||
static inline bool folio_ref_count_over_limit(const struct folio *folio)
|
||||
{
|
||||
return __folio_ref_count_over_limit(folio, folio_ref_count(folio));
|
||||
}
|
||||
|
||||
/**
|
||||
* folio_get - Increment the reference count on a folio.
|
||||
@@ -2263,7 +2308,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf);
|
||||
*/
|
||||
static inline void folio_get(struct folio *folio)
|
||||
{
|
||||
VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio);
|
||||
VM_WARN_ON_FOLIO(folio_ref_count(folio) <= 0, folio);
|
||||
folio_ref_inc(folio);
|
||||
}
|
||||
|
||||
@@ -2277,13 +2322,33 @@ static inline void get_page(struct page *page)
|
||||
folio_get(folio);
|
||||
}
|
||||
|
||||
/**
|
||||
* folio_get_checked - Try to acquire a reference to a live folio.
|
||||
* @folio: The folio.
|
||||
*
|
||||
* The caller must already hold a reference to @folio. Warn and fail on a
|
||||
* non-positive refcount, and refuse additional references at the limit.
|
||||
*
|
||||
* Use folio_try_get() for speculative reference acquisition where the
|
||||
* refcount can legitimately be zero.
|
||||
*
|
||||
* Return: True if the refcount was incremented, false otherwise.
|
||||
*/
|
||||
static inline __must_check bool folio_get_checked(struct folio *folio)
|
||||
{
|
||||
const int ref_count = folio_ref_count(folio);
|
||||
|
||||
if (WARN_ON_ONCE(ref_count <= 0))
|
||||
return false;
|
||||
if (unlikely(__folio_ref_count_over_limit(folio, ref_count)))
|
||||
return false;
|
||||
folio_ref_inc(folio);
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline __must_check bool try_get_page(struct page *page)
|
||||
{
|
||||
page = compound_head(page);
|
||||
if (WARN_ON_ONCE(page_ref_count(page) <= 0))
|
||||
return false;
|
||||
page_ref_inc(page);
|
||||
return true;
|
||||
return folio_get_checked(page_folio(page));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2925,12 +2990,8 @@ static inline bool folio_maybe_dma_pinned(struct folio *folio)
|
||||
return atomic_read(&folio->_pincount) > 0;
|
||||
|
||||
/*
|
||||
* folio_ref_count() is signed. If that refcount overflows, then
|
||||
* folio_ref_count() returns a negative value, and callers will avoid
|
||||
* further incrementing the refcount.
|
||||
*
|
||||
* Here, for that overflow case, use the sign bit to count a little
|
||||
* bit higher via unsigned math, and thus still get an accurate result.
|
||||
* Use unsigned math so even an unexpected negative refcount is
|
||||
* treated as possibly pinned.
|
||||
*/
|
||||
return ((unsigned int)folio_ref_count(folio)) >=
|
||||
GUP_PIN_COUNTING_BIAS;
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#ifdef CONFIG_MMU
|
||||
|
||||
#include <linux/mm_types.h>
|
||||
#include <linux/minmax.h>
|
||||
#include <linux/bug.h>
|
||||
#include <linux/errno.h>
|
||||
#include <asm-generic/pgtable_uffd.h>
|
||||
@@ -1518,6 +1519,29 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
|
||||
})
|
||||
#endif
|
||||
|
||||
/**
|
||||
* pte_table_clamped_end() - Limit a range to a single PTE table.
|
||||
* @addr: Start address.
|
||||
* @end: Exclusive end address.
|
||||
*
|
||||
* Only use pmd_addr_end() when traversing page tables, in a pattern like::
|
||||
*
|
||||
* next = pmd_addr_end(addr, end);
|
||||
*
|
||||
* Higher levels already bound the range when PMDs are folded. Use this
|
||||
* helper elsewhere to clamp the range regardless of PMD folding.
|
||||
*
|
||||
* Return: The exclusive end of the PTE table containing @addr, capped at @end.
|
||||
*/
|
||||
static inline unsigned long pte_table_clamped_end(unsigned long addr,
|
||||
unsigned long end)
|
||||
{
|
||||
const unsigned long pmd_end = (addr & PMD_MASK) + PMD_SIZE;
|
||||
|
||||
/* Compare inclusive ends to avoid wraparounds. */
|
||||
return min(pmd_end - 1, end - 1) + 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* When walking page tables, we usually want to skip any p?d_none entries;
|
||||
* and any p?d_bad entries - reporting the error before resetting to none.
|
||||
|
||||
+18
-10
@@ -3047,10 +3047,11 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
|
||||
EXPORT_SYMBOL(generic_file_read_iter);
|
||||
|
||||
/*
|
||||
* Splice subpages from a folio into a pipe.
|
||||
* Splice subpages from a folio into a pipe. Return the number of bytes spliced,
|
||||
* or -EFAULT if we cannot splice from the folio.
|
||||
*/
|
||||
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
|
||||
struct folio *folio, loff_t fpos, size_t size)
|
||||
ssize_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
|
||||
struct folio *folio, loff_t fpos, size_t size)
|
||||
{
|
||||
struct page *page;
|
||||
size_t spliced = 0, offset = offset_in_folio(folio, fpos);
|
||||
@@ -3063,13 +3064,15 @@ size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
|
||||
struct pipe_buffer *buf = pipe_head_buf(pipe);
|
||||
size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced);
|
||||
|
||||
if (!folio_get_checked(folio))
|
||||
return spliced ? spliced : -EFAULT;
|
||||
|
||||
*buf = (struct pipe_buffer) {
|
||||
.ops = &page_cache_pipe_buf_ops,
|
||||
.page = page,
|
||||
.offset = offset,
|
||||
.len = part,
|
||||
};
|
||||
folio_get(folio);
|
||||
pipe->head++;
|
||||
page++;
|
||||
spliced += part;
|
||||
@@ -3094,9 +3097,10 @@ size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
|
||||
* Return: On success, the number of bytes read will be returned and *@ppos
|
||||
* will be updated if appropriate; 0 will be returned if there is no more data
|
||||
* to be read; -EAGAIN will be returned if the pipe had no space, and some
|
||||
* other negative error code will be returned on error. A short read may occur
|
||||
* if the pipe has insufficient space, we reach the end of the data or we hit a
|
||||
* hole.
|
||||
* other negative error code will be returned on error; -EFAULT will be returned
|
||||
* if acquiring a folio reference fails and no data has been spliced.
|
||||
* A short read may occur if the pipe has insufficient space, we reach the
|
||||
* end of the data, hit a hole or hit a folio refcount limit.
|
||||
*/
|
||||
ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
|
||||
struct pipe_inode_info *pipe,
|
||||
@@ -3154,7 +3158,7 @@ ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
|
||||
|
||||
for (i = 0; i < folio_batch_count(&fbatch); i++) {
|
||||
struct folio *folio = fbatch.folios[i];
|
||||
size_t n;
|
||||
ssize_t n;
|
||||
|
||||
if (folio_pos(folio) >= end_offset)
|
||||
goto out;
|
||||
@@ -3170,13 +3174,15 @@ ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
|
||||
|
||||
n = min_t(loff_t, len, isize - *ppos);
|
||||
n = splice_folio_into_pipe(pipe, folio, *ppos, n);
|
||||
if (!n)
|
||||
if (n <= 0) {
|
||||
error = n;
|
||||
goto out;
|
||||
}
|
||||
len -= n;
|
||||
total_spliced += n;
|
||||
*ppos += n;
|
||||
in->f_ra.prev_pos = *ppos;
|
||||
if (pipe_is_full(pipe))
|
||||
if (pipe_is_full(pipe) || *ppos < folio_next_pos(folio))
|
||||
goto out;
|
||||
}
|
||||
|
||||
@@ -3774,6 +3780,8 @@ static struct folio *next_uptodate_folio(struct xa_state *xas,
|
||||
continue;
|
||||
if (!folio_try_get(folio))
|
||||
continue;
|
||||
if (unlikely(folio_ref_count_over_limit(folio)))
|
||||
goto skip;
|
||||
if (folio_test_locked(folio))
|
||||
goto skip;
|
||||
/* Has the page moved or been split? */
|
||||
|
||||
+26
@@ -107,6 +107,32 @@ void __folio_put(struct folio *folio)
|
||||
}
|
||||
EXPORT_SYMBOL(__folio_put);
|
||||
|
||||
bool folio_ref_count_over_limit_slowpath(const struct folio *folio,
|
||||
unsigned int ref_count)
|
||||
{
|
||||
if (ref_count >= FOLIO_REF_COUNT_ADMIN_LIMIT)
|
||||
goto over_limit;
|
||||
/* Background threads must not consume admin reserves. */
|
||||
if (current->flags & PF_KTHREAD)
|
||||
goto over_limit;
|
||||
/* Neither must interrupt contexts on behalf of the interrupted task. */
|
||||
if (!in_task())
|
||||
goto over_limit;
|
||||
/* Capability checks are not safe with IRQs disabled. */
|
||||
if (irqs_disabled())
|
||||
goto over_limit;
|
||||
/* Use the task's credentials, which filesystem overrides cannot change. */
|
||||
if (has_capability_noaudit(current, CAP_SYS_RESOURCE))
|
||||
return false;
|
||||
|
||||
over_limit:
|
||||
pr_warn_ratelimited("folio:%p refcount:%d mapcount:%d mapping:%p index:%#lx reached refcount limit\n",
|
||||
folio, (int)ref_count, folio_mapcount(folio),
|
||||
folio_mapping(folio), folio->index);
|
||||
return true;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(folio_ref_count_over_limit_slowpath);
|
||||
|
||||
typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
|
||||
|
||||
static void lru_add(struct lruvec *lruvec, struct folio *folio)
|
||||
|
||||
@@ -74,10 +74,14 @@ static inline void sanity_check_pinned_pages(struct page **pages,
|
||||
static inline struct folio *try_get_folio(struct page *page, int refs)
|
||||
{
|
||||
struct folio *folio;
|
||||
int ref_count;
|
||||
|
||||
retry:
|
||||
folio = page_folio(page);
|
||||
if (WARN_ON_ONCE(folio_ref_count(folio) < 0))
|
||||
ref_count = folio_ref_count(folio);
|
||||
if (WARN_ON_ONCE(ref_count < 0))
|
||||
return NULL;
|
||||
if (unlikely(__folio_ref_count_over_limit(folio, ref_count)))
|
||||
return NULL;
|
||||
if (unlikely(!folio_ref_try_add(folio, refs)))
|
||||
return NULL;
|
||||
@@ -140,7 +144,13 @@ static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)
|
||||
int __must_check try_grab_folio(struct folio *folio, int refs,
|
||||
unsigned int flags)
|
||||
{
|
||||
if (WARN_ON_ONCE(folio_ref_count(folio) <= 0))
|
||||
const int ref_count = folio_ref_count(folio);
|
||||
|
||||
if (WARN_ON_ONCE(ref_count <= 0))
|
||||
return -ENOMEM;
|
||||
|
||||
if ((flags & (FOLL_GET | FOLL_PIN)) &&
|
||||
unlikely(__folio_ref_count_over_limit(folio, ref_count)))
|
||||
return -ENOMEM;
|
||||
|
||||
if (unlikely(!(flags & FOLL_PCI_P2PDMA) && folio_is_pci_p2pdma(folio)))
|
||||
@@ -1482,17 +1492,17 @@ next_page:
|
||||
struct folio *folio = page_folio(page);
|
||||
|
||||
/*
|
||||
* Since we already hold refcount on the
|
||||
* large folio, this should never fail.
|
||||
* Grabbing more references can fail once we
|
||||
* hit folio refcount limits.
|
||||
*/
|
||||
if (try_grab_folio(folio, page_increm - 1,
|
||||
gup_flags)) {
|
||||
ret = try_grab_folio(folio, page_increm - 1,
|
||||
gup_flags);
|
||||
if (ret) {
|
||||
/*
|
||||
* Release the 1st page ref if the
|
||||
* folio is problematic, fail hard.
|
||||
* Release the folio reference we
|
||||
* already took.
|
||||
*/
|
||||
gup_put_folio(folio, 1, gup_flags);
|
||||
ret = -EFAULT;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
@@ -3538,10 +3548,10 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end,
|
||||
break;
|
||||
}
|
||||
|
||||
if (try_grab_folio(folio, 1, FOLL_PIN)) {
|
||||
ret = try_grab_folio(folio, 1, FOLL_PIN);
|
||||
if (ret) {
|
||||
folio_unlock(folio);
|
||||
folio_batch_release(&fbatch);
|
||||
ret = -EINVAL;
|
||||
goto err;
|
||||
}
|
||||
|
||||
|
||||
+58
-6
@@ -1626,6 +1626,7 @@ static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr,
|
||||
{
|
||||
struct mm_struct *mm = vma->vm_mm;
|
||||
pgtable_t pgtable = NULL;
|
||||
vm_fault_t ret = VM_FAULT_NOPAGE;
|
||||
spinlock_t *ptl;
|
||||
pmd_t entry;
|
||||
|
||||
@@ -1662,6 +1663,10 @@ static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr,
|
||||
if (is_huge_zero_folio(fop.folio)) {
|
||||
entry = pmd_mkspecial(entry);
|
||||
} else {
|
||||
if (unlikely(folio_ref_count_over_limit(fop.folio))) {
|
||||
ret = VM_FAULT_SIGBUS;
|
||||
goto out_unlock;
|
||||
}
|
||||
folio_get(fop.folio);
|
||||
folio_add_file_rmap_pmd(fop.folio, &fop.folio->page, vma);
|
||||
add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PMD_NR);
|
||||
@@ -1688,7 +1693,7 @@ out_unlock:
|
||||
spin_unlock(ptl);
|
||||
if (pgtable)
|
||||
pte_free(mm, pgtable);
|
||||
return VM_FAULT_NOPAGE;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1759,6 +1764,7 @@ static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr,
|
||||
pud_t *pud, struct folio_or_pfn fop, pgprot_t prot, bool write)
|
||||
{
|
||||
struct mm_struct *mm = vma->vm_mm;
|
||||
vm_fault_t ret = VM_FAULT_NOPAGE;
|
||||
spinlock_t *ptl;
|
||||
pud_t entry;
|
||||
|
||||
@@ -1784,6 +1790,10 @@ static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr,
|
||||
if (fop.is_folio) {
|
||||
entry = folio_mk_pud(fop.folio, vma->vm_page_prot);
|
||||
|
||||
if (unlikely(folio_ref_count_over_limit(fop.folio))) {
|
||||
ret = VM_FAULT_SIGBUS;
|
||||
goto out_unlock;
|
||||
}
|
||||
folio_get(fop.folio);
|
||||
folio_add_file_rmap_pud(fop.folio, &fop.folio->page, vma);
|
||||
add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PUD_NR);
|
||||
@@ -1799,7 +1809,7 @@ static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr,
|
||||
update_mmu_cache_pud(vma, addr, pud);
|
||||
out_unlock:
|
||||
spin_unlock(ptl);
|
||||
return VM_FAULT_NOPAGE;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2095,6 +2105,14 @@ int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
|
||||
if (page) {
|
||||
struct folio *folio = page_folio(page);
|
||||
|
||||
if (unlikely(folio_ref_count_over_limit(folio))) {
|
||||
/*
|
||||
* VM_MIXEDMAP may contain driver-inserted pages with no
|
||||
* fault handler to reconstruct the mapping.
|
||||
*/
|
||||
ret = (dst_vma->vm_flags & VM_MIXEDMAP) ? -ENOMEM : 0;
|
||||
goto out_unlock;
|
||||
}
|
||||
folio_get(folio);
|
||||
/*
|
||||
* TODO: once we support anonymous pages, use
|
||||
@@ -3775,6 +3793,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
|
||||
* Don't pass it around before clear_compound_head().
|
||||
*/
|
||||
struct folio *new_folio = (struct folio *)new_head;
|
||||
memdesc_flags_t new_flags;
|
||||
|
||||
VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head);
|
||||
|
||||
@@ -3791,8 +3810,15 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
|
||||
* unreferenced sub-pages of an anonymous THP: we can simply drop
|
||||
* PG_anon_exclusive (-> PG_mappedtodisk) for these here.
|
||||
*/
|
||||
new_folio->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
|
||||
new_folio->flags.f |= (folio->flags.f &
|
||||
new_flags.f = new_folio->flags.f & ~PAGE_FLAGS_CHECK_AT_PREP;
|
||||
/*
|
||||
* data_race() is used because folio->flags is only read here,
|
||||
* but a reference-less writer can still write to it, e.g.,
|
||||
* PG_waiters update from softleaf_entry_wait_on_locked().
|
||||
* The flag is not transferred to new_flags as it is
|
||||
* folio specific, so it is not lost.
|
||||
*/
|
||||
new_flags.f |= data_race(folio->flags.f) &
|
||||
((1L << PG_referenced) |
|
||||
(1L << PG_swapbacked) |
|
||||
(1L << PG_swapcache) |
|
||||
@@ -3810,7 +3836,19 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
|
||||
#endif
|
||||
(1L << PG_dirty) |
|
||||
(1L << PG_dropbehind) |
|
||||
LRU_GEN_MASK | LRU_REFS_MASK));
|
||||
LRU_GEN_MASK | LRU_REFS_MASK);
|
||||
|
||||
/*
|
||||
* Set PG_head before smp_wmb() and clear_compound_head().
|
||||
* After clear_compound_head() marks new_head as a folio of its
|
||||
* own, new_head->flags must not be modified by plain RMWs.
|
||||
* A plain RMW can cause a loss of atomic PG_* updates from
|
||||
* a reference-less writer, e.g., setting PG_waiters via
|
||||
* softleaf_entry_wait_on_locked().
|
||||
*/
|
||||
if (new_order)
|
||||
new_flags.f |= 1L << PG_head;
|
||||
WRITE_ONCE(new_folio->flags.f, new_flags.f);
|
||||
|
||||
new_folio->mapping = folio->mapping;
|
||||
new_folio->index = folio->index + i;
|
||||
@@ -3840,7 +3878,11 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
|
||||
*/
|
||||
clear_compound_head(new_head);
|
||||
if (new_order) {
|
||||
prep_compound_page(new_head, new_order);
|
||||
/*
|
||||
* PG_head is set above, so only set up the remaining
|
||||
* metadata on head page and tail pages.
|
||||
*/
|
||||
__prep_compound_page(new_head, new_order);
|
||||
folio_set_large_rmappable(new_folio);
|
||||
}
|
||||
|
||||
@@ -5215,6 +5257,16 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct folio *folio
|
||||
if (!(pvmw->pmd && !pvmw->pte))
|
||||
return;
|
||||
|
||||
if (!folio_test_anon(folio) && unlikely(folio_ref_count_over_limit(folio))) {
|
||||
/* Leave the file mapping absent so it can be faulted in. */
|
||||
pmd_clear(pvmw->pmd);
|
||||
if (arch_needs_pgtable_deposit())
|
||||
zap_deposited_table(mm, pvmw->pmd);
|
||||
update_hiwater_rss(mm);
|
||||
add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR);
|
||||
return;
|
||||
}
|
||||
|
||||
entry = softleaf_from_pmd(*pvmw->pmd);
|
||||
folio_get(folio);
|
||||
pmde = folio_mk_pmd(folio, READ_ONCE(vma->vm_page_prot));
|
||||
|
||||
+4
-2
@@ -834,6 +834,8 @@ static inline void prep_compound_tail(struct page *tail,
|
||||
VM_WARN_ON_ONCE(tail->private);
|
||||
}
|
||||
|
||||
void __prep_compound_page(struct page *page, unsigned int order);
|
||||
|
||||
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
|
||||
|
||||
/*
|
||||
@@ -1272,8 +1274,8 @@ struct migration_target_control {
|
||||
/*
|
||||
* mm/filemap.c
|
||||
*/
|
||||
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
|
||||
struct folio *folio, loff_t fpos, size_t size);
|
||||
ssize_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
|
||||
struct folio *folio, loff_t fpos, size_t size);
|
||||
|
||||
static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma)
|
||||
{
|
||||
|
||||
@@ -1888,6 +1888,12 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign
|
||||
goto drop_folio;
|
||||
}
|
||||
|
||||
/* Leave any uninitialized folios to the page fault path. */
|
||||
if (!folio_test_uptodate(folio)) {
|
||||
result = SCAN_FAIL;
|
||||
goto drop_folio;
|
||||
}
|
||||
|
||||
result = find_pmd_or_thp_or_none(mm, haddr, &pmd);
|
||||
switch (result) {
|
||||
case SCAN_SUCCEED:
|
||||
|
||||
@@ -32,6 +32,13 @@
|
||||
|
||||
static bool memfd_folio_has_extra_refs(struct folio *folio)
|
||||
{
|
||||
/*
|
||||
* Use the precise pincount where available to avoid racy refcount
|
||||
* comparisons.
|
||||
*/
|
||||
if (folio_has_pincount(folio) && folio_maybe_dma_pinned(folio))
|
||||
return true;
|
||||
|
||||
return folio_ref_count(folio) != folio_expected_ref_count(folio);
|
||||
}
|
||||
|
||||
|
||||
+36
-12
@@ -1168,7 +1168,10 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
|
||||
* consecutive pages of the same folio by copying them as well.
|
||||
*
|
||||
* Returns -EAGAIN if one preallocated page is required to copy the next PTE.
|
||||
* Otherwise, returns the number of copied PTEs (at least 1).
|
||||
* Returns -ENOMEM if the file folio refcount is too high and the PTE must
|
||||
* be copied.
|
||||
* Returns -EHWPOISON if copying a page fails.
|
||||
* Otherwise, returns the number of processed PTEs (at least 1).
|
||||
*/
|
||||
static inline int
|
||||
copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
|
||||
@@ -1186,6 +1189,20 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
|
||||
|
||||
folio = page_folio(page);
|
||||
|
||||
if (!folio_test_anon(folio) && unlikely(folio_ref_count_over_limit(folio))) {
|
||||
/*
|
||||
* VM_MIXEDMAP may contain pages inserted by a driver at mmap
|
||||
* time, with no fault handler to restore a missing PTE.
|
||||
*/
|
||||
if (dst_vma->vm_flags & VM_MIXEDMAP)
|
||||
return -ENOMEM;
|
||||
if (userfaultfd_pte_rwp(dst_vma, pte))
|
||||
return -ENOMEM;
|
||||
/* Preserve uffd-wp protection when faulting the page back in. */
|
||||
cond_install_uffd_wp_ptes(dst_vma, addr, dst_pte, pte, 1);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* If we likely have to copy, just don't bother with batching. Make
|
||||
* sure that the common "small folio" case is as fast as possible
|
||||
@@ -1367,9 +1384,9 @@ again:
|
||||
/*
|
||||
* If we need a pre-allocated page for this pte, drop the
|
||||
* locks, allocate, and try again.
|
||||
* If copy failed due to hwpoison in source page, break out.
|
||||
* Other errors abort the copy after releasing the page table locks.
|
||||
*/
|
||||
if (unlikely(ret == -EAGAIN || ret == -EHWPOISON))
|
||||
if (unlikely(ret < 0))
|
||||
break;
|
||||
if (unlikely(prealloc)) {
|
||||
/*
|
||||
@@ -1399,7 +1416,7 @@ again:
|
||||
goto out;
|
||||
}
|
||||
entry.val = 0;
|
||||
} else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) {
|
||||
} else if (ret == -EBUSY || ret == -ENOMEM || unlikely(ret == -EHWPOISON)) {
|
||||
goto out;
|
||||
} else if (ret == -EAGAIN) {
|
||||
prealloc = folio_prealloc(src_mm, src_vma, addr, false);
|
||||
@@ -2059,8 +2076,8 @@ retry:
|
||||
pte_unmap_unlock(start_pte, ptl);
|
||||
|
||||
/*
|
||||
* If we forced a TLB flush (either due to running out of
|
||||
* batch buffers or because we needed to flush dirty TLB
|
||||
* If we forced a TLB flush (either to release batched
|
||||
* references or because we needed to flush dirty TLB
|
||||
* entries before releasing the ptl), free the batched
|
||||
* memory too. Come back again if we didn't do everything.
|
||||
*/
|
||||
@@ -2474,6 +2491,8 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
|
||||
if (unlikely(is_zero_folio(folio))) {
|
||||
pteval = pte_mkspecial(pteval);
|
||||
} else {
|
||||
if (unlikely(folio_ref_count_over_limit(folio)))
|
||||
return -EOVERFLOW;
|
||||
folio_get(folio);
|
||||
pteval = mk_pte(page, prot);
|
||||
if (mkwrite) {
|
||||
@@ -4959,13 +4978,8 @@ static void folio_pte_range(const struct folio *folio,
|
||||
{
|
||||
/* Stay within the page table and VMA boundaries. */
|
||||
const unsigned long pmd_start = addr & PMD_MASK;
|
||||
const unsigned long pmd_end = pmd_start + PMD_SIZE;
|
||||
const unsigned long lower = max(pmd_start, vma->vm_start);
|
||||
/*
|
||||
* We cannot use pmd_addr_end(), as it does not clamp reliably. To
|
||||
* avoid wraparounds, compute using the inclusive end.
|
||||
*/
|
||||
const unsigned long upper = min(pmd_end - 1, vma->vm_end - 1) + 1;
|
||||
const unsigned long upper = pte_table_clamped_end(addr, vma->vm_end);
|
||||
const unsigned long offset = idx << PAGE_SHIFT;
|
||||
|
||||
/*
|
||||
@@ -5752,6 +5766,9 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa
|
||||
if (unlikely(folio_test_has_hwpoisoned(folio)))
|
||||
return ret;
|
||||
|
||||
if (unlikely(folio_ref_count_over_limit(folio)))
|
||||
return VM_FAULT_SIGBUS;
|
||||
|
||||
/*
|
||||
* Archs like ppc64 need additional space to store information
|
||||
* related to pte entry. Use the preallocated table for that.
|
||||
@@ -5806,6 +5823,9 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa
|
||||
* @page: The first page to create a PTE for.
|
||||
* @nr: The number of PTEs to create.
|
||||
* @addr: The first address to create a PTE for.
|
||||
*
|
||||
* For file mappings, the caller must check folio_ref_count_over_limit()
|
||||
* before installing PTEs.
|
||||
*/
|
||||
void set_pte_range(struct vm_fault *vmf, struct folio *folio,
|
||||
struct page *page, unsigned int nr, unsigned long addr)
|
||||
@@ -5972,6 +5992,10 @@ fallback:
|
||||
goto fallback;
|
||||
}
|
||||
|
||||
if (!is_cow && unlikely(folio_ref_count_over_limit(folio))) {
|
||||
ret = VM_FAULT_SIGBUS;
|
||||
goto unlock;
|
||||
}
|
||||
folio_ref_add(folio, nr_pages - 1);
|
||||
set_pte_range(vmf, folio, page, nr_pages, addr);
|
||||
type = is_cow ? MM_ANONPAGES : mm_counter_file(folio);
|
||||
|
||||
@@ -385,6 +385,17 @@ static bool remove_migration_pte(struct folio *folio,
|
||||
try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx))
|
||||
continue;
|
||||
|
||||
if (!folio_test_anon(folio) && !folio_test_hugetlb(folio) &&
|
||||
unlikely(folio_ref_count_over_limit(folio))) {
|
||||
/* File contents remain available for a later fault. */
|
||||
pte_clear(vma->vm_mm, pvmw.address, pvmw.pte);
|
||||
cond_install_uffd_wp_ptes(vma, pvmw.address,
|
||||
pvmw.pte, old_pte, 1);
|
||||
update_hiwater_rss(vma->vm_mm);
|
||||
dec_mm_counter(vma->vm_mm, mm_counter_file(folio));
|
||||
continue;
|
||||
}
|
||||
|
||||
folio_get(folio);
|
||||
new = folio_page(folio, idx);
|
||||
pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
|
||||
|
||||
+6
-4
@@ -179,10 +179,7 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb,
|
||||
#endif
|
||||
|
||||
batch = tlb->active;
|
||||
/*
|
||||
* Add the page and check if we are full. If so
|
||||
* force a flush.
|
||||
*/
|
||||
/* Add the pages before deciding whether we need to flush. */
|
||||
if (likely(nr_pages == 1)) {
|
||||
batch->encoded_pages[batch->nr++] = encode_page(page, flags);
|
||||
} else {
|
||||
@@ -190,6 +187,11 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb,
|
||||
batch->encoded_pages[batch->nr++] = encode_page(page, flags);
|
||||
batch->encoded_pages[batch->nr++] = encode_nr_pages(nr_pages);
|
||||
}
|
||||
/* Flush pending references well before new references are refused. */
|
||||
if (unlikely((unsigned int)folio_ref_count(page_folio(page)) >=
|
||||
FOLIO_REF_COUNT_FLUSH_LIMIT))
|
||||
return true;
|
||||
|
||||
/*
|
||||
* Make sure that we can always add another "page" + "nr_pages",
|
||||
* requiring two entries instead of only a single one.
|
||||
|
||||
+15
-2
@@ -681,18 +681,31 @@ static inline bool pcp_allowed_order(unsigned int order)
|
||||
* This usage means that zero-order pages may not be compound.
|
||||
*/
|
||||
|
||||
void prep_compound_page(struct page *page, unsigned int order)
|
||||
/*
|
||||
* Set up the metadata on head page and tail pages.
|
||||
* Caller must set PG_head before this.
|
||||
*/
|
||||
void __prep_compound_page(struct page *page, unsigned int order)
|
||||
{
|
||||
int i;
|
||||
int nr_pages = 1 << order;
|
||||
|
||||
__SetPageHead(page);
|
||||
for (i = 1; i < nr_pages; i++)
|
||||
prep_compound_tail(page + i, page, order);
|
||||
|
||||
prep_compound_head(page, order);
|
||||
}
|
||||
|
||||
/*
|
||||
* PG_head is set non-atomically. It should be used when no one else can modify
|
||||
* @page's flags.
|
||||
*/
|
||||
void prep_compound_page(struct page *page, unsigned int order)
|
||||
{
|
||||
__SetPageHead(page);
|
||||
__prep_compound_page(page, order);
|
||||
}
|
||||
|
||||
static inline void set_buddy_order(struct page *page, unsigned int order)
|
||||
{
|
||||
set_page_private(page, order);
|
||||
|
||||
@@ -973,7 +973,7 @@ static bool folio_referenced_one(struct folio *folio,
|
||||
pteval = ptep_get(pvmw.pte);
|
||||
|
||||
if (pvmw.pte && pte_present(pteval) && folio_test_large(folio)) {
|
||||
const unsigned long end_addr = pmd_addr_end(address, vma->vm_end);
|
||||
const unsigned long end_addr = pte_table_clamped_end(address, vma->vm_end);
|
||||
const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT;
|
||||
|
||||
nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr);
|
||||
@@ -1978,7 +1978,7 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
|
||||
return 1;
|
||||
|
||||
/* We may only batch within a single VMA and a single page table. */
|
||||
end_addr = pmd_addr_end(addr, vma->vm_end);
|
||||
end_addr = pte_table_clamped_end(addr, vma->vm_end);
|
||||
max_nr = (end_addr - addr) >> PAGE_SHIFT;
|
||||
|
||||
/* We only support lazyfree or file folios batching for now ... */
|
||||
|
||||
+5
-2
@@ -3737,7 +3737,8 @@ static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
|
||||
struct inode *inode = file_inode(in);
|
||||
struct address_space *mapping = inode->i_mapping;
|
||||
struct folio *folio = NULL;
|
||||
size_t total_spliced = 0, used, npages, n, part;
|
||||
size_t total_spliced = 0, used, npages, part;
|
||||
ssize_t n;
|
||||
loff_t isize;
|
||||
int error = 0;
|
||||
|
||||
@@ -3823,8 +3824,10 @@ static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
|
||||
n = splice_zeropage_into_pipe(pipe, *ppos, part);
|
||||
}
|
||||
|
||||
if (!n)
|
||||
if (n <= 0) {
|
||||
error = n;
|
||||
break;
|
||||
}
|
||||
len -= n;
|
||||
total_spliced += n;
|
||||
*ppos += n;
|
||||
|
||||
+6
-2
@@ -373,6 +373,10 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd,
|
||||
goto out_unlock;
|
||||
|
||||
if (page_in_cache) {
|
||||
if (unlikely(folio_ref_count_over_limit(folio))) {
|
||||
ret = -ENOMEM;
|
||||
goto out_unlock;
|
||||
}
|
||||
folio_add_file_rmap_pte(folio, page, dst_vma);
|
||||
} else {
|
||||
folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE);
|
||||
@@ -1266,8 +1270,8 @@ static long move_present_ptes(struct mm_struct *mm,
|
||||
unsigned long src_start = src_addr;
|
||||
unsigned long src_end;
|
||||
|
||||
len = pmd_addr_end(dst_addr, dst_addr + len) - dst_addr;
|
||||
src_end = pmd_addr_end(src_addr, src_addr + len);
|
||||
len = pte_table_clamped_end(dst_addr, dst_addr + len) - dst_addr;
|
||||
src_end = pte_table_clamped_end(src_addr, src_addr + len);
|
||||
flush_cache_range(src_vma, src_addr, src_end);
|
||||
double_pt_lock(dst_ptl, src_ptl);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user