Files
linux/mm/mprotect.c
Lorenzo Stoakes (ARM) 13fc99f463 mm: do not unintentionally overwrite atomic VMA flags
Patch series "mm: avoid losing atomic VMA flags", v1.

Since commit 2ab7f1bbaf ("mm/madvise: allow guard page install/remove
under VMA lock"), VMA_MAYBE_GUARD_BIT can be set atomically under the VMA
read lock.

However, a number of places wrongly assume the flags are stable under the
read lock, which can cause stale atomic VMA flags to overwrite existing
valid ones.

Additionally, one case in which VMA_SOFTDIRTY_BIT was overwritten is also
addressed.

This series addresses each of these, while keeping the logic inline with
existing conventions.

Each patch in this series is a hotfix.


This patch (of 4):

madvise, mlock and mprotect take a copy of a VMA's flags holding only the
VMA read lock, later writing it back under the write lock (in
madvise_update_vma(), mlock_fixup(), mlock_vma_pages_range() and
mprotect_fixup()).

This was done on the assumption that VMA flags cannot change with a read
lock held.

However this changed in commit 2ab7f1bbaf ("mm/madvise: allow guard page
install/remove under VMA lock"), which allowed VM_ATOMIC_SET_ALLOWED
flags (currently only VMA_MAYBE_GUARD_BIT) to be written concurrently under
a VMA read lock.

The actual write itself is safe as this is done under the VMA write lock
at which point no further concurrent writes can occur, however at this
point the stale flags mean that these callers can unintentionally overwrite
atomic flags incorrectly.

The consequences of this are significant - a missing VMA_MAYBE_GUARD_BIT
could result in page tables not being copied on fork (the vma_needs_copy()
check can return false) or being overwritten on file-backed THP
collapse (the file_backed_vma_is_retractable() can return true).

Resolve the issue by always taking care to retain existing atomic flags
when overwriting VMA flags, replacing the inconsistently named
vma_flags_reset_once() with vma_overwrite_flags() to do so, and always
invoking it when flags are overwritten like this.

Reads are a problem too - madvise, mlock and mprotect all compare VMA flags
without a VMA write lock held via vma_flags_same_pair().

This has more serious impact, as concurrent changes to atomic VMA flags can
cause two sets of VMA flags to be wrongly considered different.

For mprotect and madvise this causes a useless split, but mlock is impacted
more seriously as it can result in mm->locked_vm underflow.

Fix this by updating vma_flags_same_pair() to disregard
VM_ATOMIC_SET_ALLOWED VMAs - VMA equality is the same as VMA mergeability,
and atomic VMA flags are disregarded when comparing VMA flags for merge -
so this is justified semantically.

Additionally, remove the redundant vma_flags_same_mask() and the unused
vma_flags_same() as part of the change, and replace VM_ATOMIC_SET_ALLOWED
with VMA_ATOMIC_SET_ALLOWED, using the new VMA flag API.

Finally, update userland VMA tests to reflect the changes.

Link: https://lore.kernel.org/[email protected]
Link: https://lore.kernel.org/[email protected]
Fixes: 2ab7f1bbaf ("mm/madvise: allow guard page install/remove under VMA lock")
Signed-off-by: Lorenzo Stoakes (ARM) <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Suggested-by: David Hildenbrand (Arm) <[email protected]>
Acked-by: David Hildenbrand (Arm) <[email protected]>
Reviewed-by: Pedro Falcato <[email protected]>
Tested-by: Gregory Price (Meta) <[email protected]>
Cc: Liam R. Howlett <[email protected]>
Cc: Vlastimil Babka <[email protected]>
Cc: Mike Rapoport <[email protected]>
Cc: Suren Baghdasaryan <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Jann Horn <[email protected]>
Cc: Davidlohr Bueso <[email protected]>
Cc: SJ Park <[email protected]>
Cc: Kees Cook <[email protected]>
Cc: Colin Cross <[email protected]>
Cc: <[email protected]>
2026-11-02 18:14:42 -08:00

1079 lines
29 KiB
C

// SPDX-License-Identifier: GPL-2.0
/*
* mm/mprotect.c
*
* (C) Copyright 1994 Linus Torvalds
* (C) Copyright 2002 Christoph Hellwig
*
* Address space accounting code <[email protected]>
* (C) Copyright 2002 Red Hat Inc, All Rights Reserved
*/
#include <linux/pagewalk.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <linux/mman.h>
#include <linux/fs.h>
#include <linux/highmem.h>
#include <linux/security.h>
#include <linux/mempolicy.h>
#include <linux/personality.h>
#include <linux/syscalls.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/mmu_notifier.h>
#include <linux/migrate.h>
#include <linux/perf_event.h>
#include <linux/pkeys.h>
#include <linux/ksm.h>
#include <linux/uaccess.h>
#include <linux/mm_inline.h>
#include <linux/pgtable.h>
#include <linux/userfaultfd_k.h>
#include <uapi/linux/mman.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include "internal.h"
static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte)
{
if (WARN_ON_ONCE(!vma_test(vma, VMA_WRITE_BIT)))
return false;
/* Don't touch entries that are not even readable. */
if (pte_protnone(pte))
return false;
/* Do we need write faults for softdirty tracking? */
if (pte_needs_soft_dirty_wp(vma, pte))
return false;
/* Do we need write faults for uffd-wp tracking? */
if (userfaultfd_pte_wp(vma, pte))
return false;
return true;
}
static bool can_change_private_pte_writable(struct vm_area_struct *vma,
unsigned long addr, pte_t pte)
{
struct page *page;
if (!maybe_change_pte_writable(vma, pte))
return false;
/*
* Writable MAP_PRIVATE mapping: We can only special-case on
* exclusive anonymous pages, because we know that our
* write-fault handler similarly would map them writable without
* any additional checks while holding the PT lock.
*/
page = vm_normal_page(vma, addr, pte);
return page && PageAnon(page) && PageAnonExclusive(page);
}
static bool can_change_shared_pte_writable(struct vm_area_struct *vma,
pte_t pte)
{
if (!maybe_change_pte_writable(vma, pte))
return false;
VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
/*
* Writable MAP_SHARED mapping: "clean" might indicate that the FS still
* needs a real write-fault for writenotify
* (see vma_wants_writenotify()). If "dirty", the assumption is that the
* FS was already notified and we can simply mark the PTE writable
* just like the write-fault handler would do.
*/
return pte_dirty(pte);
}
bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
pte_t pte)
{
if (!vma_test(vma, VMA_SHARED_BIT))
return can_change_private_pte_writable(vma, addr, pte);
return can_change_shared_pte_writable(vma, pte);
}
static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep,
pte_t pte, int max_nr_ptes, fpb_t flags)
{
/* No underlying folio, so cannot batch */
if (!folio)
return 1;
if (!folio_test_large(folio))
return 1;
return folio_pte_batch_flags(folio, NULL, ptep, &pte, max_nr_ptes, flags);
}
/* Set nr_ptes number of ptes, starting from idx */
static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
int nr_ptes, int idx, bool set_write, struct mmu_gather *tlb)
{
/*
* Advance the position in the batch by idx; note that if idx > 0,
* then the nr_ptes passed here is <= batch size - idx.
*/
addr += idx * PAGE_SIZE;
ptep += idx;
oldpte = pte_advance_pfn(oldpte, idx);
ptent = pte_advance_pfn(ptent, idx);
if (set_write)
ptent = pte_mkwrite(ptent, vma);
modify_prot_commit_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes);
if (pte_needs_flush(oldpte, ptent))
tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
}
/*
* Get max length of consecutive ptes pointing to PageAnonExclusive() pages or
* !PageAnonExclusive() pages, starting from start_idx. Caller must enforce
* that the ptes point to consecutive pages of the same anon large folio.
*/
static __always_inline int page_anon_exclusive_batch(int start_idx, int max_len,
struct page *first_page, bool expected_anon_exclusive)
{
int idx;
for (idx = start_idx + 1; idx < start_idx + max_len; ++idx) {
if (expected_anon_exclusive != PageAnonExclusive(first_page + idx))
break;
}
return idx - start_idx;
}
/*
* This function is a result of trying our very best to retain the
* "avoid the write-fault handler" optimization. In can_change_pte_writable(),
* if the vma is a private vma, and we cannot determine whether to change
* the pte to writable just from the vma and the pte, we then need to look
* at the actual page pointed to by the pte. Unfortunately, if we have a
* batch of ptes pointing to consecutive pages of the same anon large folio,
* the anon-exclusivity (or the negation) of the first page does not guarantee
* the anon-exclusivity (or the negation) of the other pages corresponding to
* the pte batch; hence in this case it is incorrect to decide to change or
* not change the ptes to writable just by using information from the first
* pte of the batch. Therefore, we must individually check all pages and
* retrieve sub-batches.
*/
static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma,
struct folio *folio, struct page *first_page, unsigned long addr, pte_t *ptep,
pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
{
bool expected_anon_exclusive;
int batch_idx = 0;
int len;
while (nr_ptes) {
expected_anon_exclusive = PageAnonExclusive(first_page + batch_idx);
len = page_anon_exclusive_batch(batch_idx, nr_ptes,
first_page, expected_anon_exclusive);
prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len,
batch_idx, expected_anon_exclusive, tlb);
batch_idx += len;
nr_ptes -= len;
}
}
static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_struct *vma,
struct folio *folio, struct page *page, unsigned long addr, pte_t *ptep,
pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
{
bool set_write;
if (vma_test(vma, VMA_SHARED_BIT)) {
set_write = can_change_shared_pte_writable(vma, ptent);
prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes,
/* idx = */ 0, set_write, tlb);
return;
}
set_write = maybe_change_pte_writable(vma, ptent) &&
(folio && folio_test_anon(folio));
if (!set_write) {
prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes,
/* idx = */ 0, set_write, tlb);
return;
}
commit_anon_folio_batch(vma, folio, page, addr, ptep, oldpte, ptent, nr_ptes, tlb);
}
static long change_softleaf_pte(struct vm_area_struct *vma,
unsigned long addr, pte_t *pte, pte_t oldpte, unsigned long cp_flags)
{
const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
const bool uffd_prot_resolve = cp_flags &
(MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
softleaf_t entry = softleaf_from_pte(oldpte);
pte_t newpte;
if (softleaf_is_migration_write(entry)) {
const struct folio *folio = softleaf_to_folio(entry);
/*
* A protection check is difficult so
* just be safe and disable write
*/
if (folio_test_anon(folio))
entry = make_readable_exclusive_migration_entry(swp_offset(entry));
else
entry = make_readable_migration_entry(swp_offset(entry));
newpte = swp_entry_to_pte(entry);
if (pte_swp_soft_dirty(oldpte))
newpte = pte_swp_mksoft_dirty(newpte);
} else if (softleaf_is_device_private_write(entry)) {
/*
* We do not preserve soft-dirtiness. See
* copy_nonpresent_pte() for explanation.
*/
entry = make_readable_device_private_entry(swp_offset(entry));
newpte = swp_entry_to_pte(entry);
if (pte_swp_uffd(oldpte))
newpte = pte_swp_mkuffd(newpte);
} else if (softleaf_is_marker(entry)) {
/*
* Ignore error swap entries unconditionally,
* because any access should sigbus/sigsegv
* anyway.
*/
if (softleaf_is_poison_marker(entry) ||
softleaf_is_guard_marker(entry))
return 0;
/*
* If this is uffd-wp pte marker and we'd like
* to unprotect it, drop it; the next page
* fault will trigger without uffd trapping.
*/
if (uffd_prot_resolve) {
pte_clear(vma->vm_mm, addr, pte);
return 1;
}
return 0;
} else {
newpte = oldpte;
}
if (uffd_prot)
newpte = pte_swp_mkuffd(newpte);
else if (uffd_prot_resolve)
newpte = pte_swp_clear_uffd(newpte);
if (!pte_same(oldpte, newpte)) {
set_pte_at(vma->vm_mm, addr, pte, newpte);
return 1;
}
return 0;
}
static __always_inline void change_present_ptes(struct mmu_gather *tlb,
struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
int nr_ptes, unsigned long end, pgprot_t newprot,
struct folio *folio, struct page *page, unsigned long cp_flags)
{
const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
const bool uffd_prot_resolve = cp_flags &
(MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
pte_t ptent, oldpte;
oldpte = modify_prot_start_ptes(vma, addr, ptep, nr_ptes);
ptent = pte_modify(oldpte, newprot);
if (uffd_prot)
ptent = pte_mkuffd(ptent);
else if (uffd_prot_resolve)
ptent = pte_clear_uffd(ptent);
/*
* The uffd bit on a VM_UFFD_RWP VMA carries PROT_NONE
* semantics. If mprotect() or NUMA hinting changed the
* base protection, restore PAGE_NONE so the PTE still
* traps on any access. pte_modify() preserves
* _PAGE_UFFD.
*/
if (userfaultfd_rwp(vma) && pte_uffd(ptent))
ptent = pte_modify(ptent, PAGE_NONE);
/*
* In some writable, shared mappings, we might want
* to catch actual write access -- see
* vma_wants_writenotify().
*
* In all writable, private mappings, we have to
* properly handle COW.
*
* In both cases, we can sometimes still change PTEs
* writable and avoid the write-fault handler, for
* example, if a PTE is already dirty and no other
* COW or special handling is required.
*/
if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
!pte_write(ptent))
set_write_prot_commit_flush_ptes(vma, folio, page,
addr, ptep, oldpte, ptent, nr_ptes, tlb);
else
prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent,
nr_ptes, /* idx = */ 0, /* set_write = */ false, tlb);
}
static long change_pte_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
unsigned long end, pgprot_t newprot, unsigned long cp_flags)
{
pte_t *pte, oldpte;
spinlock_t *ptl;
long pages = 0;
bool is_private_single_threaded;
bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
int nr_ptes;
tlb_change_page_size(tlb, PAGE_SIZE);
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
if (!pte)
return -EAGAIN;
if (prot_numa)
is_private_single_threaded = vma_is_single_threaded_private(vma);
flush_tlb_batched_pending(vma->vm_mm);
lazy_mmu_mode_enable();
do {
nr_ptes = 1;
oldpte = ptep_get(pte);
if (pte_present(oldpte)) {
const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE;
int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
struct folio *folio = NULL;
struct page *page;
/* Already in the desired state. */
if (prot_numa && pte_protnone(oldpte))
continue;
/*
* RWP-protected PTEs carry _PAGE_UFFD as a marker on
* top of PROT_NONE. Skip only entries already in that
* exact state; plain PROT_NONE from mprotect() still needs
* to be promoted so future faults can be distinguished.
*/
if (uffd_rwp && pte_protnone(oldpte) && pte_uffd(oldpte))
continue;
page = vm_normal_page(vma, addr, oldpte);
if (page)
folio = page_folio(page);
/*
* Avoid trapping faults against the zero or KSM
* pages. See similar comment in change_huge_pmd.
* Skip this filter for uffd RWP which
* must set protnone regardless of NUMA placement.
*/
if (prot_numa &&
!folio_can_map_prot_numa(folio, vma,
is_private_single_threaded)) {
/* determine batch to skip */
nr_ptes = mprotect_folio_pte_batch(folio,
pte, oldpte, max_nr_ptes, /* flags = */ 0);
continue;
}
nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags);
/*
* Optimize for the small-folio common case by
* special-casing it here. Compiler constant propagation
* plus copious amounts of __always_inline does wonders.
*/
if (likely(nr_ptes == 1)) {
change_present_ptes(tlb, vma, addr, pte, 1,
end, newprot, folio, page, cp_flags);
} else {
change_present_ptes(tlb, vma, addr, pte,
nr_ptes, end, newprot, folio, page,
cp_flags);
}
pages += nr_ptes;
} else if (pte_none(oldpte)) {
/*
* Nobody plays with any none ptes besides
* userfaultfd when applying the protections.
*/
if (likely(!uffd_wp))
continue;
if (userfaultfd_wp_use_markers(vma)) {
/*
* For file-backed mem, we need to be able to
* wr-protect a none pte, because even if the
* pte is none, the page/swap cache could
* exist. Doing that by install a marker.
*/
set_pte_at(vma->vm_mm, addr, pte,
make_pte_marker(PTE_MARKER_UFFD_WP));
pages++;
}
} else {
pages += change_softleaf_pte(vma, addr, pte, oldpte, cp_flags);
}
} while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end);
lazy_mmu_mode_disable();
pte_unmap_unlock(pte - 1, ptl);
return pages;
}
/*
* Return true if we want to split THPs into PTE mappings in change
* protection procedure, false otherwise.
*/
static inline bool
pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags)
{
/*
* pte markers only resides in pte level, if we need pte markers,
* we need to split. For example, we cannot wr-protect a file thp
* (e.g. 2M shmem) because file thp is handled differently when
* split by erasing the pmd so far.
*/
return (cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP)) && !vma_is_anonymous(vma);
}
/*
* Return true if we want to populate pgtables in change protection
* procedure, false otherwise
*/
static inline bool
pgtable_populate_needed(struct vm_area_struct *vma, unsigned long cp_flags)
{
/* If not within ioctl(UFFDIO_WRITEPROTECT), then don't bother */
if (!(cp_flags & MM_CP_UFFD_WP))
return false;
/* Populate if the userfaultfd mode requires pte markers */
return userfaultfd_wp_use_markers(vma);
}
/*
* Populate the pgtable underneath for whatever reason if requested.
* When {pte|pmd|...}_alloc() failed we treat it the same way as pgtable
* allocation failures during page faults by kicking OOM and returning
* error.
*/
#define change_pmd_prepare(vma, pmd, cp_flags) \
({ \
long err = 0; \
if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \
if (pte_alloc(vma->vm_mm, pmd)) \
err = -ENOMEM; \
} \
err; \
})
/*
* This is the general pud/p4d/pgd version of change_pmd_prepare(). We need to
* have separate change_pmd_prepare() because pte_alloc() returns 0 on success,
* while {pmd|pud|p4d}_alloc() returns the valid pointer on success.
*/
#define change_prepare(vma, high, low, addr, cp_flags) \
({ \
long err = 0; \
if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \
low##_t *p = low##_alloc(vma->vm_mm, high, addr); \
if (p == NULL) \
err = -ENOMEM; \
} \
err; \
})
static inline long change_pmd_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, pud_t *pud, unsigned long addr,
unsigned long end, pgprot_t newprot, unsigned long cp_flags)
{
pmd_t *pmd;
unsigned long next;
long pages = 0;
unsigned long nr_huge_updates = 0;
pmd = pmd_offset(pud, addr);
do {
long ret;
pmd_t _pmd;
again:
next = pmd_addr_end(addr, end);
ret = change_pmd_prepare(vma, pmd, cp_flags);
if (ret) {
pages = ret;
break;
}
if (pmd_none(*pmd))
goto next;
_pmd = pmdp_get_lockless(pmd);
if (pmd_is_huge(_pmd)) {
if ((next - addr != HPAGE_PMD_SIZE) ||
pgtable_split_needed(vma, cp_flags)) {
__split_huge_pmd(vma, pmd, addr, false);
/*
* For file-backed, the pmd could have been
* cleared; make sure pmd populated if
* necessary, then fall-through to pte level.
*/
ret = change_pmd_prepare(vma, pmd, cp_flags);
if (ret) {
pages = ret;
break;
}
} else {
ret = change_huge_pmd(tlb, vma, pmd,
addr, newprot, cp_flags);
if (ret) {
if (ret == HPAGE_PMD_NR) {
pages += HPAGE_PMD_NR;
nr_huge_updates++;
}
/* huge pmd was handled */
goto next;
}
}
/* fall through, the trans huge pmd just split */
}
ret = change_pte_range(tlb, vma, pmd, addr, next, newprot,
cp_flags);
if (ret < 0)
goto again;
pages += ret;
next:
cond_resched();
} while (pmd++, addr = next, addr != end);
if (nr_huge_updates)
count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
return pages;
}
static inline long change_pud_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr,
unsigned long end, pgprot_t newprot, unsigned long cp_flags)
{
struct mmu_notifier_range range;
pud_t *pudp, pud;
unsigned long next;
long pages = 0, ret;
range.start = 0;
pudp = pud_offset(p4d, addr);
do {
again:
next = pud_addr_end(addr, end);
ret = change_prepare(vma, pudp, pmd, addr, cp_flags);
if (ret) {
pages = ret;
break;
}
pud = pudp_get(pudp);
if (pud_none(pud))
continue;
if (!range.start) {
mmu_notifier_range_init(&range,
MMU_NOTIFY_PROTECTION_VMA, 0,
vma->vm_mm, addr, end);
mmu_notifier_invalidate_range_start(&range);
}
if (pud_leaf(pud)) {
if ((next - addr != PUD_SIZE) ||
pgtable_split_needed(vma, cp_flags)) {
__split_huge_pud(vma, pudp, addr);
goto again;
} else {
ret = change_huge_pud(tlb, vma, pudp,
addr, newprot, cp_flags);
if (ret == 0)
goto again;
/* huge pud was handled */
if (ret == HPAGE_PUD_NR)
pages += HPAGE_PUD_NR;
continue;
}
}
pages += change_pmd_range(tlb, vma, pudp, addr, next, newprot,
cp_flags);
} while (pudp++, addr = next, addr != end);
if (range.start)
mmu_notifier_invalidate_range_end(&range);
return pages;
}
static inline long change_p4d_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr,
unsigned long end, pgprot_t newprot, unsigned long cp_flags)
{
p4d_t *p4d;
unsigned long next;
long pages = 0, ret;
p4d = p4d_offset(pgd, addr);
do {
next = p4d_addr_end(addr, end);
ret = change_prepare(vma, p4d, pud, addr, cp_flags);
if (ret)
return ret;
if (p4d_none_or_clear_bad(p4d))
continue;
pages += change_pud_range(tlb, vma, p4d, addr, next, newprot,
cp_flags);
} while (p4d++, addr = next, addr != end);
return pages;
}
static long change_protection_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, unsigned long addr,
unsigned long end, pgprot_t newprot, unsigned long cp_flags)
{
struct mm_struct *mm = vma->vm_mm;
pgd_t *pgd;
unsigned long next;
long pages = 0, ret;
BUG_ON(addr >= end);
pgd = pgd_offset(mm, addr);
tlb_start_vma(tlb, vma);
do {
next = pgd_addr_end(addr, end);
ret = change_prepare(vma, pgd, p4d, addr, cp_flags);
if (ret) {
pages = ret;
break;
}
if (pgd_none_or_clear_bad(pgd))
continue;
pages += change_p4d_range(tlb, vma, pgd, addr, next, newprot,
cp_flags);
} while (pgd++, addr = next, addr != end);
tlb_end_vma(tlb, vma);
return pages;
}
long change_protection(struct mmu_gather *tlb,
struct vm_area_struct *vma, unsigned long start,
unsigned long end, unsigned long cp_flags)
{
pgprot_t newprot = vma->vm_page_prot;
long pages;
/*
* MM_CP_UFFD_{WP,RWP} and _RESOLVE are mutually exclusive within one
* change, and WP and RWP cannot mix. Miswired callers get a warn and
* a no-op; userspace cannot reach this state.
*/
if (WARN_ON_ONCE((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL ||
(cp_flags & MM_CP_UFFD_RWP_ALL) == MM_CP_UFFD_RWP_ALL ||
((cp_flags & MM_CP_UFFD_WP_ALL) &&
(cp_flags & MM_CP_UFFD_RWP_ALL))))
return 0;
#ifdef CONFIG_NUMA_BALANCING
/*
* Ordinary protection updates (mprotect, uffd-wp, softdirty tracking)
* are expected to reflect their requirements via VMA flags such that
* vma_set_page_prot() will adjust vma->vm_page_prot accordingly.
*/
if (cp_flags & MM_CP_PROT_NUMA)
newprot = PAGE_NONE;
#else
WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA);
#endif
if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE) &&
(cp_flags & MM_CP_UFFD_RWP))
newprot = PAGE_NONE;
if (vma_is_hugetlb(vma))
pages = hugetlb_change_protection(vma, start, end, newprot,
cp_flags);
else
pages = change_protection_range(tlb, vma, start, end, newprot,
cp_flags);
return pages;
}
static int prot_none_pte_entry(pte_t *pte, unsigned long addr,
unsigned long next, struct mm_walk *walk)
{
return pfn_modify_allowed(pte_pfn(ptep_get(pte)),
*(pgprot_t *)(walk->private)) ?
0 : -EACCES;
}
#ifdef CONFIG_HUGETLB_PAGE
static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask,
unsigned long addr, unsigned long next,
struct mm_walk *walk)
{
const pte_t entry = huge_ptep_get(walk->mm, addr, pte);
if (pfn_modify_allowed(pte_pfn(entry), *(pgprot_t *)(walk->private)))
return 0;
return -EACCES;
}
#else
#define prot_none_hugetlb_entry NULL
#endif
static const struct mm_walk_ops prot_none_walk_ops = {
.pte_entry = prot_none_pte_entry,
.hugetlb_entry = prot_none_hugetlb_entry,
.walk_lock = PGWALK_WRLOCK,
};
int
mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
struct vm_area_struct *vma, struct vm_area_struct **pprev,
unsigned long start, unsigned long end, vm_flags_t newflags)
{
struct mm_struct *mm = vma->vm_mm;
const vma_flags_t old_vma_flags = READ_ONCE(vma->flags);
vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags);
long nrpages = (end - start) >> PAGE_SHIFT;
unsigned int mm_cp_flags = 0;
unsigned long charged = 0;
int error;
if (vma_is_sealed(vma))
return -EPERM;
if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags)) {
*pprev = vma;
return 0;
}
/*
* Do PROT_NONE PFN permission checks here when we can still
* bail out without undoing a lot of state. This is a rather
* uncommon case, so doesn't need to be very optimized.
*/
if (arch_has_pfn_modify_check() &&
!vma_flags_is_mm_managed(&old_vma_flags) &&
!vma_flags_test_any_mask(&new_vma_flags, VMA_ACCESS_FLAGS)) {
pgprot_t new_pgprot = vm_get_page_prot(newflags);
error = walk_page_range_vma(vma, start, end,
&prot_none_walk_ops, &new_pgprot);
if (error)
return error;
}
/*
* If we make a private mapping writable we increase our commit;
* but (without finer accounting) cannot reduce our commit if we
* make it unwritable again except in the anonymous case where the
* VMA's anon rmap has yet to be assigned.
*
* hugetlb mapping were accounted for even if read-only so there is
* no need to account for them here.
*/
if (vma_flags_test(&new_vma_flags, VMA_WRITE_BIT)) {
/* Check space limits when area turns into data. */
if (!may_expand_vm(mm, &new_vma_flags, nrpages) &&
may_expand_vm(mm, &old_vma_flags, nrpages))
return -ENOMEM;
if (!vma_flags_test_any(&old_vma_flags,
VMA_ACCOUNT_BIT, VMA_WRITE_BIT, VMA_HUGETLB_BIT,
VMA_SHARED_BIT, VMA_NORESERVE_BIT)) {
charged = nrpages;
if (security_vm_enough_memory_mm(mm, charged))
return -ENOMEM;
vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT);
}
} else if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) &&
vma_is_anonymous(vma) && !vma_has_anon_rmap(vma)) {
vma_flags_clear(&new_vma_flags, VMA_ACCOUNT_BIT);
}
vma = vma_modify_flags(vmi, *pprev, vma, start, end, &new_vma_flags);
if (IS_ERR(vma)) {
error = PTR_ERR(vma);
goto fail;
}
*pprev = vma;
/*
* vm_flags and vm_page_prot are protected by the mmap_lock
* held in write mode.
*/
vma_start_write(vma);
vma_overwrite_flags(vma, new_vma_flags);
if (vma_wants_manual_pte_write_upgrade(vma))
mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
vma_set_page_prot(vma);
change_protection(tlb, vma, start, end, mm_cp_flags);
if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) &&
!vma_flags_test(&new_vma_flags, VMA_ACCOUNT_BIT))
vm_unacct_memory(nrpages);
/*
* Private VMA_LOCKED_BIT VMA becoming writable: trigger COW to avoid
* major fault on access.
*/
if (vma_flags_test(&new_vma_flags, VMA_WRITE_BIT) &&
vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT) &&
!vma_flags_test_any(&old_vma_flags, VMA_WRITE_BIT, VMA_SHARED_BIT))
populate_vma_page_range(vma, start, end, NULL);
vm_stat_account(mm, vma_flags_to_legacy(old_vma_flags), -nrpages);
newflags = vma_flags_to_legacy(new_vma_flags);
vm_stat_account(mm, newflags, nrpages);
perf_event_mmap(vma);
return 0;
fail:
vm_unacct_memory(charged);
return error;
}
/*
* pkey==-1 when doing a legacy mprotect()
*/
static int do_mprotect_pkey(unsigned long start, size_t len,
unsigned long prot, int pkey)
{
unsigned long nstart, end, tmp, reqprot;
struct vm_area_struct *vma, *prev;
int error;
const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
(prot & PROT_READ);
struct mmu_gather tlb;
struct vma_iterator vmi;
start = untagged_addr(start);
prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
return -EINVAL;
if (start & ~PAGE_MASK)
return -EINVAL;
if (!len)
return 0;
len = PAGE_ALIGN(len);
end = start + len;
if (end <= start)
return -ENOMEM;
if (!arch_validate_prot(prot, start))
return -EINVAL;
reqprot = prot;
if (mmap_write_lock_killable(current->mm))
return -EINTR;
/*
* If userspace did not allocate the pkey, do not let
* them use it here.
*/
error = -EINVAL;
if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
goto out;
vma_iter_init(&vmi, current->mm, start);
vma = vma_find(&vmi, end);
error = -ENOMEM;
if (!vma)
goto out;
if (unlikely(grows & PROT_GROWSDOWN)) {
if (vma->vm_start >= end)
goto out;
start = vma->vm_start;
error = -EINVAL;
if (!vma_test(vma, VMA_GROWSDOWN_BIT))
goto out;
} else {
if (vma->vm_start > start)
goto out;
if (unlikely(grows & PROT_GROWSUP)) {
end = vma->vm_end;
error = -EINVAL;
if (!vma_test_single_mask(vma, VMA_GROWSUP))
goto out;
}
}
prev = vma_prev(&vmi);
if (start > vma->vm_start)
prev = vma;
tlb_gather_mmu(&tlb, current->mm);
nstart = start;
tmp = vma->vm_start;
for_each_vma_range(vmi, vma, end) {
vm_flags_t mask_off_old_flags;
vma_flags_t new_vma_flags;
vm_flags_t newflags;
int new_vma_pkey;
if (vma->vm_start != tmp) {
error = -ENOMEM;
break;
}
/* Does the application expect PROT_READ to imply PROT_EXEC */
if (rier && vma_test(vma, VMA_MAYEXEC_BIT))
prot |= PROT_EXEC;
/*
* Each mprotect() call explicitly passes r/w/x permissions.
* If a permission is not passed to mprotect(), it must be
* cleared from the VMA.
*/
mask_off_old_flags = VM_ACCESS_FLAGS | VM_FLAGS_CLEAR;
new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);
newflags = calc_vm_prot_bits(prot, new_vma_pkey);
newflags |= (vma->vm_flags & ~mask_off_old_flags);
new_vma_flags = legacy_to_vma_flags(newflags);
/* newflags >> 4 shift VM_MAY% in place of VM_% */
if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) {
error = -EACCES;
break;
}
if (map_deny_write_exec(&vma->flags, &new_vma_flags)) {
error = -EACCES;
break;
}
/* Allow architectures to sanity-check the new flags */
if (!arch_validate_flags(newflags)) {
error = -EINVAL;
break;
}
error = security_file_mprotect(vma, reqprot, prot);
if (error)
break;
tmp = vma->vm_end;
if (tmp > end)
tmp = end;
if (vma->vm_ops && vma->vm_ops->mprotect) {
error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags);
if (error)
break;
}
error = mprotect_fixup(&vmi, &tlb, vma, &prev, nstart, tmp, newflags);
if (error)
break;
tmp = vma_iter_end(&vmi);
nstart = tmp;
prot = reqprot;
}
tlb_finish_mmu(&tlb);
if (!error && tmp < end)
error = -ENOMEM;
out:
mmap_write_unlock(current->mm);
return error;
}
SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
unsigned long, prot)
{
return do_mprotect_pkey(start, len, prot, -1);
}
#ifdef CONFIG_ARCH_HAS_PKEYS
SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len,
unsigned long, prot, int, pkey)
{
return do_mprotect_pkey(start, len, prot, pkey);
}
SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val)
{
int pkey;
int ret;
/* No flags supported yet. */
if (flags)
return -EINVAL;
/* check for unsupported init values */
if (init_val & ~PKEY_ACCESS_MASK)
return -EINVAL;
mmap_write_lock(current->mm);
pkey = mm_pkey_alloc(current->mm);
ret = -ENOSPC;
if (pkey == -1)
goto out;
ret = arch_set_user_pkey_access(pkey, init_val);
if (ret) {
mm_pkey_free(current->mm, pkey);
goto out;
}
ret = pkey;
out:
mmap_write_unlock(current->mm);
return ret;
}
SYSCALL_DEFINE1(pkey_free, int, pkey)
{
int ret;
mmap_write_lock(current->mm);
ret = mm_pkey_free(current->mm, pkey);
mmap_write_unlock(current->mm);
/*
* We could provide warnings or errors if any VMA still
* has the pkey set here.
*/
return ret;
}
#endif /* CONFIG_ARCH_HAS_PKEYS */