mm: drop fake head checks

With fake head pages eliminated in the previous commit, remove the
supporting infrastructure:

  - page_fixed_fake_head(): no longer needed to detect fake heads;
  - page_is_fake_head(): no longer needed;
  - page_count_writable(): no longer needed for RCU protection;
  - RCU read_lock in page_ref_add_unless(): no longer needed;

This substantially simplifies compound_head() and page_ref_add_unless(),
removing both branches and RCU overhead from these hot paths.

RCU was required to serialize allocation of hugetlb page against
get_page_unless_zero() and prevent writing to read-only fake head.  It is
redundant without fake heads.

See bd225530a4 ("mm/hugetlb_vmemmap: fix race with speculative PFN
walkers") for more details.

synchronize_rcu() in mm/hugetlb_vmemmap.c will be removed by a separate
patch.

Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Kiryl Shutsemau <[email protected]>
Reviewed-by: Muchun Song <[email protected]>
Acked-by: David Hildenbrand (Arm) <[email protected]>
Reviewed-by: Vlastimil Babka <[email protected]>
Cc: Albert Ou <[email protected]>
Cc: Alexandre Ghiti <[email protected]>
Cc: Baoquan He <[email protected]>
Cc: Christoph Lameter <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: Frank van der Linden <[email protected]>
Cc: Harry Yoo <[email protected]>
Cc: Huacai Chen <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: Jonathan Corbet <[email protected]>
Cc: Lorenzo Stoakes <[email protected]>
Cc: Matthew Wilcox (Oracle) <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Mike Rapoport <[email protected]>
Cc: Oscar Salvador <[email protected]>
Cc: Palmer Dabbelt <[email protected]>
Cc: Paul Walmsley <[email protected]>
Cc: Roman Gushchin <[email protected]>
Cc: Usama Arif <[email protected]>
Cc: WANG Xuerui <[email protected]>
Cc: Zi Yan <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
This commit is contained in:
Kiryl Shutsemau
2026-04-05 13:53:09 -07:00
committed by Andrew Morton
parent 622026e87c
commit 32c440d67e
2 changed files with 4 additions and 97 deletions
+3 -90
View File
@@ -221,102 +221,15 @@ static __always_inline bool compound_info_has_mask(void)
return is_power_of_2(sizeof(struct page));
}
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
/*
* Return the real head page struct iff the @page is a fake head page, otherwise
* return the @page itself. See Documentation/mm/vmemmap_dedup.rst.
*/
static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
{
/* Fake heads only exists if compound_info_has_mask() is true */
if (!compound_info_has_mask())
return page;
if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
return page;
/*
* Only addresses aligned with PAGE_SIZE of struct page may be fake head
* struct page. The alignment check aims to avoid access the fields (
* e.g. compound_info) of the @page[1]. It can avoid touch a (possibly)
* cold cacheline in some cases.
*/
if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
test_bit(PG_head, &page->flags.f)) {
/*
* We can safely access the field of the @page[1] with PG_head
* because the @page is a compound page composed with at least
* two contiguous pages.
*/
unsigned long info = READ_ONCE(page[1].compound_info);
/* See set_compound_head() */
if (likely(info & 1)) {
unsigned long p = (unsigned long)page;
return (const struct page *)(p & info);
}
}
return page;
}
static __always_inline bool page_count_writable(const struct page *page, int u)
{
if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
return true;
/*
* The refcount check is ordered before the fake-head check to prevent
* the following race:
* CPU 1 (HVO) CPU 2 (speculative PFN walker)
*
* page_ref_freeze()
* synchronize_rcu()
* rcu_read_lock()
* page_is_fake_head() is false
* vmemmap_remap_pte()
* XXX: struct page[] becomes r/o
*
* page_ref_unfreeze()
* page_ref_count() is not zero
*
* atomic_add_unless(&page->_refcount)
* XXX: try to modify r/o struct page[]
*
* The refcount check also prevents modification attempts to other (r/o)
* tail pages that are not fake heads.
*/
if (atomic_read_acquire(&page->_refcount) == u)
return false;
return page_fixed_fake_head(page) == page;
}
#else
static inline const struct page *page_fixed_fake_head(const struct page *page)
{
return page;
}
static inline bool page_count_writable(const struct page *page, int u)
{
return true;
}
#endif
static __always_inline int page_is_fake_head(const struct page *page)
{
return page_fixed_fake_head(page) != page;
}
static __always_inline unsigned long _compound_head(const struct page *page)
{
unsigned long info = READ_ONCE(page->compound_info);
/* Bit 0 encodes PageTail() */
if (!(info & 1))
return (unsigned long)page_fixed_fake_head(page);
return (unsigned long)page;
/*
* If compound_info_has_mask() is false, the rest of compound_info is
@@ -396,7 +309,7 @@ static __always_inline void clear_compound_head(struct page *page)
static __always_inline int PageTail(const struct page *page)
{
return READ_ONCE(page->compound_info) & 1 || page_is_fake_head(page);
return READ_ONCE(page->compound_info) & 1;
}
static __always_inline int PageCompound(const struct page *page)
@@ -928,7 +841,7 @@ static __always_inline bool folio_test_head(const struct folio *folio)
static __always_inline int PageHead(const struct page *page)
{
PF_POISONED_CHECK(page);
return test_bit(PG_head, &page->flags.f) && !page_is_fake_head(page);
return test_bit(PG_head, &page->flags.f);
}
__SETPAGEFLAG(Head, head, PF_ANY)
+1 -7
View File
@@ -230,13 +230,7 @@ static inline int folio_ref_dec_return(struct folio *folio)
static inline bool page_ref_add_unless(struct page *page, int nr, int u)
{
bool ret = false;
rcu_read_lock();
/* avoid writing to the vmemmap area being remapped */
if (page_count_writable(page, u))
ret = atomic_add_unless(&page->_refcount, nr, u);
rcu_read_unlock();
bool ret = atomic_add_unless(&page->_refcount, nr, u);
if (page_ref_tracepoint_active(page_ref_mod_unless))
__page_ref_mod_unless(page, nr, ret);