mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
mm: drop fake head checks
With fake head pages eliminated in the previous commit, remove the
supporting infrastructure:
- page_fixed_fake_head(): no longer needed to detect fake heads;
- page_is_fake_head(): no longer needed;
- page_count_writable(): no longer needed for RCU protection;
- RCU read_lock in page_ref_add_unless(): no longer needed;
This substantially simplifies compound_head() and page_ref_add_unless(),
removing both branches and RCU overhead from these hot paths.
RCU was required to serialize allocation of hugetlb page against
get_page_unless_zero() and prevent writing to read-only fake head. It is
redundant without fake heads.
See bd225530a4 ("mm/hugetlb_vmemmap: fix race with speculative PFN
walkers") for more details.
synchronize_rcu() in mm/hugetlb_vmemmap.c will be removed by a separate
patch.
Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Kiryl Shutsemau <[email protected]>
Reviewed-by: Muchun Song <[email protected]>
Acked-by: David Hildenbrand (Arm) <[email protected]>
Reviewed-by: Vlastimil Babka <[email protected]>
Cc: Albert Ou <[email protected]>
Cc: Alexandre Ghiti <[email protected]>
Cc: Baoquan He <[email protected]>
Cc: Christoph Lameter <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: Frank van der Linden <[email protected]>
Cc: Harry Yoo <[email protected]>
Cc: Huacai Chen <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: Jonathan Corbet <[email protected]>
Cc: Lorenzo Stoakes <[email protected]>
Cc: Matthew Wilcox (Oracle) <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Mike Rapoport <[email protected]>
Cc: Oscar Salvador <[email protected]>
Cc: Palmer Dabbelt <[email protected]>
Cc: Paul Walmsley <[email protected]>
Cc: Roman Gushchin <[email protected]>
Cc: Usama Arif <[email protected]>
Cc: WANG Xuerui <[email protected]>
Cc: Zi Yan <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
This commit is contained in:
committed by
Andrew Morton
parent
622026e87c
commit
32c440d67e
@@ -221,102 +221,15 @@ static __always_inline bool compound_info_has_mask(void)
|
||||
return is_power_of_2(sizeof(struct page));
|
||||
}
|
||||
|
||||
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
|
||||
DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
|
||||
|
||||
/*
|
||||
* Return the real head page struct iff the @page is a fake head page, otherwise
|
||||
* return the @page itself. See Documentation/mm/vmemmap_dedup.rst.
|
||||
*/
|
||||
static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
|
||||
{
|
||||
/* Fake heads only exists if compound_info_has_mask() is true */
|
||||
if (!compound_info_has_mask())
|
||||
return page;
|
||||
|
||||
if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
|
||||
return page;
|
||||
|
||||
/*
|
||||
* Only addresses aligned with PAGE_SIZE of struct page may be fake head
|
||||
* struct page. The alignment check aims to avoid access the fields (
|
||||
* e.g. compound_info) of the @page[1]. It can avoid touch a (possibly)
|
||||
* cold cacheline in some cases.
|
||||
*/
|
||||
if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
|
||||
test_bit(PG_head, &page->flags.f)) {
|
||||
/*
|
||||
* We can safely access the field of the @page[1] with PG_head
|
||||
* because the @page is a compound page composed with at least
|
||||
* two contiguous pages.
|
||||
*/
|
||||
unsigned long info = READ_ONCE(page[1].compound_info);
|
||||
|
||||
/* See set_compound_head() */
|
||||
if (likely(info & 1)) {
|
||||
unsigned long p = (unsigned long)page;
|
||||
|
||||
return (const struct page *)(p & info);
|
||||
}
|
||||
}
|
||||
return page;
|
||||
}
|
||||
|
||||
static __always_inline bool page_count_writable(const struct page *page, int u)
|
||||
{
|
||||
if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
|
||||
return true;
|
||||
|
||||
/*
|
||||
* The refcount check is ordered before the fake-head check to prevent
|
||||
* the following race:
|
||||
* CPU 1 (HVO) CPU 2 (speculative PFN walker)
|
||||
*
|
||||
* page_ref_freeze()
|
||||
* synchronize_rcu()
|
||||
* rcu_read_lock()
|
||||
* page_is_fake_head() is false
|
||||
* vmemmap_remap_pte()
|
||||
* XXX: struct page[] becomes r/o
|
||||
*
|
||||
* page_ref_unfreeze()
|
||||
* page_ref_count() is not zero
|
||||
*
|
||||
* atomic_add_unless(&page->_refcount)
|
||||
* XXX: try to modify r/o struct page[]
|
||||
*
|
||||
* The refcount check also prevents modification attempts to other (r/o)
|
||||
* tail pages that are not fake heads.
|
||||
*/
|
||||
if (atomic_read_acquire(&page->_refcount) == u)
|
||||
return false;
|
||||
|
||||
return page_fixed_fake_head(page) == page;
|
||||
}
|
||||
#else
|
||||
static inline const struct page *page_fixed_fake_head(const struct page *page)
|
||||
{
|
||||
return page;
|
||||
}
|
||||
|
||||
static inline bool page_count_writable(const struct page *page, int u)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
static __always_inline int page_is_fake_head(const struct page *page)
|
||||
{
|
||||
return page_fixed_fake_head(page) != page;
|
||||
}
|
||||
|
||||
static __always_inline unsigned long _compound_head(const struct page *page)
|
||||
{
|
||||
unsigned long info = READ_ONCE(page->compound_info);
|
||||
|
||||
/* Bit 0 encodes PageTail() */
|
||||
if (!(info & 1))
|
||||
return (unsigned long)page_fixed_fake_head(page);
|
||||
return (unsigned long)page;
|
||||
|
||||
/*
|
||||
* If compound_info_has_mask() is false, the rest of compound_info is
|
||||
@@ -396,7 +309,7 @@ static __always_inline void clear_compound_head(struct page *page)
|
||||
|
||||
static __always_inline int PageTail(const struct page *page)
|
||||
{
|
||||
return READ_ONCE(page->compound_info) & 1 || page_is_fake_head(page);
|
||||
return READ_ONCE(page->compound_info) & 1;
|
||||
}
|
||||
|
||||
static __always_inline int PageCompound(const struct page *page)
|
||||
@@ -928,7 +841,7 @@ static __always_inline bool folio_test_head(const struct folio *folio)
|
||||
static __always_inline int PageHead(const struct page *page)
|
||||
{
|
||||
PF_POISONED_CHECK(page);
|
||||
return test_bit(PG_head, &page->flags.f) && !page_is_fake_head(page);
|
||||
return test_bit(PG_head, &page->flags.f);
|
||||
}
|
||||
|
||||
__SETPAGEFLAG(Head, head, PF_ANY)
|
||||
|
||||
@@ -230,13 +230,7 @@ static inline int folio_ref_dec_return(struct folio *folio)
|
||||
|
||||
static inline bool page_ref_add_unless(struct page *page, int nr, int u)
|
||||
{
|
||||
bool ret = false;
|
||||
|
||||
rcu_read_lock();
|
||||
/* avoid writing to the vmemmap area being remapped */
|
||||
if (page_count_writable(page, u))
|
||||
ret = atomic_add_unless(&page->_refcount, nr, u);
|
||||
rcu_read_unlock();
|
||||
bool ret = atomic_add_unless(&page->_refcount, nr, u);
|
||||
|
||||
if (page_ref_tracepoint_active(page_ref_mod_unless))
|
||||
__page_ref_mod_unless(page, nr, ret);
|
||||
|
||||
Reference in New Issue
Block a user