mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
hugetlb: remove VMEMMAP_SYNCHRONIZE_RCU
The VMEMMAP_SYNCHRONIZE_RCU flag triggered synchronize_rcu() calls to prevent a race between HVO remapping and page_ref_add_unless(). The race could occur when a speculative PFN walker tried to modify the refcount on a struct page that was in the process of being remapped to a fake head. With fake heads eliminated, page_ref_add_unless() no longer needs RCU protection. Remove the flag and synchronize_rcu() calls. Link: https://lkml.kernel.org/r/[email protected] Signed-off-by: Kiryl Shutsemau <[email protected]> Reviewed-by: Muchun Song <[email protected]> Reviewed-by: David Hildenbrand (Arm) <[email protected]> Cc: Albert Ou <[email protected]> Cc: Alexandre Ghiti <[email protected]> Cc: Baoquan He <[email protected]> Cc: Christoph Lameter <[email protected]> Cc: David Rientjes <[email protected]> Cc: Frank van der Linden <[email protected]> Cc: Harry Yoo <[email protected]> Cc: Huacai Chen <[email protected]> Cc: Johannes Weiner <[email protected]> Cc: Jonathan Corbet <[email protected]> Cc: Lorenzo Stoakes <[email protected]> Cc: Matthew Wilcox (Oracle) <[email protected]> Cc: Michal Hocko <[email protected]> Cc: Mike Rapoport <[email protected]> Cc: Oscar Salvador <[email protected]> Cc: Palmer Dabbelt <[email protected]> Cc: Paul Walmsley <[email protected]> Cc: Roman Gushchin <[email protected]> Cc: Usama Arif <[email protected]> Cc: Vlastimil Babka <[email protected]> Cc: WANG Xuerui <[email protected]> Cc: Zi Yan <[email protected]> Signed-off-by: Andrew Morton <[email protected]>
This commit is contained in:
committed by
Andrew Morton
parent
32c440d67e
commit
01b1d0ffb6
+4
-16
@@ -47,8 +47,6 @@ struct vmemmap_remap_walk {
|
||||
#define VMEMMAP_SPLIT_NO_TLB_FLUSH BIT(0)
|
||||
/* Skip the TLB flush when we remap the PTE */
|
||||
#define VMEMMAP_REMAP_NO_TLB_FLUSH BIT(1)
|
||||
/* synchronize_rcu() to avoid writes from page_ref_add_unless() */
|
||||
#define VMEMMAP_SYNCHRONIZE_RCU BIT(2)
|
||||
unsigned long flags;
|
||||
};
|
||||
|
||||
@@ -409,9 +407,6 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h,
|
||||
if (!folio_test_hugetlb_vmemmap_optimized(folio))
|
||||
return 0;
|
||||
|
||||
if (flags & VMEMMAP_SYNCHRONIZE_RCU)
|
||||
synchronize_rcu();
|
||||
|
||||
vmemmap_start = (unsigned long)&folio->page;
|
||||
vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h);
|
||||
|
||||
@@ -444,7 +439,7 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h,
|
||||
*/
|
||||
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio)
|
||||
{
|
||||
return __hugetlb_vmemmap_restore_folio(h, folio, VMEMMAP_SYNCHRONIZE_RCU);
|
||||
return __hugetlb_vmemmap_restore_folio(h, folio, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -467,14 +462,11 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
|
||||
struct folio *folio, *t_folio;
|
||||
long restored = 0;
|
||||
long ret = 0;
|
||||
unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH | VMEMMAP_SYNCHRONIZE_RCU;
|
||||
unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
|
||||
|
||||
list_for_each_entry_safe(folio, t_folio, folio_list, lru) {
|
||||
if (folio_test_hugetlb_vmemmap_optimized(folio)) {
|
||||
ret = __hugetlb_vmemmap_restore_folio(h, folio, flags);
|
||||
/* only need to synchronize_rcu() once for each batch */
|
||||
flags &= ~VMEMMAP_SYNCHRONIZE_RCU;
|
||||
|
||||
if (ret)
|
||||
break;
|
||||
restored++;
|
||||
@@ -554,8 +546,6 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
|
||||
|
||||
static_branch_inc(&hugetlb_optimize_vmemmap_key);
|
||||
|
||||
if (flags & VMEMMAP_SYNCHRONIZE_RCU)
|
||||
synchronize_rcu();
|
||||
/*
|
||||
* Very Subtle
|
||||
* If VMEMMAP_REMAP_NO_TLB_FLUSH is set, TLB flushing is not performed
|
||||
@@ -613,7 +603,7 @@ void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio)
|
||||
{
|
||||
LIST_HEAD(vmemmap_pages);
|
||||
|
||||
__hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, VMEMMAP_SYNCHRONIZE_RCU);
|
||||
__hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, 0);
|
||||
free_vmemmap_page_list(&vmemmap_pages);
|
||||
}
|
||||
|
||||
@@ -641,7 +631,7 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
|
||||
struct folio *folio;
|
||||
int nr_to_optimize;
|
||||
LIST_HEAD(vmemmap_pages);
|
||||
unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH | VMEMMAP_SYNCHRONIZE_RCU;
|
||||
unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
|
||||
|
||||
nr_to_optimize = 0;
|
||||
list_for_each_entry(folio, folio_list, lru) {
|
||||
@@ -694,8 +684,6 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
|
||||
int ret;
|
||||
|
||||
ret = __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, flags);
|
||||
/* only need to synchronize_rcu() once for each batch */
|
||||
flags &= ~VMEMMAP_SYNCHRONIZE_RCU;
|
||||
|
||||
/*
|
||||
* Pages to be freed may have been accumulated. If we
|
||||
|
||||
Reference in New Issue
Block a user