mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
mm/mmu_gather: flush pending references early before the folio limit
When a task zaps mappings (e.g., MADV_DONTNEED), it can drop the PT lock and defer dropping the folio references. In the meantime, page faults can refault pagecache folios, creating more references for the same folios. These deferred references can build up over multiple VMAs. Let's just request a flush after enqueueing folios whose refcount reached a certain threshold. Start flushing well before new references are refused, regardless of the caller's privileges. When returning "true" from __tlb_remove_folio_pages_size(), PTE zapping will flush the TLB before dropping the PT lock, but drop the references afterwards. Refaults can still race with that, but flushing early reduces the risk of letting others hit the limit. This is a temporary measure until we have something better. Reported-by: Chris Mason <[email protected]> Fixes:800d8c63b2("shmem: add huge pages support") Fixes:99cb0dbd47("mm,thp: add read-only THP support for (non-shmem) FS") Fixes:793917d997("mm/readahead: Add large folio readahead") Cc: [email protected] Reviewed-by: Jose A. Perez de Azpillaga <[email protected]> Reviewed-by: Lance Yang <[email protected]> Reviewed-by: Pedro Falcato <[email protected]> Link: https://patch.msgid.link/[email protected] Signed-off-by: David Hildenbrand (Arm) <[email protected]>
This commit is contained in:
@@ -75,8 +75,8 @@
|
||||
* - __tlb_remove_folio_pages_size()
|
||||
*
|
||||
* __tlb_remove_folio_pages_size() is the basic primitive that queues pages
|
||||
* for freeing. It will return a boolean indicating if the queue is (now)
|
||||
* full and a call to tlb_flush_mmu() is required.
|
||||
* for freeing. It returns true when a call to tlb_flush_mmu() is required,
|
||||
* either because the queue is full or a folio is near its refcount limit.
|
||||
*
|
||||
* tlb_remove_page() and tlb_remove_page_size() imply the call to
|
||||
* tlb_flush_mmu() when required and has no return value.
|
||||
|
||||
@@ -2252,6 +2252,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf);
|
||||
/* Leave separate reserves for administrators and concurrent reference gains. */
|
||||
#define FOLIO_REF_COUNT_LIMIT (INT_MAX - (1U << 26))
|
||||
#define FOLIO_REF_COUNT_ADMIN_LIMIT (INT_MAX - (1U << 25))
|
||||
#define FOLIO_REF_COUNT_FLUSH_LIMIT (INT_MAX - (1U << 27))
|
||||
|
||||
bool folio_ref_count_over_limit_slowpath(const struct folio *folio,
|
||||
unsigned int ref_count);
|
||||
|
||||
+2
-2
@@ -2076,8 +2076,8 @@ retry:
|
||||
pte_unmap_unlock(start_pte, ptl);
|
||||
|
||||
/*
|
||||
* If we forced a TLB flush (either due to running out of
|
||||
* batch buffers or because we needed to flush dirty TLB
|
||||
* If we forced a TLB flush (either to release batched
|
||||
* references or because we needed to flush dirty TLB
|
||||
* entries before releasing the ptl), free the batched
|
||||
* memory too. Come back again if we didn't do everything.
|
||||
*/
|
||||
|
||||
+6
-4
@@ -179,10 +179,7 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb,
|
||||
#endif
|
||||
|
||||
batch = tlb->active;
|
||||
/*
|
||||
* Add the page and check if we are full. If so
|
||||
* force a flush.
|
||||
*/
|
||||
/* Add the pages before deciding whether we need to flush. */
|
||||
if (likely(nr_pages == 1)) {
|
||||
batch->encoded_pages[batch->nr++] = encode_page(page, flags);
|
||||
} else {
|
||||
@@ -190,6 +187,11 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb,
|
||||
batch->encoded_pages[batch->nr++] = encode_page(page, flags);
|
||||
batch->encoded_pages[batch->nr++] = encode_nr_pages(nr_pages);
|
||||
}
|
||||
/* Flush pending references well before new references are refused. */
|
||||
if (unlikely((unsigned int)folio_ref_count(page_folio(page)) >=
|
||||
FOLIO_REF_COUNT_FLUSH_LIMIT))
|
||||
return true;
|
||||
|
||||
/*
|
||||
* Make sure that we can always add another "page" + "nr_pages",
|
||||
* requiring two entries instead of only a single one.
|
||||
|
||||
Reference in New Issue
Block a user