diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 3f4a180b48f3..4bba02fd5d54 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -75,8 +75,8 @@ * - __tlb_remove_folio_pages_size() * * __tlb_remove_folio_pages_size() is the basic primitive that queues pages - * for freeing. It will return a boolean indicating if the queue is (now) - * full and a call to tlb_flush_mmu() is required. + * for freeing. It returns true when a call to tlb_flush_mmu() is required, + * either because the queue is full or a folio is near its refcount limit. * * tlb_remove_page() and tlb_remove_page_size() imply the call to * tlb_flush_mmu() when required and has no return value. diff --git a/include/linux/mm.h b/include/linux/mm.h index 95d5e9a13e51..12a090d8d07f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2252,6 +2252,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf); /* Leave separate reserves for administrators and concurrent reference gains. */ #define FOLIO_REF_COUNT_LIMIT (INT_MAX - (1U << 26)) #define FOLIO_REF_COUNT_ADMIN_LIMIT (INT_MAX - (1U << 25)) +#define FOLIO_REF_COUNT_FLUSH_LIMIT (INT_MAX - (1U << 27)) bool folio_ref_count_over_limit_slowpath(const struct folio *folio, unsigned int ref_count); diff --git a/mm/memory.c b/mm/memory.c index 115b994cabfd..7f019afc35ca 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2076,8 +2076,8 @@ retry: pte_unmap_unlock(start_pte, ptl); /* - * If we forced a TLB flush (either due to running out of - * batch buffers or because we needed to flush dirty TLB + * If we forced a TLB flush (either to release batched + * references or because we needed to flush dirty TLB * entries before releasing the ptl), free the batched * memory too. Come back again if we didn't do everything. */ diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 9f353f0e2ef4..e98397ac577c 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -179,10 +179,7 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb, #endif batch = tlb->active; - /* - * Add the page and check if we are full. If so - * force a flush. - */ + /* Add the pages before deciding whether we need to flush. */ if (likely(nr_pages == 1)) { batch->encoded_pages[batch->nr++] = encode_page(page, flags); } else { @@ -190,6 +187,11 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb, batch->encoded_pages[batch->nr++] = encode_page(page, flags); batch->encoded_pages[batch->nr++] = encode_nr_pages(nr_pages); } + /* Flush pending references well before new references are refused. */ + if (unlikely((unsigned int)folio_ref_count(page_folio(page)) >= + FOLIO_REF_COUNT_FLUSH_LIMIT)) + return true; + /* * Make sure that we can always add another "page" + "nr_pages", * requiring two entries instead of only a single one.