mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
mm/gup: limit pagecache folio refcount when grabbing references
Let's add __folio_ref_count_over_limit() checks in fast and slow GUP before acquiring additional references or pins. Keep it simple and use the check for all folio types. Also propagate try_grab_folio() errors when grabbing additional references in __get_user_pages() and in memfd_pin_folios(), so hitting the limit returns -ENOMEM instead of -EFAULT and -EINVAL, respectively. Keep the existing invalid-refcount warnings. Fixes:800d8c63b2("shmem: add huge pages support") Fixes:99cb0dbd47("mm,thp: add read-only THP support for (non-shmem) FS") Fixes:793917d997("mm/readahead: Add large folio readahead") Cc: [email protected] Reviewed-by: Jose A. Perez de Azpillaga <[email protected]> Reviewed-by: Pedro Falcato <[email protected]> Link: https://patch.msgid.link/[email protected] Signed-off-by: David Hildenbrand (Arm) <[email protected]>
This commit is contained in:
+5
-6
@@ -2266,6 +2266,9 @@ bool folio_ref_count_over_limit_slowpath(const struct folio *folio,
|
||||
* to pagecache folios (!hugetlb), either before acquiring references or
|
||||
* afterwards with rollback if the operation is refused.
|
||||
*
|
||||
* Also required when pinning folios without a dedicated pincount that use
|
||||
* GUP_PIN_COUNTING_BIAS.
|
||||
*
|
||||
* Checking other folios does no harm.
|
||||
*
|
||||
* This check is an approximation; we leave some headroom such that concurrent
|
||||
@@ -2987,12 +2990,8 @@ static inline bool folio_maybe_dma_pinned(struct folio *folio)
|
||||
return atomic_read(&folio->_pincount) > 0;
|
||||
|
||||
/*
|
||||
* folio_ref_count() is signed. If that refcount overflows, then
|
||||
* folio_ref_count() returns a negative value, and callers will avoid
|
||||
* further incrementing the refcount.
|
||||
*
|
||||
* Here, for that overflow case, use the sign bit to count a little
|
||||
* bit higher via unsigned math, and thus still get an accurate result.
|
||||
* Use unsigned math so even an unexpected negative refcount is
|
||||
* treated as possibly pinned.
|
||||
*/
|
||||
return ((unsigned int)folio_ref_count(folio)) >=
|
||||
GUP_PIN_COUNTING_BIAS;
|
||||
|
||||
@@ -74,10 +74,14 @@ static inline void sanity_check_pinned_pages(struct page **pages,
|
||||
static inline struct folio *try_get_folio(struct page *page, int refs)
|
||||
{
|
||||
struct folio *folio;
|
||||
int ref_count;
|
||||
|
||||
retry:
|
||||
folio = page_folio(page);
|
||||
if (WARN_ON_ONCE(folio_ref_count(folio) < 0))
|
||||
ref_count = folio_ref_count(folio);
|
||||
if (WARN_ON_ONCE(ref_count < 0))
|
||||
return NULL;
|
||||
if (unlikely(__folio_ref_count_over_limit(folio, ref_count)))
|
||||
return NULL;
|
||||
if (unlikely(!folio_ref_try_add(folio, refs)))
|
||||
return NULL;
|
||||
@@ -140,7 +144,13 @@ static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)
|
||||
int __must_check try_grab_folio(struct folio *folio, int refs,
|
||||
unsigned int flags)
|
||||
{
|
||||
if (WARN_ON_ONCE(folio_ref_count(folio) <= 0))
|
||||
const int ref_count = folio_ref_count(folio);
|
||||
|
||||
if (WARN_ON_ONCE(ref_count <= 0))
|
||||
return -ENOMEM;
|
||||
|
||||
if ((flags & (FOLL_GET | FOLL_PIN)) &&
|
||||
unlikely(__folio_ref_count_over_limit(folio, ref_count)))
|
||||
return -ENOMEM;
|
||||
|
||||
if (unlikely(!(flags & FOLL_PCI_P2PDMA) && folio_is_pci_p2pdma(folio)))
|
||||
@@ -1482,17 +1492,17 @@ next_page:
|
||||
struct folio *folio = page_folio(page);
|
||||
|
||||
/*
|
||||
* Since we already hold refcount on the
|
||||
* large folio, this should never fail.
|
||||
* Grabbing more references can fail once we
|
||||
* hit folio refcount limits.
|
||||
*/
|
||||
if (try_grab_folio(folio, page_increm - 1,
|
||||
gup_flags)) {
|
||||
ret = try_grab_folio(folio, page_increm - 1,
|
||||
gup_flags);
|
||||
if (ret) {
|
||||
/*
|
||||
* Release the 1st page ref if the
|
||||
* folio is problematic, fail hard.
|
||||
* Release the folio reference we
|
||||
* already took.
|
||||
*/
|
||||
gup_put_folio(folio, 1, gup_flags);
|
||||
ret = -EFAULT;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
@@ -3538,10 +3548,10 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end,
|
||||
break;
|
||||
}
|
||||
|
||||
if (try_grab_folio(folio, 1, FOLL_PIN)) {
|
||||
ret = try_grab_folio(folio, 1, FOLL_PIN);
|
||||
if (ret) {
|
||||
folio_unlock(folio);
|
||||
folio_batch_release(&fbatch);
|
||||
ret = -EINVAL;
|
||||
goto err;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user