mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
mm: page_frag: use initial zero offset for page_frag_alloc_align()
We are about to use page_frag_alloc_*() API to not just allocate memory for skb->data, but also use them to do the memory allocation for skb frag too. Currently the implementation of page_frag in mm subsystem is running the offset as a countdown rather than count-up value, there may have several advantages to that as mentioned in [1], but it may have some disadvantages, for example, it may disable skb frag coalescing and more correct cache prefetching We have a trade-off to make in order to have a unified implementation and API for page_frag, so use a initial zero offset in this patch, and the following patch will try to make some optimization to avoid the disadvantages as much as possible. 1. https://lore.kernel.org/all/[email protected]/ CC: Andrew Morton <[email protected]> CC: Linux-MM <[email protected]> Signed-off-by: Yunsheng Lin <[email protected]> Reviewed-by: Alexander Duyck <[email protected]> Link: https://patch.msgid.link/[email protected] Signed-off-by: Jakub Kicinski <[email protected]>
This commit is contained in:
committed by
Jakub Kicinski
parent
65941f10ca
commit
8218f62c9c
+27
-27
@@ -63,9 +63,13 @@ void *__page_frag_alloc_align(struct page_frag_cache *nc,
|
||||
unsigned int fragsz, gfp_t gfp_mask,
|
||||
unsigned int align_mask)
|
||||
{
|
||||
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
|
||||
unsigned int size = nc->size;
|
||||
#else
|
||||
unsigned int size = PAGE_SIZE;
|
||||
#endif
|
||||
unsigned int offset;
|
||||
struct page *page;
|
||||
int offset;
|
||||
|
||||
if (unlikely(!nc->va)) {
|
||||
refill:
|
||||
@@ -85,32 +89,12 @@ refill:
|
||||
/* reset page count bias and offset to start of new frag */
|
||||
nc->pfmemalloc = page_is_pfmemalloc(page);
|
||||
nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
|
||||
nc->offset = size;
|
||||
nc->offset = 0;
|
||||
}
|
||||
|
||||
offset = nc->offset - fragsz;
|
||||
if (unlikely(offset < 0)) {
|
||||
page = virt_to_page(nc->va);
|
||||
|
||||
if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
|
||||
goto refill;
|
||||
|
||||
if (unlikely(nc->pfmemalloc)) {
|
||||
free_unref_page(page, compound_order(page));
|
||||
goto refill;
|
||||
}
|
||||
|
||||
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
|
||||
/* if size can vary use size else just use PAGE_SIZE */
|
||||
size = nc->size;
|
||||
#endif
|
||||
/* OK, page count is 0, we can safely set it */
|
||||
set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
|
||||
|
||||
/* reset page count bias and offset to start of new frag */
|
||||
nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
|
||||
offset = size - fragsz;
|
||||
if (unlikely(offset < 0)) {
|
||||
offset = __ALIGN_KERNEL_MASK(nc->offset, ~align_mask);
|
||||
if (unlikely(offset + fragsz > size)) {
|
||||
if (unlikely(fragsz > PAGE_SIZE)) {
|
||||
/*
|
||||
* The caller is trying to allocate a fragment
|
||||
* with fragsz > PAGE_SIZE but the cache isn't big
|
||||
@@ -122,11 +106,27 @@ refill:
|
||||
*/
|
||||
return NULL;
|
||||
}
|
||||
|
||||
page = virt_to_page(nc->va);
|
||||
|
||||
if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
|
||||
goto refill;
|
||||
|
||||
if (unlikely(nc->pfmemalloc)) {
|
||||
free_unref_page(page, compound_order(page));
|
||||
goto refill;
|
||||
}
|
||||
|
||||
/* OK, page count is 0, we can safely set it */
|
||||
set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
|
||||
|
||||
/* reset page count bias and offset to start of new frag */
|
||||
nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
|
||||
offset = 0;
|
||||
}
|
||||
|
||||
nc->pagecnt_bias--;
|
||||
offset &= align_mask;
|
||||
nc->offset = offset;
|
||||
nc->offset = offset + fragsz;
|
||||
|
||||
return nc->va + offset;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user