diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 5ca0aefee9aa..615f7730a5bd 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1247,6 +1247,75 @@ static noinline void check_multi_find_3(struct xarray *xa) } } +static noinline void check_multi_find_4(struct xarray *xa) +{ +#ifdef CONFIG_XARRAY_MULTI + unsigned int order = XA_CHUNK_SHIFT + 1; + unsigned long next = 1UL << order; + unsigned long start = next - 1; + XA_STATE(xas, xa, start); + XA_STATE_ORDER(split, xa, 0, 0); + void *entry; + unsigned long i; + + /* Multi-index entry in two slots of a non-leaf node. */ + xa_store_order(xa, 0, order, xa_mk_index(0), GFP_KERNEL); + XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL); + + rcu_read_lock(); + entry = xas_find(&xas, ULONG_MAX); + XA_BUG_ON(xa, entry != xa_mk_index(0)); + XA_BUG_ON(xa, xas.xa_index != start); + + entry = xas_find(&xas, ULONG_MAX); + XA_BUG_ON(xa, entry != xa_mk_index(next)); + XA_BUG_ON(xa, xas.xa_index != next); + + entry = xas_find(&xas, ULONG_MAX); + XA_BUG_ON(xa, entry != NULL); + rcu_read_unlock(); + + xa_erase_index(xa, next); + xa_erase_index(xa, 0); + XA_BUG_ON(xa, !xa_empty(xa)); + + /* Split the multi-index entry after a lookup begins inside it. */ + xa_store_order(xa, 0, order, xa_mk_index(0), GFP_KERNEL); + XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL); + + xas_set(&xas, start); + rcu_read_lock(); + entry = xas_find(&xas, ULONG_MAX); + XA_BUG_ON(xa, entry != xa_mk_index(0)); + XA_BUG_ON(xa, xas.xa_index != start); + rcu_read_unlock(); + + xas_split_alloc(&split, xa_mk_index(0), order, GFP_KERNEL); + if (xas_error(&split)) { + XA_BUG_ON(xa, true); + goto out; + } + xas_lock(&split); + xas_split(&split, xa_mk_index(0), order); + for (i = 0; i < next; i++) + __xa_store(xa, i, xa_mk_index(i), 0); + xas_unlock(&split); + + rcu_read_lock(); + entry = xas_find(&xas, ULONG_MAX); + XA_BUG_ON(xa, entry != xa_mk_index(next)); + XA_BUG_ON(xa, xas.xa_index != next); + + entry = xas_find(&xas, ULONG_MAX); + XA_BUG_ON(xa, entry != NULL); + rcu_read_unlock(); + +out: + xa_destroy(xa); + XA_BUG_ON(xa, !xa_empty(xa)); +#endif +} + static noinline void check_find_1(struct xarray *xa) { unsigned long i, j, k; @@ -1370,6 +1439,7 @@ static noinline void check_find(struct xarray *xa) check_multi_find_1(xa, i); check_multi_find_2(xa); check_multi_find_3(xa); + check_multi_find_4(xa); } /* See find_swap_entry() in mm/shmem.c */ diff --git a/lib/xarray.c b/lib/xarray.c index ae9d7fc68861..48f440d114e1 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1410,9 +1410,11 @@ void *xas_find(struct xa_state *xas, unsigned long max) entry = xas_load(xas); if (entry || xas_not_node(xas->xa_node)) return entry; - } else if (!xas->xa_node->shift && - xas->xa_offset != (xas->xa_index & XA_CHUNK_MASK)) { - xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1; + } else if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node)) { + if (!xas->xa_node->shift) + xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1; + else + xas->xa_offset = get_offset(xas->xa_index, xas->xa_node); } xas_next_offset(xas);