diff --git a/Documentation/core-api/assoc_array.rst b/Documentation/core-api/assoc_array.rst index 19d89f92bf8d..c5ada1bff06f 100644 --- a/Documentation/core-api/assoc_array.rst +++ b/Documentation/core-api/assoc_array.rst @@ -123,10 +123,12 @@ This points to a number of methods, all of which need to be provided: unsigned long (*get_key_chunk)(const void *index_key, int level); - This should return a chunk of caller-supplied index key starting at the - *bit* position given by the level argument. The level argument will be a - multiple of ``ASSOC_ARRAY_KEY_CHUNK_SIZE`` and the function should return - ``ASSOC_ARRAY_KEY_CHUNK_SIZE bits``. No error is possible. + This should return the ``ASSOC_ARRAY_KEY_CHUNK_SIZE`` bits of + caller-supplied index key that contain the *bit* position given by the + level argument, starting at level rounded down to a multiple of + ``ASSOC_ARRAY_KEY_CHUNK_SIZE``. The level is often that of a node, which + is a multiple of ``ASSOC_ARRAY_LEVEL_STEP``, so the function has to do the + rounding. No error is possible. 2. Get a chunk of an object's index key:: diff --git a/include/linux/assoc_array_priv.h b/include/linux/assoc_array_priv.h index dca733ef6750..37a9f76e1505 100644 --- a/include/linux/assoc_array_priv.h +++ b/include/linux/assoc_array_priv.h @@ -80,7 +80,7 @@ struct assoc_array_edit { struct assoc_array_ptr **leaf_p; struct assoc_array_ptr *dead_leaf; struct assoc_array_ptr *new_meta[3]; - struct assoc_array_ptr *excised_meta[1]; + struct assoc_array_ptr *excised_meta[2]; struct assoc_array_ptr *excised_subtree; struct assoc_array_ptr **set_backpointers[ASSOC_ARRAY_FAN_OUT]; struct assoc_array_ptr *set_backpointers_to; diff --git a/lib/assoc_array.c b/lib/assoc_array.c index b6c9723e12ce..f19888ec3e09 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c @@ -470,33 +470,32 @@ static bool assoc_array_insert_in_empty_tree(struct assoc_array_edit *edit) } /* - * Handle insertion into a terminal node. + * The pointer that leads to a node or shortcut with the given back pointer + * and parent slot: the root, a slot in a node, or a shortcut's next node. */ -static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit, - const struct assoc_array_ops *ops, - const void *index_key, - struct assoc_array_walk_result *result) +static struct assoc_array_ptr **assoc_array_parent_link(struct assoc_array *array, + struct assoc_array_ptr *back_pointer, + int parent_slot) +{ + if (!back_pointer) + return &array->root; + if (assoc_array_ptr_is_node(back_pointer)) + return &assoc_array_ptr_to_node(back_pointer)->slots[parent_slot]; + return &assoc_array_ptr_to_shortcut(back_pointer)->next_node; +} + +/* + * Insert into a terminal node without changing its shape, if that can be + * done: in place of a leaf with the same index key, or in a free slot. + */ +static bool assoc_array_insert_in_node(struct assoc_array_edit *edit, + const struct assoc_array_ops *ops, + const void *index_key, + struct assoc_array_node *node) { - struct assoc_array_shortcut *shortcut, *new_s0; - struct assoc_array_node *node, *new_n0, *new_n1, *side; struct assoc_array_ptr *ptr; - unsigned long dissimilarity, base_seg, blank; - size_t keylen; - bool have_meta; - int level, diff; - int slot, next_slot, free_slot, i, j; + int free_slot, i; - node = result->terminal_node.node; - level = result->terminal_node.level; - edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = result->terminal_node.slot; - - pr_devel("-->%s()\n", __func__); - - /* We arrived at a node which doesn't have an onward node or shortcut - * pointer that we have to follow. This means that (a) the leaf we - * want must go here (either by insertion or replacement) or (b) we - * need to split this node and insert in one of the fragments. - */ free_slot = -1; /* Firstly, we have to check the leaves in this node to see if there's @@ -514,7 +513,6 @@ static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit, pr_devel("replace in slot %d\n", i); edit->leaf_p = &node->slots[i]; edit->dead_leaf = node->slots[i]; - pr_devel("<--%s() = ok [replace]\n", __func__); return true; } } @@ -526,28 +524,28 @@ static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit, pr_devel("insert in free slot %d\n", free_slot); edit->leaf_p = &node->slots[free_slot]; edit->adjust_count_on = node; - pr_devel("<--%s() = ok [insert]\n", __func__); return true; } - /* The node has no spare slots - so we're either going to have to split - * it or insert another node before it. - * - * Whatever, we're going to need at least two new nodes - so allocate - * those now. We may also need a new shortcut, but we deal with that - * when we need it. - */ - new_n0 = kzalloc_obj(struct assoc_array_node); - if (!new_n0) - return false; - edit->new_meta[0] = assoc_array_node_to_ptr(new_n0); - new_n1 = kzalloc_obj(struct assoc_array_node); - if (!new_n1) - return false; - edit->new_meta[1] = assoc_array_node_to_ptr(new_n1); + return false; +} + +/* + * Fill in the edit script's segment cache for a full node: the segment at + * this level of each leaf's index key, or 0xff for a metadata pointer. + * The new key's segment, in the last entry, is left to the caller. Returns + * true if there are any metadata pointers. + */ +static bool assoc_array_fill_segment_cache(struct assoc_array_edit *edit, + const struct assoc_array_ops *ops, + const struct assoc_array_node *node, + int level) +{ + struct assoc_array_ptr *ptr; + unsigned long base_seg; + bool have_meta; + int i; - /* We need to find out how similar the leaves are. */ - pr_devel("no spare slots\n"); have_meta = false; for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) { ptr = node->slots[i]; @@ -561,65 +559,42 @@ static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit, base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK; edit->segment_cache[i] = base_seg & ASSOC_ARRAY_FAN_MASK; } + return have_meta; +} - if (have_meta) { - pr_devel("have meta\n"); - goto split_node; - } +/* + * Whether the leaves of a full node and the new key all share one segment, + * so that splitting the node cannot make room. + */ +static bool assoc_array_all_one_segment(const struct assoc_array_edit *edit) +{ + int i; - /* The node contains only leaves */ - dissimilarity = 0; - base_seg = edit->segment_cache[0]; - for (i = 1; i < ASSOC_ARRAY_FAN_OUT; i++) - dissimilarity |= edit->segment_cache[i] ^ base_seg; + for (i = 1; i < ASSOC_ARRAY_FAN_OUT + 1; i++) + if (edit->segment_cache[i] != edit->segment_cache[0]) + return false; + return true; +} - pr_devel("only leaves; dissimilarity=%lx\n", dissimilarity); +/* + * Split a full node into new_n0, which takes its place, and new_n1, below + * new_n0 in the slot of a segment that at least two of the leaves share, + * the new one included. The caller has filled in the segment cache and + * set new_n0's back pointer and parent slot. + */ +static void assoc_array_split_node(struct assoc_array_edit *edit, + struct assoc_array_node *node, + struct assoc_array_node *new_n0, + struct assoc_array_node *new_n1) +{ + struct assoc_array_shortcut *shortcut; + struct assoc_array_node *side; + struct assoc_array_ptr *ptr; + int slot, next_slot, free_slot, i, j; - if ((dissimilarity & ASSOC_ARRAY_FAN_MASK) == 0) { - /* The old leaves all cluster in the same slot. We will need - * to insert a shortcut if the new node wants to cluster with them. - */ - if ((edit->segment_cache[ASSOC_ARRAY_FAN_OUT] ^ base_seg) == 0) - goto all_leaves_cluster_together; - - /* Otherwise all the old leaves cluster in the same slot, but - * the new leaf wants to go into a different slot - so we - * create a new node (n0) to hold the new leaf and a pointer to - * a new node (n1) holding all the old leaves. - * - * This can be done by falling through to the node splitting - * path. - */ - pr_devel("present leaves cluster but not new leaf\n"); - } - -split_node: pr_devel("split node\n"); - /* We need to split the current node. The node must contain anything - * from a single leaf (in the one leaf case, this leaf will cluster - * with the new leaf) and the rest meta-pointers, to all leaves, some - * of which may cluster. - * - * It won't contain the case in which all the current leaves plus the - * new leaves want to cluster in the same slot. - * - * We need to expel at least two leaves out of a set consisting of the - * leaves in the node and the new leaf. The current meta pointers can - * just be copied as they shouldn't cluster with any of the leaves. - * - * We need a new node (n0) to replace the current one and a new node to - * take the expelled nodes (n1). - */ - edit->set[0].to = assoc_array_node_to_ptr(new_n0); - new_n0->back_pointer = node->back_pointer; - new_n0->parent_slot = node->parent_slot; new_n1->back_pointer = assoc_array_node_to_ptr(new_n0); - new_n1->parent_slot = -1; /* Need to calculate this */ - -do_split_node: - pr_devel("do_split_node\n"); - new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch; new_n1->nr_leaves_on_branch = 0; @@ -699,26 +674,29 @@ found_slot_for_multiple_occupancy: } } } +} - ptr = node->back_pointer; - if (!ptr) - edit->set[0].ptr = &edit->array->root; - else if (assoc_array_ptr_is_node(ptr)) - edit->set[0].ptr = &assoc_array_ptr_to_node(ptr)->slots[node->parent_slot]; - else - edit->set[0].ptr = &assoc_array_ptr_to_shortcut(ptr)->next_node; - edit->excised_meta[0] = assoc_array_node_to_ptr(node); - pr_devel("<--%s() = ok [split node]\n", __func__); - return true; +/* + * All the leaves, new and old, want to cluster together in this node in the + * same slot, so we have to replace this node with a shortcut to skip over + * the identical parts of the key, leading to new_n0. If the node is the + * target of a shortcut, the new one replaces that shortcut as well. The + * caller then splits the leaves between new_n0 and new_n1 at the shortcut's + * far end, for which this fills in the segment cache. + */ +static struct assoc_array_shortcut * +assoc_array_insert_shortcut(struct assoc_array_edit *edit, + const struct assoc_array_ops *ops, + const void *index_key, + struct assoc_array_node *node, int level, + struct assoc_array_node *new_n0) +{ + struct assoc_array_shortcut *new_s0; + unsigned long base_seg, blank; + size_t keylen; + int diff, i; -all_leaves_cluster_together: - /* All the leaves, new and old, want to cluster together in this node - * in the same slot, so we have to replace this node with a shortcut to - * skip over the identical parts of the key and then place a pair of - * nodes, one inside the other, at the end of the shortcut and - * distribute the keys between them. - * - * Firstly we need to work out where the leaves start diverging as a + /* Firstly we need to work out where the leaves start diverging as a * bit position into their keys so that we know how big the shortcut * needs to be. * @@ -744,17 +722,28 @@ all_leaves_cluster_together: new_s0 = kzalloc_flex(*new_s0, index_key, keylen); if (!new_s0) - return false; + return NULL; edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s0); - edit->set[0].to = assoc_array_shortcut_to_ptr(new_s0); new_s0->back_pointer = node->back_pointer; new_s0->parent_slot = node->parent_slot; + + /* A shortcut has to lead to a node, so a node at the end of a shortcut + * is replaced together with that shortcut. The new one holds the index + * key from the root, so it covers what the old one did. + */ + if (node->back_pointer && + assoc_array_ptr_is_shortcut(node->back_pointer)) { + struct assoc_array_shortcut *above = + assoc_array_ptr_to_shortcut(node->back_pointer); + + new_s0->back_pointer = above->back_pointer; + new_s0->parent_slot = above->parent_slot; + edit->excised_meta[1] = node->back_pointer; + } new_s0->next_node = assoc_array_node_to_ptr(new_n0); new_n0->back_pointer = assoc_array_shortcut_to_ptr(new_s0); new_n0->parent_slot = 0; - new_n1->back_pointer = assoc_array_node_to_ptr(new_n0); - new_n1->parent_slot = -1; /* Need to calculate this */ new_s0->skip_to_level = level = diff & ~ASSOC_ARRAY_LEVEL_STEP_MASK; pr_devel("skip_to_level = %d [diff %d]\n", level, diff); @@ -773,18 +762,104 @@ all_leaves_cluster_together: /* This now reduces to a node splitting exercise for which we'll need * to regenerate the disparity table. */ - for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) { - ptr = node->slots[i]; - base_seg = ops->get_object_key_chunk(assoc_array_ptr_to_leaf(ptr), - level); - base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK; - edit->segment_cache[i] = base_seg & ASSOC_ARRAY_FAN_MASK; - } - + assoc_array_fill_segment_cache(edit, ops, node, level); base_seg = ops->get_key_chunk(index_key, level); base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK; edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = base_seg & ASSOC_ARRAY_FAN_MASK; - goto do_split_node; + return new_s0; +} + +/* + * Handle insertion into a terminal node. + */ +static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit, + const struct assoc_array_ops *ops, + const void *index_key, + struct assoc_array_walk_result *result) +{ + struct assoc_array_shortcut *new_s0; + struct assoc_array_node *node, *new_n0, *new_n1; + bool have_meta; + int level; + + node = result->terminal_node.node; + level = result->terminal_node.level; + + pr_devel("-->%s()\n", __func__); + + /* We arrived at a node which doesn't have an onward node or shortcut + * pointer that we have to follow. This means that (a) the leaf we + * want must go here (either by insertion or replacement) or (b) we + * need to split this node and insert in one of the fragments. + */ + if (assoc_array_insert_in_node(edit, ops, index_key, node)) { + pr_devel("<--%s() = ok [in node]\n", __func__); + return true; + } + + /* The node has no spare slots - so we're either going to have to split + * it or insert another node before it. + * + * Whatever, we're going to need at least two new nodes - so allocate + * those now. We may also need a new shortcut, but we deal with that + * when we need it. + */ + new_n0 = kzalloc_obj(struct assoc_array_node); + if (!new_n0) + return false; + edit->new_meta[0] = assoc_array_node_to_ptr(new_n0); + new_n1 = kzalloc_obj(struct assoc_array_node); + if (!new_n1) + return false; + edit->new_meta[1] = assoc_array_node_to_ptr(new_n1); + + /* We need to find out how similar the leaves are. */ + pr_devel("no spare slots\n"); + have_meta = assoc_array_fill_segment_cache(edit, ops, node, level); + edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = result->terminal_node.slot; + + if (!have_meta && assoc_array_all_one_segment(edit)) { + /* The node only contains leaves and all the leaves cluster in + * the same slot, so we need to insert a shortcut to skip the + * common portion of the key and split the node at the + * shortcut's far end. The shortcut goes wherever its back + * pointer says, which may be further up than the node. + */ + new_s0 = assoc_array_insert_shortcut(edit, ops, index_key, + node, level, new_n0); + if (!new_s0) + return false; + edit->set[0].ptr = assoc_array_parent_link(edit->array, + new_s0->back_pointer, + new_s0->parent_slot); + edit->set[0].to = assoc_array_shortcut_to_ptr(new_s0); + } else { + /* We need to split the current node. The node must contain + * anything from a single leaf (in the one leaf case, this leaf + * will cluster with the new leaf) and the rest meta-pointers, + * to all leaves, some of which may cluster. + * + * We need to expel at least two leaves out of a set consisting + * of the leaves in the node and the new leaf. The current meta + * pointers can just be copied as they shouldn't cluster with + * any of the leaves. If all the old leaves cluster in one slot + * and the new leaf does not, they all go and the new leaf stays. + * + * We need a new node (n0) to replace the current one and a new + * node to take the expelled nodes (n1). + */ + new_n0->back_pointer = node->back_pointer; + new_n0->parent_slot = node->parent_slot; + edit->set[0].ptr = assoc_array_parent_link(edit->array, + new_n0->back_pointer, + new_n0->parent_slot); + edit->set[0].to = assoc_array_node_to_ptr(new_n0); + } + + assoc_array_split_node(edit, node, new_n0, new_n1); + edit->excised_meta[0] = assoc_array_node_to_ptr(node); + pr_devel("<--%s() = ok [split node]\n", __func__); + return true; } /* @@ -795,7 +870,7 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, struct assoc_array_walk_result *result) { struct assoc_array_shortcut *shortcut, *new_s0, *new_s1; - struct assoc_array_node *node, *new_n0, *side; + struct assoc_array_node *new_n0, *side; unsigned long sc_segments, dissimilarity, blank; size_t keylen; int level, sc_level, diff; @@ -821,14 +896,11 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, diff += sc_level & ~ASSOC_ARRAY_KEY_CHUNK_MASK; pr_devel("diff=%d\n", diff); - if (!shortcut->back_pointer) { - edit->set[0].ptr = &edit->array->root; - } else if (assoc_array_ptr_is_node(shortcut->back_pointer)) { - node = assoc_array_ptr_to_node(shortcut->back_pointer); - edit->set[0].ptr = &node->slots[shortcut->parent_slot]; - } else { - BUG(); - } + BUG_ON(shortcut->back_pointer && + !assoc_array_ptr_is_node(shortcut->back_pointer)); + edit->set[0].ptr = assoc_array_parent_link(edit->array, + shortcut->back_pointer, + shortcut->parent_slot); edit->excised_meta[0] = assoc_array_shortcut_to_ptr(shortcut); @@ -843,7 +915,8 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, * zero length - otherwise we just connect the new node directly to the * parent. */ - level += ASSOC_ARRAY_LEVEL_STEP; + if (shortcut->back_pointer) + level += ASSOC_ARRAY_LEVEL_STEP; if (diff > level) { pr_devel("pre-shortcut %d...%d\n", level, diff); keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE); @@ -865,9 +938,11 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, memcpy(new_s0->index_key, shortcut->index_key, flex_array_size(new_s0, index_key, keylen)); - blank = ULONG_MAX << (diff & ASSOC_ARRAY_KEY_CHUNK_MASK); - pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, diff, blank); - new_s0->index_key[keylen - 1] &= ~blank; + if (diff & ASSOC_ARRAY_KEY_CHUNK_MASK) { + blank = ULONG_MAX << (diff & ASSOC_ARRAY_KEY_CHUNK_MASK); + pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, diff, blank); + new_s0->index_key[keylen - 1] &= ~blank; + } } else { pr_devel("no pre-shortcut\n"); edit->set[0].to = assoc_array_node_to_ptr(new_n0); @@ -1055,6 +1130,92 @@ static int assoc_array_delete_collapse_iterator(const void *leaf, return 0; } +/* + * Deleting a leaf from node leaves its subtree with FAN_OUT or fewer leaves: + * collapse the subtree into one new node, together with as much of the tree + * above it as also fits. Returns false if the new node cannot be allocated. + */ +static bool assoc_array_delete_collapse(struct assoc_array_edit *edit, + struct assoc_array_node *node) +{ + struct assoc_array_delete_collapse_context collapse; + struct assoc_array_node *parent, *grandparent, *new_n0; + struct assoc_array_ptr *ptr; + bool has_meta; + int i; + + /* First of all, we need to know if this node has metadata so + * that we don't try collapsing if all the leaves are already + * here. + */ + has_meta = false; + for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) { + ptr = node->slots[i]; + if (assoc_array_ptr_is_meta(ptr)) { + has_meta = true; + break; + } + } + + pr_devel("leaves: %ld [m=%d]\n", + node->nr_leaves_on_branch - 1, has_meta); + + /* Look further up the tree to see if we can collapse this node + * into a more proximal node too. + */ + for (parent = node;; parent = grandparent) { + pr_devel("collapse subtree: %ld\n", parent->nr_leaves_on_branch); + + ptr = parent->back_pointer; + if (ptr && assoc_array_ptr_is_shortcut(ptr)) + ptr = assoc_array_ptr_to_shortcut(ptr)->back_pointer; + if (!ptr) + break; + + grandparent = assoc_array_ptr_to_node(ptr); + if (grandparent->nr_leaves_on_branch > ASSOC_ARRAY_FAN_OUT + 1) + break; + } + + /* There's no point collapsing if the original node has no meta + * pointers to discard and if we didn't merge into one of that + * node's ancestry. + */ + if (has_meta || parent != node) { + node = parent; + + /* Create a new node to collapse into */ + new_n0 = kzalloc_obj(struct assoc_array_node); + if (!new_n0) + return false; + edit->new_meta[0] = assoc_array_node_to_ptr(new_n0); + + new_n0->back_pointer = node->back_pointer; + new_n0->parent_slot = node->parent_slot; + new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch; + edit->adjust_count_on = new_n0; + + collapse.node = new_n0; + collapse.skip_leaf = assoc_array_ptr_to_leaf(edit->dead_leaf); + collapse.slot = 0; + assoc_array_subtree_iterate(assoc_array_node_to_ptr(node), + node->back_pointer, + assoc_array_delete_collapse_iterator, + &collapse); + pr_devel("collapsed %d,%lu\n", collapse.slot, new_n0->nr_leaves_on_branch); + BUG_ON(collapse.slot != new_n0->nr_leaves_on_branch - 1); + + BUG_ON(node->back_pointer && + assoc_array_ptr_is_leaf(node->back_pointer)); + edit->set[1].ptr = assoc_array_parent_link(edit->array, + node->back_pointer, + node->parent_slot); + edit->set[1].to = assoc_array_node_to_ptr(new_n0); + edit->excised_subtree = assoc_array_node_to_ptr(node); + } + return true; +} + /** * assoc_array_delete - Script deletion of an object from an associative array * @array: The array to search. @@ -1078,13 +1239,11 @@ struct assoc_array_edit *assoc_array_delete(struct assoc_array *array, const struct assoc_array_ops *ops, const void *index_key) { - struct assoc_array_delete_collapse_context collapse; struct assoc_array_walk_result result; - struct assoc_array_node *node, *new_n0; + struct assoc_array_node *node; struct assoc_array_edit *edit; struct assoc_array_ptr *ptr; - bool has_meta; - int slot, i; + int slot; pr_devel("-->%s()\n", __func__); @@ -1095,8 +1254,10 @@ struct assoc_array_edit *assoc_array_delete(struct assoc_array *array, edit->ops = ops; edit->adjust_count_by = -1; - switch (assoc_array_walk(array, ops, index_key, &result)) { - case assoc_array_walk_found_terminal_node: + node = NULL; + slot = ASSOC_ARRAY_FAN_OUT; + if (assoc_array_walk(array, ops, index_key, &result) == + assoc_array_walk_found_terminal_node) { /* We found a node that should contain the leaf we've been * asked to remove - *if* it's in the tree. */ @@ -1109,18 +1270,15 @@ struct assoc_array_edit *assoc_array_delete(struct assoc_array *array, assoc_array_ptr_is_leaf(ptr) && ops->compare_object(assoc_array_ptr_to_leaf(ptr), index_key)) - goto found_leaf; + break; } - fallthrough; - case assoc_array_walk_tree_empty: - case assoc_array_walk_found_wrong_shortcut: - default: + } + if (slot == ASSOC_ARRAY_FAN_OUT) { assoc_array_cancel_edit(edit); pr_devel("not found\n"); return NULL; } -found_leaf: BUG_ON(array->nr_leaves_on_tree <= 0); /* In the simplest form of deletion we just clear the slot and release @@ -1153,103 +1311,15 @@ found_leaf: * We could also try and collapse in partially filled subtrees to take * up space in this node. */ - if (node->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1) { - struct assoc_array_node *parent, *grandparent; - struct assoc_array_ptr *ptr; - - /* First of all, we need to know if this node has metadata so - * that we don't try collapsing if all the leaves are already - * here. - */ - has_meta = false; - for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) { - ptr = node->slots[i]; - if (assoc_array_ptr_is_meta(ptr)) { - has_meta = true; - break; - } - } - - pr_devel("leaves: %ld [m=%d]\n", - node->nr_leaves_on_branch - 1, has_meta); - - /* Look further up the tree to see if we can collapse this node - * into a more proximal node too. - */ - parent = node; - collapse_up: - pr_devel("collapse subtree: %ld\n", parent->nr_leaves_on_branch); - - ptr = parent->back_pointer; - if (!ptr) - goto do_collapse; - if (assoc_array_ptr_is_shortcut(ptr)) { - struct assoc_array_shortcut *s = assoc_array_ptr_to_shortcut(ptr); - ptr = s->back_pointer; - if (!ptr) - goto do_collapse; - } - - grandparent = assoc_array_ptr_to_node(ptr); - if (grandparent->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1) { - parent = grandparent; - goto collapse_up; - } - - do_collapse: - /* There's no point collapsing if the original node has no meta - * pointers to discard and if we didn't merge into one of that - * node's ancestry. - */ - if (has_meta || parent != node) { - node = parent; - - /* Create a new node to collapse into */ - new_n0 = kzalloc_obj(struct assoc_array_node); - if (!new_n0) - goto enomem; - edit->new_meta[0] = assoc_array_node_to_ptr(new_n0); - - new_n0->back_pointer = node->back_pointer; - new_n0->parent_slot = node->parent_slot; - new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch; - edit->adjust_count_on = new_n0; - - collapse.node = new_n0; - collapse.skip_leaf = assoc_array_ptr_to_leaf(edit->dead_leaf); - collapse.slot = 0; - assoc_array_subtree_iterate(assoc_array_node_to_ptr(node), - node->back_pointer, - assoc_array_delete_collapse_iterator, - &collapse); - pr_devel("collapsed %d,%lu\n", collapse.slot, new_n0->nr_leaves_on_branch); - BUG_ON(collapse.slot != new_n0->nr_leaves_on_branch - 1); - - if (!node->back_pointer) { - edit->set[1].ptr = &array->root; - } else if (assoc_array_ptr_is_leaf(node->back_pointer)) { - BUG(); - } else if (assoc_array_ptr_is_node(node->back_pointer)) { - struct assoc_array_node *p = - assoc_array_ptr_to_node(node->back_pointer); - edit->set[1].ptr = &p->slots[node->parent_slot]; - } else if (assoc_array_ptr_is_shortcut(node->back_pointer)) { - struct assoc_array_shortcut *s = - assoc_array_ptr_to_shortcut(node->back_pointer); - edit->set[1].ptr = &s->next_node; - } - edit->set[1].to = assoc_array_node_to_ptr(new_n0); - edit->excised_subtree = assoc_array_node_to_ptr(node); - } + if (node->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1 && + !assoc_array_delete_collapse(edit, node)) { + /* Clean up after an out of memory error */ + pr_devel("enomem\n"); + assoc_array_cancel_edit(edit); + return ERR_PTR(-ENOMEM); } return edit; - -enomem: - /* Clean up after an out of memory error */ - pr_devel("enomem\n"); - assoc_array_cancel_edit(edit); - return ERR_PTR(-ENOMEM); } /** @@ -1425,115 +1495,115 @@ void assoc_array_cancel_edit(struct assoc_array_edit *edit) kfree(edit); } -/** - * assoc_array_gc - Garbage collect an associative array. - * @array: The array to clean. - * @ops: The operations to use. - * @iterator: A callback function to pass judgement on each object. - * @iterator_data: Private data for the callback function. - * - * Collect garbage from an associative array and pack down the internal tree to - * save memory. - * - * The iterator function is asked to pass judgement upon each object in the - * array. If it returns false, the object is discard and if it returns true, - * the object is kept. If it returns true, it must increment the object's - * usage count (or whatever it needs to do to retain it) before returning. - * - * This function returns 0 if successful or -ENOMEM if out of memory. In the - * latter case, the array is not changed. - * - * The caller should lock against other modifications and must continue to hold - * the lock until assoc_array_apply_edit() has been called. - * - * Accesses to the tree may take place concurrently with this function, - * provided they hold the RCU read lock. +/* + * Where assoc_array_gc() has got to. It copies the tree depth first without + * recursion: on reaching a subtree it copies it and carries on there, and on + * finishing a node it climbs back up the old tree to the node above. */ -int assoc_array_gc(struct assoc_array *array, - const struct assoc_array_ops *ops, - bool (*iterator)(void *object, void *iterator_data), - void *iterator_data) +struct assoc_array_gc_walk { + struct assoc_array_node *node; /* The node being copied */ + struct assoc_array_node *new_n; /* Its copy */ + int slot; /* The next slot of node to look at */ + struct assoc_array_ptr *new_root; /* The root of the copy */ + unsigned long nr_leaves_on_tree; + bool (*iterator)(void *object, void *iterator_data); + void *iterator_data; +}; + +/* + * Copy the node at cursor, and the shortcut in front of it if there is one, + * into the new tree at *link, below the copy of the node being worked on, + * and work on that node next. + */ +static bool assoc_array_gc_copy(struct assoc_array_gc_walk *w, + struct assoc_array_ptr *cursor, + struct assoc_array_ptr **link) { struct assoc_array_shortcut *shortcut, *new_s; - struct assoc_array_node *node, *new_n; - struct assoc_array_edit *edit; - struct assoc_array_ptr *cursor, *ptr; - struct assoc_array_ptr *new_root, *new_parent, **new_ptr_pp; - unsigned long nr_leaves_on_tree; - bool retained; - int keylen, slot, nr_free, next_slot, i; + struct assoc_array_node *new_n; + struct assoc_array_ptr *new_parent; + int keylen; - pr_devel("-->%s()\n", __func__); + new_parent = w->new_n ? assoc_array_node_to_ptr(w->new_n) : NULL; - if (!array->root) - return 0; - - edit = kzalloc_obj(struct assoc_array_edit); - if (!edit) - return -ENOMEM; - edit->array = array; - edit->ops = ops; - edit->ops_for_excised_subtree = ops; - edit->set[0].ptr = &array->root; - edit->excised_subtree = array->root; - - new_root = new_parent = NULL; - new_ptr_pp = &new_root; - cursor = array->root; - -descend: - /* If this point is a shortcut, then we need to duplicate it and - * advance the target cursor. + /* The node at this position is duplicated, and so is the shortcut in + * front of it if there is one. The node is allocated first: the copy + * of the shortcut starts out pointing at the old node, and must not be + * in the new tree if the node cannot be allocated, or the cleanup + * would follow it into the old tree. */ + new_n = kzalloc_obj(struct assoc_array_node); + if (!new_n) + return false; + if (assoc_array_ptr_is_shortcut(cursor)) { shortcut = assoc_array_ptr_to_shortcut(cursor); keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; new_s = kmalloc_flex(*new_s, index_key, keylen); - if (!new_s) - goto enomem; + if (!new_s) { + kfree(new_n); + return false; + } pr_devel("dup shortcut %p -> %p\n", shortcut, new_s); memcpy(new_s, shortcut, struct_size(new_s, index_key, keylen)); new_s->back_pointer = new_parent; new_s->parent_slot = shortcut->parent_slot; - *new_ptr_pp = new_parent = assoc_array_shortcut_to_ptr(new_s); - new_ptr_pp = &new_s->next_node; + *link = new_parent = assoc_array_shortcut_to_ptr(new_s); + link = &new_s->next_node; cursor = shortcut->next_node; } - /* Duplicate the node at this position */ - node = assoc_array_ptr_to_node(cursor); - new_n = kzalloc_obj(struct assoc_array_node); - if (!new_n) - goto enomem; - pr_devel("dup node %p -> %p\n", node, new_n); + w->node = assoc_array_ptr_to_node(cursor); + pr_devel("dup node %p -> %p\n", w->node, new_n); new_n->back_pointer = new_parent; - new_n->parent_slot = node->parent_slot; - *new_ptr_pp = new_parent = assoc_array_node_to_ptr(new_n); - new_ptr_pp = NULL; - slot = 0; + new_n->parent_slot = w->node->parent_slot; + *link = assoc_array_node_to_ptr(new_n); + w->new_n = new_n; + w->slot = 0; + return true; +} + +/* + * Carry on through the slots of the node being copied, putting the leaves + * the iterator keeps into the copy, as far as the next pointer to a subtree. + * Returns that pointer, with w->slot at it, or NULL when the node is done. + */ +static struct assoc_array_ptr *assoc_array_gc_next(struct assoc_array_gc_walk *w) +{ + struct assoc_array_ptr *ptr; -continue_node: /* Filter across any leaves and gc any subtrees */ - for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) { - ptr = node->slots[slot]; + for (; w->slot < ASSOC_ARRAY_FAN_OUT; w->slot++) { + ptr = w->node->slots[w->slot]; if (!ptr) continue; if (assoc_array_ptr_is_leaf(ptr)) { - if (iterator(assoc_array_ptr_to_leaf(ptr), - iterator_data)) + if (w->iterator(assoc_array_ptr_to_leaf(ptr), + w->iterator_data)) /* The iterator will have done any reference * counting on the object for us. */ - new_n->slots[slot] = ptr; + w->new_n->slots[w->slot] = ptr; continue; } - new_ptr_pp = &new_n->slots[slot]; - cursor = ptr; - goto descend; + return ptr; } + return NULL; +} + +/* + * Fold the children of a finished copy into it where their leaves fit in + * its free slots, and count its leaves. Returns the number of free slots + * left. + */ +static int assoc_array_gc_fold(struct assoc_array_node *new_n) +{ + struct assoc_array_ptr *ptr; + bool retained; + int slot, nr_free, next_slot, i; retry_compress: pr_devel("-- compress node %p --\n", new_n); @@ -1612,8 +1682,24 @@ retry_compress: goto retry_compress; } pr_devel("after: %lu\n", new_n->nr_leaves_on_branch); + return nr_free; +} - nr_leaves_on_tree = new_n->nr_leaves_on_branch; +/* + * The copy of the node being worked on is finished: fold its children into + * it, then remove it if all it holds is a shortcut, or remove the shortcut + * in front of it if it is small. Returns what is then the copy's parent, or + * NULL if it is at the root, when the whole tree has been copied. + */ +static struct assoc_array_ptr *assoc_array_gc_pack(struct assoc_array_gc_walk *w) +{ + struct assoc_array_shortcut *new_s; + struct assoc_array_node *new_n = w->new_n; + struct assoc_array_ptr *ptr, *new_parent; + int nr_free, slot; + + nr_free = assoc_array_gc_fold(new_n); + w->nr_leaves_on_tree = new_n->nr_leaves_on_branch; /* Excise this node if it is singly occupied by a shortcut */ if (nr_free == ASSOC_ARRAY_FAN_OUT - 1) { @@ -1631,8 +1717,8 @@ retry_compress: if (!new_parent) { new_s->back_pointer = NULL; new_s->parent_slot = 0; - new_root = ptr; - goto gc_complete; + w->new_root = ptr; + return NULL; } if (assoc_array_ptr_is_shortcut(new_parent)) { @@ -1648,16 +1734,15 @@ retry_compress: if (!new_parent) { new_s->back_pointer = NULL; new_s->parent_slot = 0; - new_root = ptr; - goto gc_complete; + w->new_root = ptr; + return NULL; } } new_s->back_pointer = new_parent; new_s->parent_slot = slot; - new_n = assoc_array_ptr_to_node(new_parent); - new_n->slots[slot] = ptr; - goto ascend_old_tree; + assoc_array_ptr_to_node(new_parent)->slots[slot] = ptr; + return new_parent; } } @@ -1666,59 +1751,135 @@ retry_compress: */ ptr = new_n->back_pointer; if (!ptr) - goto gc_complete; + return NULL; - if (assoc_array_ptr_is_shortcut(ptr)) { - new_s = assoc_array_ptr_to_shortcut(ptr); - new_parent = new_s->back_pointer; - slot = new_s->parent_slot; + if (!assoc_array_ptr_is_shortcut(ptr)) + return ptr; - if (new_n->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT) { - struct assoc_array_node *n; + new_s = assoc_array_ptr_to_shortcut(ptr); + new_parent = new_s->back_pointer; + slot = new_s->parent_slot; - pr_devel("excise shortcut\n"); - new_n->back_pointer = new_parent; - new_n->parent_slot = slot; - kfree(new_s); - if (!new_parent) { - new_root = assoc_array_node_to_ptr(new_n); - goto gc_complete; - } - - n = assoc_array_ptr_to_node(new_parent); - n->slots[slot] = assoc_array_node_to_ptr(new_n); + if (new_n->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT) { + pr_devel("excise shortcut\n"); + new_n->back_pointer = new_parent; + new_n->parent_slot = slot; + kfree(new_s); + if (!new_parent) { + w->new_root = assoc_array_node_to_ptr(new_n); + return NULL; } - } else { - new_parent = ptr; - } - new_n = assoc_array_ptr_to_node(new_parent); -ascend_old_tree: - ptr = node->back_pointer; + assoc_array_ptr_to_node(new_parent)->slots[slot] = + assoc_array_node_to_ptr(new_n); + } + return new_parent; +} + +/* + * Go back up from the node just finished to the node above it, whose copy is + * parent, and on to that node's next slot. + */ +static void assoc_array_gc_ascend(struct assoc_array_gc_walk *w, + struct assoc_array_ptr *parent) +{ + struct assoc_array_shortcut *shortcut; + struct assoc_array_ptr *ptr, *cursor; + int slot; + + w->new_n = assoc_array_ptr_to_node(parent); + + ptr = w->node->back_pointer; if (assoc_array_ptr_is_shortcut(ptr)) { shortcut = assoc_array_ptr_to_shortcut(ptr); slot = shortcut->parent_slot; cursor = shortcut->back_pointer; - if (!cursor) - goto gc_complete; } else { - slot = node->parent_slot; + slot = w->node->parent_slot; cursor = ptr; } BUG_ON(!cursor); - node = assoc_array_ptr_to_node(cursor); - slot++; - goto continue_node; + w->node = assoc_array_ptr_to_node(cursor); + w->slot = slot + 1; +} -gc_complete: - edit->set[0].to = new_root; +/** + * assoc_array_gc - Garbage collect an associative array. + * @array: The array to clean. + * @ops: The operations to use. + * @iterator: A callback function to pass judgement on each object. + * @iterator_data: Private data for the callback function. + * + * Collect garbage from an associative array and pack down the internal tree to + * save memory. + * + * The iterator function is asked to pass judgement upon each object in the + * array. If it returns false, the object is discard and if it returns true, + * the object is kept. If it returns true, it must increment the object's + * usage count (or whatever it needs to do to retain it) before returning. + * + * This function returns 0 if successful or -ENOMEM if out of memory. In the + * latter case, the array is not changed. + * + * The caller should lock against other modifications and must continue to hold + * the lock until assoc_array_apply_edit() has been called. + * + * Accesses to the tree may take place concurrently with this function, + * provided they hold the RCU read lock. + */ +int assoc_array_gc(struct assoc_array *array, + const struct assoc_array_ops *ops, + bool (*iterator)(void *object, void *iterator_data), + void *iterator_data) +{ + struct assoc_array_gc_walk w = { + .iterator = iterator, + .iterator_data = iterator_data, + }; + struct assoc_array_edit *edit; + struct assoc_array_ptr *ptr, *parent, **link; + + pr_devel("-->%s()\n", __func__); + + if (!array->root) + return 0; + + edit = kzalloc_obj(struct assoc_array_edit); + if (!edit) + return -ENOMEM; + edit->array = array; + edit->ops = ops; + edit->ops_for_excised_subtree = ops; + edit->set[0].ptr = &array->root; + edit->excised_subtree = array->root; + + ptr = array->root; + link = &w.new_root; + for (;;) { + if (!assoc_array_gc_copy(&w, ptr, link)) + goto enomem; + + /* Find the next subtree to copy: further on in this node, or, + * when it is done, in the nodes above it. + */ + while (!(ptr = assoc_array_gc_next(&w))) { + parent = assoc_array_gc_pack(&w); + if (!parent) + goto copied; + assoc_array_gc_ascend(&w, parent); + } + link = &w.new_n->slots[w.slot]; + } + +copied: + edit->set[0].to = w.new_root; assoc_array_apply_edit(edit); - array->nr_leaves_on_tree = nr_leaves_on_tree; + array->nr_leaves_on_tree = w.nr_leaves_on_tree; return 0; enomem: pr_devel("enomem\n"); - assoc_array_destroy_subtree(new_root, edit->ops); + assoc_array_destroy_subtree(w.new_root, edit->ops); kfree(edit); return -ENOMEM; }