diff --git a/src/liballoc/collections/btree/map.rs b/src/liballoc/collections/btree/map.rs index e70f881bc3d7e..9e6482b7346fb 100644 --- a/src/liballoc/collections/btree/map.rs +++ b/src/liballoc/collections/btree/map.rs @@ -156,10 +156,10 @@ impl Clone for BTreeMap { let mut in_edge = leaf.first_edge(); while let Ok(kv) = in_edge.right_kv() { - let (k, v) = kv.into_kv(); + let (k, v) = unsafe { kv.into_kv() }; in_edge = kv.right_edge(); - out_node.push(k.clone(), v.clone()); + unsafe { out_node.push(k.clone(), v.clone()) }; out_tree.length += 1; } } @@ -170,10 +170,10 @@ impl Clone for BTreeMap { let mut out_tree = clone_subtree(internal.first_edge().descend()); { - let mut out_node = out_tree.root.push_level(); + let mut out_node = unsafe { out_tree.root.push_level() }; let mut in_edge = internal.first_edge(); while let Ok(kv) = in_edge.right_kv() { - let (k, v) = kv.into_kv(); + let (k, v) = unsafe { kv.into_kv() }; in_edge = kv.right_edge(); let k = (*k).clone(); @@ -189,7 +189,7 @@ impl Clone for BTreeMap { (root, length) }; - out_node.push(k, v, subroot); + unsafe { out_node.push(k, v, subroot) }; out_tree.length += 1 + sublength; } } @@ -218,7 +218,7 @@ where fn get(&self, key: &Q) -> Option<&K> { match search::search_tree(self.root.as_ref(), key) { - Found(handle) => Some(handle.into_kv().0), + Found(handle) => Some(unsafe { handle.into_kv().0 }), GoDown(_) => None, } } @@ -237,7 +237,7 @@ where fn replace(&mut self, key: K) -> Option { self.ensure_root_is_owned(); match search::search_tree::, K, (), K>(self.root.as_mut(), &key) { - Found(handle) => Some(mem::replace(handle.into_kv_mut().0, key)), + Found(handle) => Some(mem::replace(unsafe { handle.into_kv_mut().0 }, key)), GoDown(handle) => { VacantEntry { key, handle, length: &mut self.length, _marker: PhantomData } .insert(()); @@ -536,7 +536,7 @@ impl BTreeMap { Q: Ord, { match search::search_tree(self.root.as_ref(), key) { - Found(handle) => Some(handle.into_kv().1), + Found(handle) => Some(unsafe { handle.into_kv().1 }), GoDown(_) => None, } } @@ -563,7 +563,7 @@ impl BTreeMap { Q: Ord, { match search::search_tree(self.root.as_ref(), k) { - Found(handle) => Some(handle.into_kv()), + Found(handle) => Some(unsafe { handle.into_kv() }), GoDown(_) => None, } } @@ -592,7 +592,8 @@ impl BTreeMap { K: Borrow, { let front = first_leaf_edge(self.root.as_ref()); - front.right_kv().ok().map(Handle::into_kv) + let handle = front.right_kv().ok()?; + Some(unsafe { handle.into_kv() }) } /// Returns the first entry in the map for in-place manipulation. @@ -623,7 +624,7 @@ impl BTreeMap { match self.length { 0 => None, _ => Some(OccupiedEntry { - handle: self.root.as_mut().first_kv(), + handle: unsafe { self.root.as_mut().first_kv() }, length: &mut self.length, _marker: PhantomData, }), @@ -653,7 +654,8 @@ impl BTreeMap { K: Borrow, { let back = last_leaf_edge(self.root.as_ref()); - back.left_kv().ok().map(Handle::into_kv) + let handle = back.left_kv().ok()?; + unsafe { Some(handle.into_kv()) } } /// Returns the last entry in the map for in-place manipulation. @@ -684,7 +686,7 @@ impl BTreeMap { match self.length { 0 => None, _ => Some(OccupiedEntry { - handle: self.root.as_mut().last_kv(), + handle: unsafe { self.root.as_mut().last_kv() }, length: &mut self.length, _marker: PhantomData, }), @@ -744,7 +746,7 @@ impl BTreeMap { Q: Ord, { match search::search_tree(self.root.as_mut(), key) { - Found(handle) => Some(handle.into_kv_mut().1), + Found(handle) => Some(unsafe { handle.into_kv_mut().1 }), GoDown(_) => None, } } @@ -995,7 +997,7 @@ impl BTreeMap { for (key, value) in iter { // Try to push key-value pair into the current leaf node. if cur_node.len() < node::CAPACITY { - cur_node.push(key, value); + unsafe { cur_node.push(key, value) }; } else { // No space left, go up and push there. let mut open_node; @@ -1015,7 +1017,7 @@ impl BTreeMap { } Err(node) => { // We are at the top, create a new root node and push there. - open_node = node.into_root_mut().push_level(); + open_node = unsafe { node.into_root_mut().push_level() }; break; } } @@ -1025,9 +1027,9 @@ impl BTreeMap { let tree_height = open_node.height() - 1; let mut right_tree = node::Root::new_leaf(); for _ in 0..tree_height { - right_tree.push_level(); + unsafe { right_tree.push_level() }; } - open_node.push(key, value, right_tree); + unsafe { open_node.push(key, value, right_tree) }; // Go down to the right-most leaf again. cur_node = last_leaf_edge(open_node.forget_type()).into_node(); @@ -1050,7 +1052,7 @@ impl BTreeMap { Ok(left) => left, Err(_) => unreachable!(), }; - last_kv.bulk_steal_left(node::MIN_LEN - right_child_len); + unsafe { last_kv.bulk_steal_left(node::MIN_LEN - right_child_len) }; last_edge = last_kv.right_edge(); } @@ -1102,7 +1104,7 @@ impl BTreeMap { let mut right = Self::new(); right.root = node::Root::new_leaf(); for _ in 0..(self.root.as_ref().height()) { - right.root.push_level(); + unsafe { right.root.push_level() }; } { @@ -1116,7 +1118,7 @@ impl BTreeMap { GoDown(handle) => handle, }; - split_edge.move_suffix(&mut right_node); + unsafe { split_edge.move_suffix(&mut right_node) }; match (split_edge.force(), right_node.force()) { (Internal(edge), Internal(node)) => { @@ -1186,7 +1188,7 @@ impl BTreeMap { break; } } - self.root.pop_level(); + unsafe { self.root.pop_level() }; } } @@ -1197,15 +1199,15 @@ impl BTreeMap { let mut cur_node = self.root.as_mut(); while let Internal(node) = cur_node.force() { - let mut last_kv = node.last_kv(); + let mut last_kv = unsafe { node.last_kv() }; if last_kv.can_merge() { - cur_node = last_kv.merge().descend(); + cur_node = unsafe { last_kv.merge() }.descend(); } else { let right_len = last_kv.reborrow().right_edge().descend().len(); // `MINLEN + 1` to avoid readjust if merge happens on the next level. if right_len < node::MIN_LEN + 1 { - last_kv.bulk_steal_left(node::MIN_LEN + 1 - right_len); + unsafe { last_kv.bulk_steal_left(node::MIN_LEN + 1 - right_len) }; } cur_node = last_kv.right_edge().descend(); } @@ -1223,14 +1225,14 @@ impl BTreeMap { let mut cur_node = self.root.as_mut(); while let Internal(node) = cur_node.force() { - let mut first_kv = node.first_kv(); + let mut first_kv = unsafe { node.first_kv() }; if first_kv.can_merge() { - cur_node = first_kv.merge().descend(); + cur_node = unsafe { first_kv.merge().descend() }; } else { let left_len = first_kv.reborrow().left_edge().descend().len(); if left_len < node::MIN_LEN + 1 { - first_kv.bulk_steal_right(node::MIN_LEN + 1 - left_len); + unsafe { first_kv.bulk_steal_right(node::MIN_LEN + 1 - left_len) }; } cur_node = first_kv.left_edge().descend(); } @@ -2387,7 +2389,7 @@ impl<'a, K: Ord, V> VacantEntry<'a, K, V> { let mut ins_edge; let mut cur_parent = match self.handle.insert(self.key, value) { - (Fit(handle), _) => return handle.into_kv_mut().1, + (Fit(handle), _) => return unsafe { handle.into_kv_mut().1 }, (Split(left, k, v, right), ptr) => { ins_k = k; ins_v = v; @@ -2409,7 +2411,7 @@ impl<'a, K: Ord, V> VacantEntry<'a, K, V> { } }, Err(root) => { - root.push_level().push(ins_k, ins_v, ins_edge); + unsafe { root.push_level().push(ins_k, ins_v, ins_edge) }; return unsafe { &mut *out_ptr }; } } @@ -2431,7 +2433,7 @@ impl<'a, K: Ord, V> OccupiedEntry<'a, K, V> { /// ``` #[stable(feature = "map_entry_keys", since = "1.10.0")] pub fn key(&self) -> &K { - self.handle.reborrow().into_kv().0 + unsafe { self.handle.reborrow().into_kv().0 } } /// Take ownership of the key and value from the map. @@ -2475,7 +2477,7 @@ impl<'a, K: Ord, V> OccupiedEntry<'a, K, V> { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn get(&self) -> &V { - self.handle.reborrow().into_kv().1 + unsafe { self.handle.reborrow().into_kv().1 } } /// Gets a mutable reference to the value in the entry. @@ -2506,7 +2508,7 @@ impl<'a, K: Ord, V> OccupiedEntry<'a, K, V> { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn get_mut(&mut self) -> &mut V { - self.handle.kv_mut().1 + unsafe { self.handle.kv_mut().1 } } /// Converts the entry into a mutable reference to its value. @@ -2532,7 +2534,7 @@ impl<'a, K: Ord, V> OccupiedEntry<'a, K, V> { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn into_mut(self) -> &'a mut V { - self.handle.into_kv_mut().1 + unsafe { self.handle.into_kv_mut().1 } } /// Sets the value of the entry with the `OccupiedEntry`'s key, @@ -2584,17 +2586,17 @@ impl<'a, K: Ord, V> OccupiedEntry<'a, K, V> { let (small_leaf, old_key, old_val) = match self.handle.force() { Leaf(leaf) => { - let (hole, old_key, old_val) = leaf.remove(); + let (hole, old_key, old_val) = unsafe { leaf.remove() }; (hole.into_node(), old_key, old_val) } Internal(mut internal) => { - let key_loc = internal.kv_mut().0 as *mut K; - let val_loc = internal.kv_mut().1 as *mut V; + let key_loc = unsafe { internal.kv_mut().0 as *mut K }; + let val_loc = unsafe { internal.kv_mut().1 as *mut V }; let to_remove = first_leaf_edge(internal.right_edge().descend()).right_kv().ok(); let to_remove = unsafe { unwrap_unchecked(to_remove) }; - let (hole, key, val) = to_remove.remove(); + let (hole, key, val) = unsafe { to_remove.remove() }; let old_key = unsafe { mem::replace(&mut *key_loc, key) }; let old_val = unsafe { mem::replace(&mut *val_loc, val) }; @@ -2612,7 +2614,7 @@ impl<'a, K: Ord, V> OccupiedEntry<'a, K, V> { Merged(parent) => { if parent.len() == 0 { // We must be at the root - parent.into_root_mut().pop_level(); + unsafe { parent.into_root_mut().pop_level() }; break; } else { cur_node = parent.forget_type(); @@ -2653,7 +2655,7 @@ fn handle_underfull_node( }; if handle.can_merge() { - Merged(handle.merge().into_node()) + Merged(unsafe { handle.merge() }.into_node()) } else { if is_left { handle.steal_left(); diff --git a/src/liballoc/collections/btree/node.rs b/src/liballoc/collections/btree/node.rs index d9cdebb4f7354..75ca4679e6ac3 100644 --- a/src/liballoc/collections/btree/node.rs +++ b/src/liballoc/collections/btree/node.rs @@ -239,10 +239,10 @@ impl Root { /// Adds a new internal node with a single edge, pointing to the previous root, and make that /// new node the root. This increases the height by 1 and is the opposite of `pop_level`. - pub fn push_level(&mut self) -> NodeRef, K, V, marker::Internal> { + pub unsafe fn push_level(&mut self) -> NodeRef, K, V, marker::Internal> { debug_assert!(!self.is_shared_root()); - let mut new_node = Box::new(unsafe { InternalNode::new() }); - new_node.edges[0].write(unsafe { BoxedNode::from_ptr(self.node.as_ptr()) }); + let mut new_node = Box::new(InternalNode::new()); + new_node.edges[0].write(BoxedNode::from_ptr(self.node.as_ptr())); self.node = BoxedNode::from_internal(new_node); self.height += 1; @@ -254,9 +254,7 @@ impl Root { _marker: PhantomData, }; - unsafe { - ret.reborrow_mut().first_edge().correct_parent_link(); - } + ret.reborrow_mut().first_edge().correct_parent_link(); ret } @@ -265,24 +263,18 @@ impl Root { /// the tree consists only of a leaf node. As it is intended only to be called when the root /// has only one edge, no cleanup is done on any of the other children are elements of the root. /// This decreases the height by 1 and is the opposite of `push_level`. - pub fn pop_level(&mut self) { + pub unsafe fn pop_level(&mut self) { debug_assert!(self.height > 0); let top = self.node.ptr; - self.node = unsafe { - BoxedNode::from_ptr( - self.as_mut().cast_unchecked::().first_edge().descend().node, - ) - }; + self.node = BoxedNode::from_ptr( + self.as_mut().cast_unchecked::().first_edge().descend().node, + ); self.height -= 1; - unsafe { - (*self.as_mut().as_leaf_mut()).parent = ptr::null(); - } + (*self.as_mut().as_leaf_mut()).parent = ptr::null(); - unsafe { - Global.dealloc(NonNull::from(top).cast(), Layout::new::>()); - } + Global.dealloc(NonNull::from(top).cast(), Layout::new::>()); } } @@ -392,7 +384,7 @@ impl NodeRef { /// Borrows a view into the values stored in the node. /// The caller must ensure that the node is not the shared root. - fn vals(&self) -> &[V] { + unsafe fn vals(&self) -> &[V] { self.reborrow().into_val_slice() } @@ -433,13 +425,13 @@ impl NodeRef { } /// Note that `self` must be nonempty. - pub fn first_kv(self) -> Handle { + pub unsafe fn first_kv(self) -> Handle { debug_assert!(self.len() > 0); Handle::new_kv(self, 0) } /// Note that `self` must be nonempty. - pub fn last_kv(self) -> Handle { + pub unsafe fn last_kv(self) -> Handle { let len = self.len(); debug_assert!(len > 0); Handle::new_kv(self, len - 1) @@ -528,16 +520,16 @@ impl<'a, K: 'a, V: 'a, Type> NodeRef, K, V, Type> { } /// The caller must ensure that the node is not the shared root. - fn into_val_slice(self) -> &'a [V] { + unsafe fn into_val_slice(self) -> &'a [V] { debug_assert!(!self.is_shared_root()); // We cannot be the shared root, so `as_leaf` is okay. - unsafe { slice::from_raw_parts(MaybeUninit::first_ptr(&self.as_leaf().vals), self.len()) } + slice::from_raw_parts(MaybeUninit::first_ptr(&self.as_leaf().vals), self.len()) } /// The caller must ensure that the node is not the shared root. - fn into_slices(self) -> (&'a [K], &'a [V]) { - let k = unsafe { ptr::read(&self) }; - (unsafe { k.into_key_slice() }, self.into_val_slice()) + unsafe fn into_slices(self) -> (&'a [K], &'a [V]) { + let k = ptr::read(&self); + (k.into_key_slice(), self.into_val_slice()) } } @@ -549,84 +541,72 @@ impl<'a, K: 'a, V: 'a, Type> NodeRef, K, V, Type> { } /// The caller must ensure that the node is not the shared root. - fn into_key_slice_mut(mut self) -> &'a mut [K] { + unsafe fn into_key_slice_mut(mut self) -> &'a mut [K] { debug_assert!(!self.is_shared_root()); // We cannot be the shared root, so `as_leaf_mut` is okay. - unsafe { - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut (*self.as_leaf_mut()).keys), - self.len(), - ) - } + slice::from_raw_parts_mut( + MaybeUninit::first_ptr_mut(&mut (*self.as_leaf_mut()).keys), + self.len(), + ) } /// The caller must ensure that the node is not the shared root. - fn into_val_slice_mut(mut self) -> &'a mut [V] { + unsafe fn into_val_slice_mut(mut self) -> &'a mut [V] { debug_assert!(!self.is_shared_root()); - unsafe { - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut (*self.as_leaf_mut()).vals), - self.len(), - ) - } + slice::from_raw_parts_mut( + MaybeUninit::first_ptr_mut(&mut (*self.as_leaf_mut()).vals), + self.len(), + ) } /// The caller must ensure that the node is not the shared root. - fn into_slices_mut(mut self) -> (&'a mut [K], &'a mut [V]) { + unsafe fn into_slices_mut(mut self) -> (&'a mut [K], &'a mut [V]) { debug_assert!(!self.is_shared_root()); // We cannot use the getters here, because calling the second one // invalidates the reference returned by the first. // More precisely, it is the call to `len` that is the culprit, // because that creates a shared reference to the header, which *can* // overlap with the keys (and even the values, for ZST keys). - unsafe { - let len = self.len(); - let leaf = self.as_leaf_mut(); - let keys = - slice::from_raw_parts_mut(MaybeUninit::first_ptr_mut(&mut (*leaf).keys), len); - let vals = - slice::from_raw_parts_mut(MaybeUninit::first_ptr_mut(&mut (*leaf).vals), len); - (keys, vals) - } + let len = self.len(); + let leaf = self.as_leaf_mut(); + let keys = slice::from_raw_parts_mut(MaybeUninit::first_ptr_mut(&mut (*leaf).keys), len); + let vals = slice::from_raw_parts_mut(MaybeUninit::first_ptr_mut(&mut (*leaf).vals), len); + (keys, vals) } } impl<'a, K, V> NodeRef, K, V, marker::Leaf> { /// Adds a key/value pair the end of the node. - pub fn push(&mut self, key: K, val: V) { + pub unsafe fn push(&mut self, key: K, val: V) { // Necessary for correctness, but this is an internal module debug_assert!(self.len() < CAPACITY); debug_assert!(!self.is_shared_root()); let idx = self.len(); - unsafe { - ptr::write(self.keys_mut().get_unchecked_mut(idx), key); - ptr::write(self.vals_mut().get_unchecked_mut(idx), val); + ptr::write(self.keys_mut().get_unchecked_mut(idx), key); + ptr::write(self.vals_mut().get_unchecked_mut(idx), val); - (*self.as_leaf_mut()).len += 1; - } + (*self.as_leaf_mut()).len += 1; } /// Adds a key/value pair to the beginning of the node. - pub fn push_front(&mut self, key: K, val: V) { + pub unsafe fn push_front(&mut self, key: K, val: V) { // Necessary for correctness, but this is an internal module debug_assert!(self.len() < CAPACITY); debug_assert!(!self.is_shared_root()); - unsafe { - slice_insert(self.keys_mut(), 0, key); - slice_insert(self.vals_mut(), 0, val); + slice_insert(self.keys_mut(), 0, key); + slice_insert(self.vals_mut(), 0, val); - (*self.as_leaf_mut()).len += 1; - } + (*self.as_leaf_mut()).len += 1; } } impl<'a, K, V> NodeRef, K, V, marker::Internal> { /// Adds a key/value pair and an edge to go to the right of that pair to /// the end of the node. - pub fn push(&mut self, key: K, val: V, edge: Root) { + pub unsafe fn push(&mut self, key: K, val: V, edge: Root) { // Necessary for correctness, but this is an internal module debug_assert!(edge.height == self.height - 1); debug_assert!(self.len() < CAPACITY); @@ -634,15 +614,13 @@ impl<'a, K, V> NodeRef, K, V, marker::Internal> { let idx = self.len(); - unsafe { - ptr::write(self.keys_mut().get_unchecked_mut(idx), key); - ptr::write(self.vals_mut().get_unchecked_mut(idx), val); - self.as_internal_mut().edges.get_unchecked_mut(idx + 1).write(edge.node); + ptr::write(self.keys_mut().get_unchecked_mut(idx), key); + ptr::write(self.vals_mut().get_unchecked_mut(idx), val); + self.as_internal_mut().edges.get_unchecked_mut(idx + 1).write(edge.node); - (*self.as_leaf_mut()).len += 1; + (*self.as_leaf_mut()).len += 1; - Handle::new_edge(self.reborrow_mut(), idx + 1).correct_parent_link(); - } + Handle::new_edge(self.reborrow_mut(), idx + 1).correct_parent_link(); } fn correct_childrens_parent_links(&mut self, first: usize, after_last: usize) { @@ -658,96 +636,89 @@ impl<'a, K, V> NodeRef, K, V, marker::Internal> { /// Adds a key/value pair and an edge to go to the left of that pair to /// the beginning of the node. - pub fn push_front(&mut self, key: K, val: V, edge: Root) { + pub unsafe fn push_front(&mut self, key: K, val: V, edge: Root) { // Necessary for correctness, but this is an internal module debug_assert!(edge.height == self.height - 1); debug_assert!(self.len() < CAPACITY); debug_assert!(!self.is_shared_root()); - unsafe { - slice_insert(self.keys_mut(), 0, key); - slice_insert(self.vals_mut(), 0, val); - slice_insert( - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut self.as_internal_mut().edges), - self.len() + 1, - ), - 0, - edge.node, - ); + slice_insert(self.keys_mut(), 0, key); + slice_insert(self.vals_mut(), 0, val); + slice_insert( + slice::from_raw_parts_mut( + MaybeUninit::first_ptr_mut(&mut self.as_internal_mut().edges), + self.len() + 1, + ), + 0, + edge.node, + ); - (*self.as_leaf_mut()).len += 1; + (*self.as_leaf_mut()).len += 1; - self.correct_all_childrens_parent_links(); - } + self.correct_all_childrens_parent_links(); } } impl<'a, K, V> NodeRef, K, V, marker::LeafOrInternal> { /// Removes a key/value pair from the end of this node. If this is an internal node, /// also removes the edge that was to the right of that pair. - pub fn pop(&mut self) -> (K, V, Option>) { + pub unsafe fn pop(&mut self) -> (K, V, Option>) { // Necessary for correctness, but this is an internal module debug_assert!(self.len() > 0); let idx = self.len() - 1; - unsafe { - let key = ptr::read(self.keys().get_unchecked(idx)); - let val = ptr::read(self.vals().get_unchecked(idx)); - let edge = match self.reborrow_mut().force() { - ForceResult::Leaf(_) => None, - ForceResult::Internal(internal) => { - let edge = - ptr::read(internal.as_internal().edges.get_unchecked(idx + 1).as_ptr()); - let mut new_root = Root { node: edge, height: internal.height - 1 }; - (*new_root.as_mut().as_leaf_mut()).parent = ptr::null(); - Some(new_root) - } - }; + let key = ptr::read(self.keys().get_unchecked(idx)); + let val = ptr::read(self.vals().get_unchecked(idx)); + let edge = match self.reborrow_mut().force() { + ForceResult::Leaf(_) => None, + ForceResult::Internal(internal) => { + let edge = ptr::read(internal.as_internal().edges.get_unchecked(idx + 1).as_ptr()); + let mut new_root = Root { node: edge, height: internal.height - 1 }; + (*new_root.as_mut().as_leaf_mut()).parent = ptr::null(); + Some(new_root) + } + }; - (*self.as_leaf_mut()).len -= 1; - (key, val, edge) - } + (*self.as_leaf_mut()).len -= 1; + (key, val, edge) } /// Removes a key/value pair from the beginning of this node. If this is an internal node, /// also removes the edge that was to the left of that pair. - pub fn pop_front(&mut self) -> (K, V, Option>) { + pub unsafe fn pop_front(&mut self) -> (K, V, Option>) { // Necessary for correctness, but this is an internal module debug_assert!(self.len() > 0); let old_len = self.len(); - unsafe { - let key = slice_remove(self.keys_mut(), 0); - let val = slice_remove(self.vals_mut(), 0); - let edge = match self.reborrow_mut().force() { - ForceResult::Leaf(_) => None, - ForceResult::Internal(mut internal) => { - let edge = slice_remove( - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut internal.as_internal_mut().edges), - old_len + 1, - ), - 0, - ); - - let mut new_root = Root { node: edge, height: internal.height - 1 }; - (*new_root.as_mut().as_leaf_mut()).parent = ptr::null(); - - for i in 0..old_len { - Handle::new_edge(internal.reborrow_mut(), i).correct_parent_link(); - } - - Some(new_root) + let key = slice_remove(self.keys_mut(), 0); + let val = slice_remove(self.vals_mut(), 0); + let edge = match self.reborrow_mut().force() { + ForceResult::Leaf(_) => None, + ForceResult::Internal(mut internal) => { + let edge = slice_remove( + slice::from_raw_parts_mut( + MaybeUninit::first_ptr_mut(&mut internal.as_internal_mut().edges), + old_len + 1, + ), + 0, + ); + + let mut new_root = Root { node: edge, height: internal.height - 1 }; + (*new_root.as_mut().as_leaf_mut()).parent = ptr::null(); + + for i in 0..old_len { + Handle::new_edge(internal.reborrow_mut(), i).correct_parent_link(); } - }; - (*self.as_leaf_mut()).len -= 1; + Some(new_root) + } + }; - (key, val, edge) - } + (*self.as_leaf_mut()).len -= 1; + + (key, val, edge) } /// The caller must ensure that the node is not the shared root. @@ -892,19 +863,17 @@ impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Edge /// pair to fit. /// /// The returned pointer points to the inserted value. - fn insert_fit(&mut self, key: K, val: V) -> *mut V { + unsafe fn insert_fit(&mut self, key: K, val: V) -> *mut V { // Necessary for correctness, but in a private module debug_assert!(self.node.len() < CAPACITY); debug_assert!(!self.node.is_shared_root()); - unsafe { - slice_insert(self.node.keys_mut(), self.idx, key); - slice_insert(self.node.vals_mut(), self.idx, val); + slice_insert(self.node.keys_mut(), self.idx, key); + slice_insert(self.node.vals_mut(), self.idx, val); - (*self.node.as_leaf_mut()).len += 1; + (*self.node.as_leaf_mut()).len += 1; - self.node.vals_mut().get_unchecked_mut(self.idx) - } + self.node.vals_mut().get_unchecked_mut(self.idx) } /// Inserts a new key/value pair between the key/value pairs to the right and left of @@ -913,11 +882,11 @@ impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Edge /// The returned pointer points to the inserted value. pub fn insert(mut self, key: K, val: V) -> (InsertResult<'a, K, V, marker::Leaf>, *mut V) { if self.node.len() < CAPACITY { - let ptr = self.insert_fit(key, val); + let ptr = unsafe { self.insert_fit(key, val) }; (InsertResult::Fit(Handle::new_kv(self.node, self.idx)), ptr) } else { let middle = Handle::new_kv(self.node, B); - let (mut left, k, v, mut right) = middle.split(); + let (mut left, k, v, mut right) = unsafe { middle.split() }; let ptr = if self.idx <= B { unsafe { Handle::new_edge(left.reborrow_mut(), self.idx).insert_fit(key, val) } } else { @@ -958,27 +927,25 @@ impl<'a, K, V> Handle, K, V, marker::Internal>, marker:: /// Inserts a new key/value pair and an edge that will go to the right of that new pair /// between this edge and the key/value pair to the right of this edge. This method assumes /// that there is enough space in the node for the new pair to fit. - fn insert_fit(&mut self, key: K, val: V, edge: Root) { + unsafe fn insert_fit(&mut self, key: K, val: V, edge: Root) { // Necessary for correctness, but in an internal module debug_assert!(self.node.len() < CAPACITY); debug_assert!(edge.height == self.node.height - 1); - unsafe { - // This cast is a lie, but it allows us to reuse the key/value insertion logic. - self.cast_unchecked::().insert_fit(key, val); - - slice_insert( - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut self.node.as_internal_mut().edges), - self.node.len(), - ), - self.idx + 1, - edge.node, - ); + // This cast is a lie, but it allows us to reuse the key/value insertion logic. + self.cast_unchecked::().insert_fit(key, val); - for i in (self.idx + 1)..(self.node.len() + 1) { - Handle::new_edge(self.node.reborrow_mut(), i).correct_parent_link(); - } + slice_insert( + slice::from_raw_parts_mut( + MaybeUninit::first_ptr_mut(&mut self.node.as_internal_mut().edges), + self.node.len(), + ), + self.idx + 1, + edge.node, + ); + + for i in (self.idx + 1)..(self.node.len() + 1) { + Handle::new_edge(self.node.reborrow_mut(), i).correct_parent_link(); } } @@ -991,11 +958,12 @@ impl<'a, K, V> Handle, K, V, marker::Internal>, marker:: val: V, edge: Root, ) -> InsertResult<'a, K, V, marker::Internal> { - // Necessary for correctness, but this is an internal module - debug_assert!(edge.height == self.node.height - 1); + assert!(edge.height == self.node.height - 1); if self.node.len() < CAPACITY { - self.insert_fit(key, val, edge); + unsafe { + self.insert_fit(key, val, edge); + } InsertResult::Fit(Handle::new_kv(self.node, self.idx)) } else { let middle = Handle::new_kv(self.node, B); @@ -1036,25 +1004,23 @@ impl Handle, marke } impl<'a, K: 'a, V: 'a, NodeType> Handle, K, V, NodeType>, marker::KV> { - pub fn into_kv(self) -> (&'a K, &'a V) { + pub unsafe fn into_kv(self) -> (&'a K, &'a V) { let (keys, vals) = self.node.into_slices(); - unsafe { (keys.get_unchecked(self.idx), vals.get_unchecked(self.idx)) } + (keys.get_unchecked(self.idx), vals.get_unchecked(self.idx)) } } impl<'a, K: 'a, V: 'a, NodeType> Handle, K, V, NodeType>, marker::KV> { - pub fn into_kv_mut(self) -> (&'a mut K, &'a mut V) { + pub unsafe fn into_kv_mut(self) -> (&'a mut K, &'a mut V) { let (keys, vals) = self.node.into_slices_mut(); - unsafe { (keys.get_unchecked_mut(self.idx), vals.get_unchecked_mut(self.idx)) } + (keys.get_unchecked_mut(self.idx), vals.get_unchecked_mut(self.idx)) } } impl<'a, K, V, NodeType> Handle, K, V, NodeType>, marker::KV> { - pub fn kv_mut(&mut self) -> (&mut K, &mut V) { - unsafe { - let (keys, vals) = self.node.reborrow_mut().into_slices_mut(); - (keys.get_unchecked_mut(self.idx), vals.get_unchecked_mut(self.idx)) - } + pub unsafe fn kv_mut(&mut self) -> (&mut K, &mut V) { + let (keys, vals) = self.node.reborrow_mut().into_slices_mut(); + (keys.get_unchecked_mut(self.idx), vals.get_unchecked_mut(self.idx)) } } @@ -1066,46 +1032,44 @@ impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::KV> /// - The key and value pointed to by this handle and extracted. /// - All the key/value pairs to the right of this handle are put into a newly /// allocated node. - pub fn split(mut self) -> (NodeRef, K, V, marker::Leaf>, K, V, Root) { + pub unsafe fn split( + mut self, + ) -> (NodeRef, K, V, marker::Leaf>, K, V, Root) { debug_assert!(!self.node.is_shared_root()); - unsafe { - let mut new_node = Box::new(LeafNode::new()); + let mut new_node = Box::new(LeafNode::new()); - let k = ptr::read(self.node.keys().get_unchecked(self.idx)); - let v = ptr::read(self.node.vals().get_unchecked(self.idx)); + let k = ptr::read(self.node.keys().get_unchecked(self.idx)); + let v = ptr::read(self.node.vals().get_unchecked(self.idx)); - let new_len = self.node.len() - self.idx - 1; + let new_len = self.node.len() - self.idx - 1; - ptr::copy_nonoverlapping( - self.node.keys().as_ptr().add(self.idx + 1), - new_node.keys.as_mut_ptr() as *mut K, - new_len, - ); - ptr::copy_nonoverlapping( - self.node.vals().as_ptr().add(self.idx + 1), - new_node.vals.as_mut_ptr() as *mut V, - new_len, - ); + ptr::copy_nonoverlapping( + self.node.keys().as_ptr().add(self.idx + 1), + new_node.keys.as_mut_ptr() as *mut K, + new_len, + ); + ptr::copy_nonoverlapping( + self.node.vals().as_ptr().add(self.idx + 1), + new_node.vals.as_mut_ptr() as *mut V, + new_len, + ); - (*self.node.as_leaf_mut()).len = self.idx as u16; - new_node.len = new_len as u16; + (*self.node.as_leaf_mut()).len = self.idx as u16; + new_node.len = new_len as u16; - (self.node, k, v, Root { node: BoxedNode::from_leaf(new_node), height: 0 }) - } + (self.node, k, v, Root { node: BoxedNode::from_leaf(new_node), height: 0 }) } /// Removes the key/value pair pointed to by this handle and returns it, along with the edge /// between the now adjacent key/value pairs (if any) to the left and right of this handle. - pub fn remove( + pub unsafe fn remove( mut self, ) -> (Handle, K, V, marker::Leaf>, marker::Edge>, K, V) { debug_assert!(!self.node.is_shared_root()); - unsafe { - let k = slice_remove(self.node.keys_mut(), self.idx); - let v = slice_remove(self.node.vals_mut(), self.idx); - (*self.node.as_leaf_mut()).len -= 1; - (self.left_edge(), k, v) - } + let k = slice_remove(self.node.keys_mut(), self.idx); + let v = slice_remove(self.node.vals_mut(), self.idx); + (*self.node.as_leaf_mut()).len -= 1; + (self.left_edge(), k, v) } } @@ -1171,11 +1135,11 @@ impl<'a, K, V> Handle, K, V, marker::Internal>, marker:: /// child of the underlying node, returning an edge referencing that new child. /// /// Assumes that this edge `.can_merge()`. - pub fn merge( + pub unsafe fn merge( mut self, ) -> Handle, K, V, marker::Internal>, marker::Edge> { - let self1 = unsafe { ptr::read(&self) }; - let self2 = unsafe { ptr::read(&self) }; + let self1 = ptr::read(&self); + let self2 = ptr::read(&self); let mut left_node = self1.left_edge().descend(); let left_len = left_node.len(); let mut right_node = self2.right_edge().descend(); @@ -1184,58 +1148,51 @@ impl<'a, K, V> Handle, K, V, marker::Internal>, marker:: // necessary for correctness, but in a private module debug_assert!(left_len + right_len + 1 <= CAPACITY); - unsafe { - ptr::write( - left_node.keys_mut().get_unchecked_mut(left_len), - slice_remove(self.node.keys_mut(), self.idx), - ); - ptr::copy_nonoverlapping( - right_node.keys().as_ptr(), - left_node.keys_mut().as_mut_ptr().add(left_len + 1), - right_len, - ); - ptr::write( - left_node.vals_mut().get_unchecked_mut(left_len), - slice_remove(self.node.vals_mut(), self.idx), - ); - ptr::copy_nonoverlapping( - right_node.vals().as_ptr(), - left_node.vals_mut().as_mut_ptr().add(left_len + 1), - right_len, - ); + ptr::write( + left_node.keys_mut().get_unchecked_mut(left_len), + slice_remove(self.node.keys_mut(), self.idx), + ); + ptr::copy_nonoverlapping( + right_node.keys().as_ptr(), + left_node.keys_mut().as_mut_ptr().add(left_len + 1), + right_len, + ); + ptr::write( + left_node.vals_mut().get_unchecked_mut(left_len), + slice_remove(self.node.vals_mut(), self.idx), + ); + ptr::copy_nonoverlapping( + right_node.vals().as_ptr(), + left_node.vals_mut().as_mut_ptr().add(left_len + 1), + right_len, + ); + + slice_remove(&mut self.node.as_internal_mut().edges, self.idx + 1); + for i in self.idx + 1..self.node.len() { + Handle::new_edge(self.node.reborrow_mut(), i).correct_parent_link(); + } + (*self.node.as_leaf_mut()).len -= 1; - slice_remove(&mut self.node.as_internal_mut().edges, self.idx + 1); - for i in self.idx + 1..self.node.len() { - Handle::new_edge(self.node.reborrow_mut(), i).correct_parent_link(); - } - (*self.node.as_leaf_mut()).len -= 1; - - (*left_node.as_leaf_mut()).len += right_len as u16 + 1; - - if self.node.height > 1 { - ptr::copy_nonoverlapping( - right_node.cast_unchecked().as_internal().edges.as_ptr(), - left_node - .cast_unchecked() - .as_internal_mut() - .edges - .as_mut_ptr() - .add(left_len + 1), - right_len + 1, - ); + (*left_node.as_leaf_mut()).len += right_len as u16 + 1; - for i in left_len + 1..left_len + right_len + 2 { - Handle::new_edge(left_node.cast_unchecked().reborrow_mut(), i) - .correct_parent_link(); - } + if self.node.height > 1 { + ptr::copy_nonoverlapping( + right_node.cast_unchecked().as_internal().edges.as_ptr(), + left_node.cast_unchecked().as_internal_mut().edges.as_mut_ptr().add(left_len + 1), + right_len + 1, + ); - Global.dealloc(right_node.node.cast(), Layout::new::>()); - } else { - Global.dealloc(right_node.node.cast(), Layout::new::>()); + for i in left_len + 1..left_len + right_len + 2 { + Handle::new_edge(left_node.cast_unchecked().reborrow_mut(), i) + .correct_parent_link(); } - Handle::new_edge(self.node, self.idx) + Global.dealloc(right_node.node.cast(), Layout::new::>()); + } else { + Global.dealloc(right_node.node.cast(), Layout::new::>()); } + + Handle::new_edge(self.node, self.idx) } /// This removes a key/value pair from the left child and places it in the key/value storage @@ -1273,115 +1230,111 @@ impl<'a, K, V> Handle, K, V, marker::Internal>, marker:: } /// This does stealing similar to `steal_left` but steals multiple elements at once. - pub fn bulk_steal_left(&mut self, count: usize) { - unsafe { - let mut left_node = ptr::read(self).left_edge().descend(); - let left_len = left_node.len(); - let mut right_node = ptr::read(self).right_edge().descend(); - let right_len = right_node.len(); - - // Make sure that we may steal safely. - debug_assert!(right_len + count <= CAPACITY); - debug_assert!(left_len >= count); - - let new_left_len = left_len - count; - - // Move data. - { - let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); - let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); - let parent_kv = { - let kv = self.reborrow_mut().into_kv_mut(); - (kv.0 as *mut K, kv.1 as *mut V) - }; - - // Make room for stolen elements in the right child. - ptr::copy(right_kv.0, right_kv.0.add(count), right_len); - ptr::copy(right_kv.1, right_kv.1.add(count), right_len); - - // Move elements from the left child to the right one. - move_kv(left_kv, new_left_len + 1, right_kv, 0, count - 1); - - // Move parent's key/value pair to the right child. - move_kv(parent_kv, 0, right_kv, count - 1, 1); - - // Move the left-most stolen pair to the parent. - move_kv(left_kv, new_left_len, parent_kv, 0, 1); - } + pub unsafe fn bulk_steal_left(&mut self, count: usize) { + let mut left_node = ptr::read(self).left_edge().descend(); + let left_len = left_node.len(); + let mut right_node = ptr::read(self).right_edge().descend(); + let right_len = right_node.len(); - (*left_node.reborrow_mut().as_leaf_mut()).len -= count as u16; - (*right_node.reborrow_mut().as_leaf_mut()).len += count as u16; + // Make sure that we may steal safely. + debug_assert!(right_len + count <= CAPACITY); + debug_assert!(left_len >= count); - match (left_node.force(), right_node.force()) { - (ForceResult::Internal(left), ForceResult::Internal(mut right)) => { - // Make room for stolen edges. - let right_edges = right.reborrow_mut().as_internal_mut().edges.as_mut_ptr(); - ptr::copy(right_edges, right_edges.add(count), right_len + 1); - right.correct_childrens_parent_links(count, count + right_len + 1); + let new_left_len = left_len - count; - move_edges(left, new_left_len + 1, right, 0, count); - } - (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} - _ => { - unreachable!(); - } + // Move data. + { + let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); + let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); + let parent_kv = { + let kv = self.reborrow_mut().into_kv_mut(); + (kv.0 as *mut K, kv.1 as *mut V) + }; + + // Make room for stolen elements in the right child. + ptr::copy(right_kv.0, right_kv.0.add(count), right_len); + ptr::copy(right_kv.1, right_kv.1.add(count), right_len); + + // Move elements from the left child to the right one. + move_kv(left_kv, new_left_len + 1, right_kv, 0, count - 1); + + // Move parent's key/value pair to the right child. + move_kv(parent_kv, 0, right_kv, count - 1, 1); + + // Move the left-most stolen pair to the parent. + move_kv(left_kv, new_left_len, parent_kv, 0, 1); + } + + (*left_node.reborrow_mut().as_leaf_mut()).len -= count as u16; + (*right_node.reborrow_mut().as_leaf_mut()).len += count as u16; + + match (left_node.force(), right_node.force()) { + (ForceResult::Internal(left), ForceResult::Internal(mut right)) => { + // Make room for stolen edges. + let right_edges = right.reborrow_mut().as_internal_mut().edges.as_mut_ptr(); + ptr::copy(right_edges, right_edges.add(count), right_len + 1); + right.correct_childrens_parent_links(count, count + right_len + 1); + + move_edges(left, new_left_len + 1, right, 0, count); + } + (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} + _ => { + unreachable!(); } } } /// The symmetric clone of `bulk_steal_left`. - pub fn bulk_steal_right(&mut self, count: usize) { - unsafe { - let mut left_node = ptr::read(self).left_edge().descend(); - let left_len = left_node.len(); - let mut right_node = ptr::read(self).right_edge().descend(); - let right_len = right_node.len(); - - // Make sure that we may steal safely. - debug_assert!(left_len + count <= CAPACITY); - debug_assert!(right_len >= count); - - let new_right_len = right_len - count; - - // Move data. - { - let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); - let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); - let parent_kv = { - let kv = self.reborrow_mut().into_kv_mut(); - (kv.0 as *mut K, kv.1 as *mut V) - }; - - // Move parent's key/value pair to the left child. - move_kv(parent_kv, 0, left_kv, left_len, 1); - - // Move elements from the right child to the left one. - move_kv(right_kv, 0, left_kv, left_len + 1, count - 1); - - // Move the right-most stolen pair to the parent. - move_kv(right_kv, count - 1, parent_kv, 0, 1); - - // Fix right indexing - ptr::copy(right_kv.0.add(count), right_kv.0, new_right_len); - ptr::copy(right_kv.1.add(count), right_kv.1, new_right_len); - } + pub unsafe fn bulk_steal_right(&mut self, count: usize) { + let mut left_node = ptr::read(self).left_edge().descend(); + let left_len = left_node.len(); + let mut right_node = ptr::read(self).right_edge().descend(); + let right_len = right_node.len(); - (*left_node.reborrow_mut().as_leaf_mut()).len += count as u16; - (*right_node.reborrow_mut().as_leaf_mut()).len -= count as u16; + // Make sure that we may steal safely. + debug_assert!(left_len + count <= CAPACITY); + debug_assert!(right_len >= count); - match (left_node.force(), right_node.force()) { - (ForceResult::Internal(left), ForceResult::Internal(mut right)) => { - move_edges(right.reborrow_mut(), 0, left, left_len + 1, count); + let new_right_len = right_len - count; - // Fix right indexing. - let right_edges = right.reborrow_mut().as_internal_mut().edges.as_mut_ptr(); - ptr::copy(right_edges.add(count), right_edges, new_right_len + 1); - right.correct_childrens_parent_links(0, new_right_len + 1); - } - (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} - _ => { - unreachable!(); - } + // Move data. + { + let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); + let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); + let parent_kv = { + let kv = self.reborrow_mut().into_kv_mut(); + (kv.0 as *mut K, kv.1 as *mut V) + }; + + // Move parent's key/value pair to the left child. + move_kv(parent_kv, 0, left_kv, left_len, 1); + + // Move elements from the right child to the left one. + move_kv(right_kv, 0, left_kv, left_len + 1, count - 1); + + // Move the right-most stolen pair to the parent. + move_kv(right_kv, count - 1, parent_kv, 0, 1); + + // Fix right indexing + ptr::copy(right_kv.0.add(count), right_kv.0, new_right_len); + ptr::copy(right_kv.1.add(count), right_kv.1, new_right_len); + } + + (*left_node.reborrow_mut().as_leaf_mut()).len += count as u16; + (*right_node.reborrow_mut().as_leaf_mut()).len -= count as u16; + + match (left_node.force(), right_node.force()) { + (ForceResult::Internal(left), ForceResult::Internal(mut right)) => { + move_edges(right.reborrow_mut(), 0, left, left_len + 1, count); + + // Fix right indexing. + let right_edges = right.reborrow_mut().as_internal_mut().edges.as_mut_ptr(); + ptr::copy(right_edges.add(count), right_edges, new_right_len + 1); + right.correct_childrens_parent_links(0, new_right_len + 1); + } + (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} + _ => { + unreachable!(); } } } @@ -1436,36 +1389,34 @@ impl impl<'a, K, V> Handle, K, V, marker::LeafOrInternal>, marker::Edge> { /// Move the suffix after `self` from one node to another one. `right` must be empty. /// The first edge of `right` remains unchanged. - pub fn move_suffix( + pub unsafe fn move_suffix( &mut self, right: &mut NodeRef, K, V, marker::LeafOrInternal>, ) { - unsafe { - let left_new_len = self.idx; - let mut left_node = self.reborrow_mut().into_node(); + let left_new_len = self.idx; + let mut left_node = self.reborrow_mut().into_node(); - let right_new_len = left_node.len() - left_new_len; - let mut right_node = right.reborrow_mut(); + let right_new_len = left_node.len() - left_new_len; + let mut right_node = right.reborrow_mut(); - debug_assert!(right_node.len() == 0); - debug_assert!(left_node.height == right_node.height); + debug_assert!(right_node.len() == 0); + debug_assert!(left_node.height == right_node.height); - let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); - let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); + let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); + let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); - move_kv(left_kv, left_new_len, right_kv, 0, right_new_len); + move_kv(left_kv, left_new_len, right_kv, 0, right_new_len); - (*left_node.reborrow_mut().as_leaf_mut()).len = left_new_len as u16; - (*right_node.reborrow_mut().as_leaf_mut()).len = right_new_len as u16; + (*left_node.reborrow_mut().as_leaf_mut()).len = left_new_len as u16; + (*right_node.reborrow_mut().as_leaf_mut()).len = right_new_len as u16; - match (left_node.force(), right_node.force()) { - (ForceResult::Internal(left), ForceResult::Internal(right)) => { - move_edges(left, left_new_len + 1, right, 1, right_new_len); - } - (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} - _ => { - unreachable!(); - } + match (left_node.force(), right_node.force()) { + (ForceResult::Internal(left), ForceResult::Internal(right)) => { + move_edges(left, left_new_len + 1, right, 1, right_new_len); + } + (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} + _ => { + unreachable!(); } } }