Rollup merge of #160813 - fereidani:linked_list_optimization, r=clarfonthey

Optimize linked list iterator performance

This PR optimizes the linked list iterator performance by removing an extra branch check.

This resulted in less instructions for single access(`last()`, `next()`, etc.) and optimal iterator optimization(results in longer instructions) as the compiler requires only check for `len==0` exit, I expect slight binary-size increase for much better performance.

I did my best to find a counter-example where `len > 0` but head or tail are not initialized, I couldn't. Could you please double-check the claim just in case I missed something?

I tested it using `alloctests` available benchmarks too, It seems that compiler is optimizing the benchmark test away, results seems weird(too good), I tried using `black_box` around the object but results stayed the same(0.18ns/iter from 60.49ns/iter):
```rust
#[bench]
fn bench_iter(b: &mut Bencher) {
    let v = &[0; 128];
    let m: LinkedList<_> = v.iter().cloned().collect();
    b.iter(|| {
        assert!(black_box(&m).iter().count() == 128);
    })
}
```

*9950x results:*

Current:
```
    linked_list::bench_collect_into         396.14ns/iter +/- 3.12
    linked_list::bench_iter                  60.49ns/iter +/- 0.91
    linked_list::bench_iter_mut              60.32ns/iter +/- 0.97
    linked_list::bench_iter_mut_rev          61.72ns/iter +/- 0.06
    linked_list::bench_iter_rev              59.61ns/iter +/- 0.08
    linked_list::bench_push_back             10.06ns/iter +/- 0.33
    linked_list::bench_push_back_pop_back     4.54ns/iter +/- 0.12
    linked_list::bench_push_front             2.87ns/iter +/- 0.05
    linked_list::bench_push_front_pop_front   4.52ns/iter +/- 0.05
```

New:
```
    linked_list::bench_collect_into         391.06ns/iter +/- 24.49
    linked_list::bench_iter                   0.18ns/iter  +/- 0.00
    linked_list::bench_iter_mut               0.18ns/iter  +/- 0.00
    linked_list::bench_iter_mut_rev           0.18ns/iter  +/- 0.00
    linked_list::bench_iter_rev               0.18ns/iter  +/- 0.00
    linked_list::bench_push_back             10.11ns/iter  +/- 0.18
    linked_list::bench_push_back_pop_back     4.35ns/iter  +/- 0.10
    linked_list::bench_push_front             3.27ns/iter  +/- 0.46
    linked_list::bench_push_front_pop_front   4.66ns/iter  +/- 0.04
```

Author: jhpratt

library/alloc/src/collections/linked_list.rs

@@ -15,7 +15,7 @@
 use core::alloc::AllocatorClone;
 use core::cmp::Ordering;
 use core::hash::{Hash, Hasher};
-use core::iter::FusedIterator;
+use core::iter::{FusedIterator, TrustedLen};
 use core::marker::PhantomData;
 use core::ptr::NonNull;
 use core::{fmt, mem};
@@ -1201,16 +1201,18 @@ impl<'a, T> Iterator for Iter<'a, T> {
     #[inline]
     fn next(&mut self) -> Option<&'a T> {
         if self.len == 0 {
-            None
-        } else {
-            self.head.map(|node| unsafe {
-                // Need an unbound lifetime to get 'a
-                let node = &*node.as_ptr();
-                self.len -= 1;
-                self.head = node.next;
-                &node.element
-            })
+            return None;
         }
+        // SAFETY: When `len > 0`, `head` and `tail` are guaranteed to be `Some`.
+        // The lifetime of the returned reference is bound to the lifetime of the iterator,
+        // which is valid because the iterator holds a reference to the list.
+        Some(unsafe {
+            // Need an unbound lifetime to get 'a
+            let node = &*self.head.unwrap_unchecked().as_ptr();
+            self.len -= 1;
+            self.head = node.next;
+            &node.element
+        })
     }
 
     #[inline]
@@ -1229,16 +1231,18 @@ impl<'a, T> DoubleEndedIterator for Iter<'a, T> {
     #[inline]
     fn next_back(&mut self) -> Option<&'a T> {
         if self.len == 0 {
-            None
-        } else {
-            self.tail.map(|node| unsafe {
-                // Need an unbound lifetime to get 'a
-                let node = &*node.as_ptr();
-                self.len -= 1;
-                self.tail = node.prev;
-                &node.element
-            })
+            return None;
         }
+        // SAFETY: When `len > 0`, `head` and `tail` are guaranteed to be `Some`.
+        // The lifetime of the returned reference is bound to the lifetime of the iterator,
+        // which is valid because the iterator holds a reference to the list.
+        Some(unsafe {
+            // Need an unbound lifetime to get 'a
+            let node = &*self.tail.unwrap_unchecked().as_ptr();
+            self.len -= 1;
+            self.tail = node.prev;
+            &node.element
+        })
     }
 }
 
@@ -1248,6 +1252,9 @@ impl<T> ExactSizeIterator for Iter<'_, T> {}
 #[stable(feature = "fused", since = "1.26.0")]
 impl<T> FusedIterator for Iter<'_, T> {}
 
+#[unstable(feature = "trusted_len", issue = "37572")]
+unsafe impl<T> TrustedLen for Iter<'_, T> {}
+
 #[stable(feature = "default_iters", since = "1.70.0")]
 impl<T> Default for Iter<'_, T> {
     /// Creates an empty `linked_list::Iter`.
@@ -1269,16 +1276,18 @@ impl<'a, T> Iterator for IterMut<'a, T> {
     #[inline]
     fn next(&mut self) -> Option<&'a mut T> {
         if self.len == 0 {
-            None
-        } else {
-            self.head.map(|node| unsafe {
-                // Need an unbound lifetime to get 'a
-                let node = &mut *node.as_ptr();
-                self.len -= 1;
-                self.head = node.next;
-                &mut node.element
-            })
+            return None;
         }
+        // SAFETY: When `len > 0`, `head` and `tail` are guaranteed to be `Some`.
+        // The lifetime of the returned reference is bound to the lifetime of the iterator,
+        // which is valid because the iterator holds a reference to the list.
+        Some(unsafe {
+            // Need an unbound lifetime to get 'a
+            let node = &mut *self.head.unwrap_unchecked().as_ptr();
+            self.len -= 1;
+            self.head = node.next;
+            &mut node.element
+        })
     }
 
     #[inline]
@@ -1297,16 +1306,18 @@ impl<'a, T> DoubleEndedIterator for IterMut<'a, T> {
     #[inline]
     fn next_back(&mut self) -> Option<&'a mut T> {
         if self.len == 0 {
-            None
-        } else {
-            self.tail.map(|node| unsafe {
-                // Need an unbound lifetime to get 'a
-                let node = &mut *node.as_ptr();
-                self.len -= 1;
-                self.tail = node.prev;
-                &mut node.element
-            })
+            return None;
         }
+        // SAFETY: When `len > 0`, `head` and `tail` are guaranteed to be `Some`.
+        // The lifetime of the returned reference is bound to the lifetime of the iterator,
+        // which is valid because the iterator holds a reference to the list.
+        Some(unsafe {
+            // Need an unbound lifetime to get 'a
+            let node = &mut *self.tail.unwrap_unchecked().as_ptr();
+            self.len -= 1;
+            self.tail = node.prev;
+            &mut node.element
+        })
     }
 }
 
@@ -1316,6 +1327,9 @@ impl<T> ExactSizeIterator for IterMut<'_, T> {}
 #[stable(feature = "fused", since = "1.26.0")]
 impl<T> FusedIterator for IterMut<'_, T> {}
 
+#[unstable(feature = "trusted_len", issue = "37572")]
+unsafe impl<T> TrustedLen for IterMut<'_, T> {}
+
 #[stable(feature = "default_iters", since = "1.70.0")]
 impl<T> Default for IterMut<'_, T> {
     fn default() -> Self {
@@ -2032,6 +2046,9 @@ impl<T, A: Allocator> ExactSizeIterator for IntoIter<T, A> {}
 #[stable(feature = "fused", since = "1.26.0")]
 impl<T, A: Allocator> FusedIterator for IntoIter<T, A> {}
 
+#[unstable(feature = "trusted_len", issue = "37572")]
+unsafe impl<T, A: Allocator> TrustedLen for IntoIter<T, A> {}
+
 #[stable(feature = "default_iters", since = "1.70.0")]
 impl<T> Default for IntoIter<T> {
     /// Creates an empty `linked_list::IntoIter`.

library/alloctests/benches/linked_list.rs

@@ -1,6 +1,6 @@
 use std::collections::LinkedList;
 
-use test::Bencher;
+use test::{Bencher, black_box};
 
 #[bench]
 fn bench_collect_into(b: &mut Bencher) {
@@ -44,35 +44,40 @@ fn bench_push_front_pop_front(b: &mut Bencher) {
     })
 }
 
+#[bench]
+fn bench_iter_count(b: &mut Bencher) {
+    let m: LinkedList<_> = (0..128).collect();
+    b.iter(|| {
+        assert!(black_box(&m).iter().count() == 128);
+    })
+}
+
 #[bench]
 fn bench_iter(b: &mut Bencher) {
-    let v = &[0; 128];
-    let m: LinkedList<_> = v.iter().cloned().collect();
+    let m: LinkedList<usize> = (0..128).collect();
     b.iter(|| {
-        assert!(m.iter().count() == 128);
+        assert!((0..128).sum::<usize>() == black_box(&m).iter().sum());
     })
 }
+
 #[bench]
 fn bench_iter_mut(b: &mut Bencher) {
-    let v = &[0; 128];
-    let mut m: LinkedList<_> = v.iter().cloned().collect();
+    let mut m: LinkedList<usize> = (0..128).collect();
     b.iter(|| {
-        assert!(m.iter_mut().count() == 128);
+        black_box(&mut m).iter_mut().for_each(|x| *x += 1);
     })
 }
 #[bench]
 fn bench_iter_rev(b: &mut Bencher) {
-    let v = &[0; 128];
-    let m: LinkedList<_> = v.iter().cloned().collect();
+    let m: LinkedList<usize> = (0..128).collect();
     b.iter(|| {
-        assert!(m.iter().rev().count() == 128);
+        assert!((0..128).sum::<usize>() == black_box(&m).iter().rev().sum());
     })
 }
 #[bench]
 fn bench_iter_mut_rev(b: &mut Bencher) {
-    let v = &[0; 128];
-    let mut m: LinkedList<_> = v.iter().cloned().collect();
+    let mut m: LinkedList<usize> = (0..128).collect();
     b.iter(|| {
-        assert!(m.iter_mut().rev().count() == 128);
+        black_box(&mut m).iter_mut().rev().for_each(|x| *x += 1);
     })
 }