Permalink
Browse files

Auto merge of #37972 - bluss:iter-find-is-on-a-roll, r=sfackler

Improve the slice iterator's searching methods

Improve all, any, find, position, rposition by explicitly unrolling the loop for the slice iterators.

- Introduce a few extension methods and functions for raw pointers make the new code easy to express
- Introduce helper methods `search_while, rsearch_while` that generalize all the searching methods

LLVM doesn't unroll the loop in `.find()` by default (clang is the same), so performance benefits a lot from explicit unrolling here. An iterator method without conditional exits (like `.fold()`) does not need this on the other hand.

One of the raw pointer extension methods is `fn post_inc(&mut self) -> Self` which is the rustic equivalent of “`ptr++`”, and it is a nice way to express the raw pointer loop (see commit 3).

Specific development notes about `search_while`: I tried both computing an end pointer "rounded" to 4, as well as the `ptrdistance >= 4` loop condition, ptrdistance was better. I tried handling the last 0-3 elements unrolled or with a while loop, the loop was better.
  • Loading branch information...
2 parents bd8e9b0 + a54ddfb commit c07a6ae77cd4ceb3cf591d34c5608ca91d1f75d4 @bors bors committed Jan 17, 2017
Showing with 185 additions and 14 deletions.
  1. +185 −14 src/libcore/slice.rs
View
@@ -172,21 +172,35 @@ macro_rules! slice_offset {
($ptr:expr, $by:expr) => {{
let ptr = $ptr;
if size_from_ptr(ptr) == 0 {
- ::intrinsics::arith_offset(ptr as *mut i8, $by) as *mut _
+ (ptr as *mut i8).wrapping_offset($by) as _
} else {
ptr.offset($by)
}
}};
}
-macro_rules! slice_ref {
+// make a &T from a *const T
+macro_rules! make_ref {
+ ($ptr:expr) => {{
+ let ptr = $ptr;
+ if size_from_ptr(ptr) == 0 {
+ // Use a non-null pointer value
+ &*(1 as *mut _)
+ } else {
+ &*ptr
+ }
+ }};
+}
+
+// make a &mut T from a *mut T
+macro_rules! make_ref_mut {
($ptr:expr) => {{
let ptr = $ptr;
if size_from_ptr(ptr) == 0 {
// Use a non-null pointer value
&mut *(1 as *mut _)
} else {
- mem::transmute(ptr)
+ &mut *ptr
}
}};
}
@@ -963,7 +977,7 @@ fn size_from_ptr<T>(_: *const T) -> usize {
// The shared definition of the `Iter` and `IterMut` iterators
macro_rules! iterator {
- (struct $name:ident -> $ptr:ty, $elem:ty) => {
+ (struct $name:ident -> $ptr:ty, $elem:ty, $mkref:ident) => {
#[stable(feature = "rust1", since = "1.0.0")]
impl<'a, T> Iterator for $name<'a, T> {
type Item = $elem;
@@ -979,18 +993,14 @@ macro_rules! iterator {
if self.ptr == self.end {
None
} else {
- let old = self.ptr;
- self.ptr = slice_offset!(self.ptr, 1);
- Some(slice_ref!(old))
+ Some($mkref!(self.ptr.post_inc()))
}
}
}
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) {
- let diff = (self.end as usize).wrapping_sub(self.ptr as usize);
- let size = mem::size_of::<T>();
- let exact = diff / (if size == 0 {1} else {size});
+ let exact = ptrdistance(self.ptr, self.end);
(exact, Some(exact))
}
@@ -1009,6 +1019,64 @@ macro_rules! iterator {
fn last(mut self) -> Option<$elem> {
self.next_back()
}
+
+ fn all<F>(&mut self, mut predicate: F) -> bool
+ where F: FnMut(Self::Item) -> bool,
+ {
+ self.search_while(true, move |elt| {
+ if predicate(elt) {
+ SearchWhile::Continue
+ } else {
+ SearchWhile::Done(false)
+ }
+ })
+ }
+
+ fn any<F>(&mut self, mut predicate: F) -> bool
+ where F: FnMut(Self::Item) -> bool,
+ {
+ !self.all(move |elt| !predicate(elt))
+ }
+
+ fn find<F>(&mut self, mut predicate: F) -> Option<Self::Item>
+ where F: FnMut(&Self::Item) -> bool,
+ {
+ self.search_while(None, move |elt| {
+ if predicate(&elt) {
+ SearchWhile::Done(Some(elt))
+ } else {
+ SearchWhile::Continue
+ }
+ })
+ }
+
+ fn position<F>(&mut self, mut predicate: F) -> Option<usize>
+ where F: FnMut(Self::Item) -> bool,
+ {
+ let mut index = 0;
+ self.search_while(None, move |elt| {
+ if predicate(elt) {
+ SearchWhile::Done(Some(index))
+ } else {
+ index += 1;
+ SearchWhile::Continue
+ }
+ })
+ }
+
+ fn rposition<F>(&mut self, mut predicate: F) -> Option<usize>
+ where F: FnMut(Self::Item) -> bool,
+ {
+ let mut index = self.len();
+ self.rsearch_while(None, move |elt| {
+ index -= 1;
+ if predicate(elt) {
+ SearchWhile::Done(Some(index))
+ } else {
+ SearchWhile::Continue
+ }
+ })
+ }
}
#[stable(feature = "rust1", since = "1.0.0")]
@@ -1024,10 +1092,51 @@ macro_rules! iterator {
if self.end == self.ptr {
None
} else {
- self.end = slice_offset!(self.end, -1);
- Some(slice_ref!(self.end))
+ Some($mkref!(self.end.pre_dec()))
+ }
+ }
+ }
+ }
+
+ // search_while is a generalization of the internal iteration methods.
+ impl<'a, T> $name<'a, T> {
+ // search through the iterator's element using the closure `g`.
+ // if no element was found, return `default`.
+ fn search_while<Acc, G>(&mut self, default: Acc, mut g: G) -> Acc
+ where Self: Sized,
+ G: FnMut($elem) -> SearchWhile<Acc>
+ {
+ // manual unrolling is needed when there are conditional exits from the loop
+ unsafe {
+ while ptrdistance(self.ptr, self.end) >= 4 {
+ search_while!(g($mkref!(self.ptr.post_inc())));
+ search_while!(g($mkref!(self.ptr.post_inc())));
+ search_while!(g($mkref!(self.ptr.post_inc())));
+ search_while!(g($mkref!(self.ptr.post_inc())));
+ }
+ while self.ptr != self.end {
+ search_while!(g($mkref!(self.ptr.post_inc())));
+ }
+ }
+ default
+ }
+
+ fn rsearch_while<Acc, G>(&mut self, default: Acc, mut g: G) -> Acc
+ where Self: Sized,
+ G: FnMut($elem) -> SearchWhile<Acc>
+ {
+ unsafe {
+ while ptrdistance(self.ptr, self.end) >= 4 {
+ search_while!(g($mkref!(self.end.pre_dec())));
+ search_while!(g($mkref!(self.end.pre_dec())));
+ search_while!(g($mkref!(self.end.pre_dec())));
+ search_while!(g($mkref!(self.end.pre_dec())));
+ }
+ while self.ptr != self.end {
+ search_while!(g($mkref!(self.end.pre_dec())));
}
}
+ default
}
}
}
@@ -1061,6 +1170,24 @@ macro_rules! make_mut_slice {
}}
}
+// An enum used for controlling the execution of `.search_while()`.
+enum SearchWhile<T> {
+ // Continue searching
+ Continue,
+ // Fold is complete and will return this value
+ Done(T),
+}
+
+// helper macro for search while's control flow
+macro_rules! search_while {
+ ($e:expr) => {
+ match $e {
+ SearchWhile::Continue => { }
+ SearchWhile::Done(done) => return done,
+ }
+ }
+}
+
/// Immutable slice iterator
///
/// This struct is created by the [`iter`] method on [slices].
@@ -1147,7 +1274,7 @@ impl<'a, T> Iter<'a, T> {
}
}
-iterator!{struct Iter -> *const T, &'a T}
+iterator!{struct Iter -> *const T, &'a T, make_ref}
#[stable(feature = "rust1", since = "1.0.0")]
impl<'a, T> ExactSizeIterator for Iter<'a, T> {
@@ -1275,7 +1402,7 @@ impl<'a, T> IterMut<'a, T> {
}
}
-iterator!{struct IterMut -> *mut T, &'a mut T}
+iterator!{struct IterMut -> *mut T, &'a mut T, make_ref_mut}
#[stable(feature = "rust1", since = "1.0.0")]
impl<'a, T> ExactSizeIterator for IterMut<'a, T> {
@@ -1290,6 +1417,50 @@ impl<'a, T> FusedIterator for IterMut<'a, T> {}
#[unstable(feature = "trusted_len", issue = "37572")]
unsafe impl<'a, T> TrustedLen for IterMut<'a, T> {}
+
+// Return the number of elements of `T` from `start` to `end`.
+// Return the arithmetic difference if `T` is zero size.
+#[inline(always)]
+fn ptrdistance<T>(start: *const T, end: *const T) -> usize {
+ let diff = (end as usize).wrapping_sub(start as usize);
+ let size = mem::size_of::<T>();
+ diff / (if size == 0 { 1 } else { size })
+}
+
+// Extension methods for raw pointers, used by the iterators
+trait PointerExt : Copy {
+ unsafe fn slice_offset(self, i: isize) -> Self;
+
+ /// Increment self by 1, but return the old value
+ #[inline(always)]
+ unsafe fn post_inc(&mut self) -> Self {
+ let current = *self;
+ *self = self.slice_offset(1);
+ current
+ }
+
+ /// Decrement self by 1, and return the new value
+ #[inline(always)]
+ unsafe fn pre_dec(&mut self) -> Self {
+ *self = self.slice_offset(-1);
+ *self
+ }
+}
+
+impl<T> PointerExt for *const T {
+ #[inline(always)]
+ unsafe fn slice_offset(self, i: isize) -> Self {
+ slice_offset!(self, i)
+ }
+}
+
+impl<T> PointerExt for *mut T {
+ #[inline(always)]
+ unsafe fn slice_offset(self, i: isize) -> Self {
+ slice_offset!(self, i)
+ }
+}
+
/// An internal abstraction over the splitting iterators, so that
/// splitn, splitn_mut etc can be implemented once.
#[doc(hidden)]

0 comments on commit c07a6ae

Please sign in to comment.