From 68e0ea9d47f797f815225e4f2fbd9bb1cde6e19e Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Fri, 23 Mar 2018 01:30:23 -0700 Subject: [PATCH] Introduce unsafe offset_from on pointers Adds intrinsics::exact_div to take advantage of the unsafe, which reduces the implementation from ```asm sub rcx, rdx mov rax, rcx sar rax, 63 shr rax, 62 lea rax, [rax + rcx] sar rax, 2 ret ``` down to ```asm sub rcx, rdx sar rcx, 2 mov rax, rcx ret ``` (for `*const i32`) --- src/libcore/intrinsics.rs | 10 ++ src/libcore/ptr.rs | 207 +++++++++++++++++++++++++ src/librustc_llvm/ffi.rs | 5 + src/librustc_trans/builder.rs | 7 + src/librustc_trans/intrinsic.rs | 8 +- src/librustc_typeck/check/intrinsic.rs | 2 +- 6 files changed, 237 insertions(+), 2 deletions(-) diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 830ebad065427..3b740adc46832 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -1314,6 +1314,11 @@ extern "rust-intrinsic" { /// [`std::u32::overflowing_mul`](../../std/primitive.u32.html#method.overflowing_mul) pub fn mul_with_overflow(x: T, y: T) -> (T, bool); + /// Performs an exact division, resulting in undefined behavior where + /// `x % y != 0` or `y == 0` or `x == T::min_value() && y == -1` + #[cfg(not(stage0))] + pub fn exact_div(x: T, y: T) -> T; + /// Performs an unchecked division, resulting in undefined behavior /// where y = 0 or x = `T::min_value()` and y = -1 pub fn unchecked_div(x: T, y: T) -> T; @@ -1396,3 +1401,8 @@ extern "rust-intrinsic" { /// Probably will never become stable. pub fn nontemporal_store(ptr: *mut T, val: T); } + +#[cfg(stage0)] +pub unsafe fn exact_div(a: T, b: T) -> T { + unchecked_div(a, b) +} diff --git a/src/libcore/ptr.rs b/src/libcore/ptr.rs index 6270e5892b3a0..cbd45bb6a39e1 100644 --- a/src/libcore/ptr.rs +++ b/src/libcore/ptr.rs @@ -700,6 +700,114 @@ impl *const T { } } + /// Calculates the distance between two pointers. The returned value is in + /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// + /// This function is the inverse of [`offset`]. + /// + /// [`offset`]: #method.offset + /// [`wrapping_offset_from`]: #method.wrapping_offset_from + /// + /// # Safety + /// + /// If any of the following conditions are violated, the result is Undefined + /// Behavior: + /// + /// * Both the starting and other pointer must be either in bounds or one + /// byte past the end of the same allocated object. + /// + /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. + /// + /// * The distance between the pointers, in bytes, must be an exact multiple + /// of the size of `T` and `T` must not be a Zero-Sized Type ("ZST"). + /// + /// * The distance being in bounds cannot rely on "wrapping around" the address space. + /// + /// The compiler and standard library generally try to ensure allocations + /// never reach a size where an offset is a concern. For instance, `Vec` + /// and `Box` ensure they never allocate more than `isize::MAX` bytes, so + /// `ptr_into_vec.offset_from(vec.as_ptr())` is always safe. + /// + /// Most platforms fundamentally can't even construct such an allocation. + /// For instance, no known 64-bit platform can ever serve a request + /// for 263 bytes due to page-table limitations or splitting the address space. + /// However, some 32-bit and 16-bit platforms may successfully serve a request for + /// more than `isize::MAX` bytes with things like Physical Address + /// Extension. As such, memory acquired directly from allocators or memory + /// mapped files *may* be too large to handle with this function. + /// + /// Consider using [`wrapping_offset_from`] instead if these constraints are + /// difficult to satisfy. The only advantage of this method is that it + /// enables more aggressive compiler optimizations. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_offset_from)] + /// + /// let a = [0; 5]; + /// let ptr1: *const i32 = &a[1]; + /// let ptr2: *const i32 = &a[3]; + /// unsafe { + /// assert_eq!(ptr2.offset_from(ptr1), 2); + /// assert_eq!(ptr1.offset_from(ptr2), -2); + /// assert_eq!(ptr1.offset(2), ptr2); + /// assert_eq!(ptr2.offset(-2), ptr1); + /// } + /// ``` + #[unstable(feature = "ptr_offset_from", issue = "41079")] + #[inline] + pub unsafe fn offset_from(self, other: *const T) -> isize where T: Sized { + let pointee_size = mem::size_of::(); + assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize); + + // FIXME: can this be nuw/nsw? + let d = isize::wrapping_sub(self as _, other as _); + intrinsics::exact_div(d, pointee_size as _) + } + + /// Calculates the distance between two pointers. The returned value is in + /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// + /// If the address different between the two pointers is not a multiple of + /// `mem::size_of::()` then the result of the division is rounded towards + /// zero. + /// + /// # Panics + /// + /// This function panics if `T` is a zero-sized typed. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_wrapping_offset_from)] + /// + /// let a = [0; 5]; + /// let ptr1: *const i32 = &a[1]; + /// let ptr2: *const i32 = &a[3]; + /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// assert_eq!(ptr1.wrapping_offset_from(ptr2), -2); + /// assert_eq!(ptr1.wrapping_offset(2), ptr2); + /// assert_eq!(ptr2.wrapping_offset(-2), ptr1); + /// + /// let ptr1: *const i32 = 3 as _; + /// let ptr2: *const i32 = 13 as _; + /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// ``` + #[unstable(feature = "ptr_wrapping_offset_from", issue = "41079")] + #[inline] + pub fn wrapping_offset_from(self, other: *const T) -> isize where T: Sized { + let pointee_size = mem::size_of::(); + assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize); + + let d = isize::wrapping_sub(self as _, other as _); + d.wrapping_div(pointee_size as _) + } + /// Calculates the offset from a pointer (convenience for `.offset(count as isize)`). /// /// `count` is in units of T; e.g. a `count` of 3 represents a pointer @@ -1347,6 +1455,105 @@ impl *mut T { } } + /// Calculates the distance between two pointers. The returned value is in + /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// + /// This function is the inverse of [`offset`]. + /// + /// [`offset`]: #method.offset-1 + /// [`wrapping_offset_from`]: #method.wrapping_offset_from-1 + /// + /// # Safety + /// + /// If any of the following conditions are violated, the result is Undefined + /// Behavior: + /// + /// * Both the starting and other pointer must be either in bounds or one + /// byte past the end of the same allocated object. + /// + /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. + /// + /// * The distance between the pointers, in bytes, must be an exact multiple + /// of the size of `T` and `T` must not be a Zero-Sized Type ("ZST"). + /// + /// * The distance being in bounds cannot rely on "wrapping around" the address space. + /// + /// The compiler and standard library generally try to ensure allocations + /// never reach a size where an offset is a concern. For instance, `Vec` + /// and `Box` ensure they never allocate more than `isize::MAX` bytes, so + /// `ptr_into_vec.offset_from(vec.as_ptr())` is always safe. + /// + /// Most platforms fundamentally can't even construct such an allocation. + /// For instance, no known 64-bit platform can ever serve a request + /// for 263 bytes due to page-table limitations or splitting the address space. + /// However, some 32-bit and 16-bit platforms may successfully serve a request for + /// more than `isize::MAX` bytes with things like Physical Address + /// Extension. As such, memory acquired directly from allocators or memory + /// mapped files *may* be too large to handle with this function. + /// + /// Consider using [`wrapping_offset_from`] instead if these constraints are + /// difficult to satisfy. The only advantage of this method is that it + /// enables more aggressive compiler optimizations. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_offset_from)] + /// + /// let a = [0; 5]; + /// let ptr1: *mut i32 = &mut a[1]; + /// let ptr2: *mut i32 = &mut a[3]; + /// unsafe { + /// assert_eq!(ptr2.offset_from(ptr1), 2); + /// assert_eq!(ptr1.offset_from(ptr2), -2); + /// assert_eq!(ptr1.offset(2), ptr2); + /// assert_eq!(ptr2.offset(-2), ptr1); + /// } + /// ``` + #[unstable(feature = "ptr_offset_from", issue = "41079")] + #[inline] + pub unsafe fn offset_from(self, other: *const T) -> isize where T: Sized { + (self as *const T).offset_from(other) + } + + /// Calculates the distance between two pointers. The returned value is in + /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// + /// If the address different between the two pointers is not a multiple of + /// `mem::size_of::()` then the result of the division is rounded towards + /// zero. + /// + /// # Panics + /// + /// This function panics if `T` is a zero-sized typed. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_wrapping_offset_from)] + /// + /// let a = [0; 5]; + /// let ptr1: *mut i32 = &mut a[1]; + /// let ptr2: *mut i32 = &mut a[3]; + /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// assert_eq!(ptr1.wrapping_offset_from(ptr2), -2); + /// assert_eq!(ptr1.wrapping_offset(2), ptr2); + /// assert_eq!(ptr2.wrapping_offset(-2), ptr1); + /// + /// let ptr1: *mut i32 = 3 as _; + /// let ptr2: *mut i32 = 13 as _; + /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// ``` + #[unstable(feature = "ptr_wrapping_offset_from", issue = "41079")] + #[inline] + pub fn wrapping_offset_from(self, other: *const T) -> isize where T: Sized { + (self as *const T).wrapping_offset_from(other) + } + /// Computes the byte offset that needs to be applied in order to /// make the pointer aligned to `align`. /// If it is not possible to align the pointer, the implementation returns diff --git a/src/librustc_llvm/ffi.rs b/src/librustc_llvm/ffi.rs index c0cdd21277062..403fe4731f118 100644 --- a/src/librustc_llvm/ffi.rs +++ b/src/librustc_llvm/ffi.rs @@ -935,6 +935,11 @@ extern "C" { RHS: ValueRef, Name: *const c_char) -> ValueRef; + pub fn LLVMBuildExactUDiv(B: BuilderRef, + LHS: ValueRef, + RHS: ValueRef, + Name: *const c_char) + -> ValueRef; pub fn LLVMBuildSDiv(B: BuilderRef, LHS: ValueRef, RHS: ValueRef, diff --git a/src/librustc_trans/builder.rs b/src/librustc_trans/builder.rs index 91eabb9998f4a..5e2d32b359698 100644 --- a/src/librustc_trans/builder.rs +++ b/src/librustc_trans/builder.rs @@ -344,6 +344,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } } + pub fn exactudiv(&self, lhs: ValueRef, rhs: ValueRef) -> ValueRef { + self.count_insn("exactudiv"); + unsafe { + llvm::LLVMBuildExactUDiv(self.llbuilder, lhs, rhs, noname()) + } + } + pub fn sdiv(&self, lhs: ValueRef, rhs: ValueRef) -> ValueRef { self.count_insn("sdiv"); unsafe { diff --git a/src/librustc_trans/intrinsic.rs b/src/librustc_trans/intrinsic.rs index c3de9e0ffcce2..ca5b48be4d546 100644 --- a/src/librustc_trans/intrinsic.rs +++ b/src/librustc_trans/intrinsic.rs @@ -289,7 +289,7 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bx: &Builder<'a, 'tcx>, "ctlz" | "ctlz_nonzero" | "cttz" | "cttz_nonzero" | "ctpop" | "bswap" | "bitreverse" | "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" | "overflowing_add" | "overflowing_sub" | "overflowing_mul" | - "unchecked_div" | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" => { + "unchecked_div" | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" | "exact_div" => { let ty = arg_tys[0]; match int_type_width_signed(ty, cx) { Some((width, signed)) => @@ -343,6 +343,12 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bx: &Builder<'a, 'tcx>, "overflowing_add" => bx.add(args[0].immediate(), args[1].immediate()), "overflowing_sub" => bx.sub(args[0].immediate(), args[1].immediate()), "overflowing_mul" => bx.mul(args[0].immediate(), args[1].immediate()), + "exact_div" => + if signed { + bx.exactsdiv(args[0].immediate(), args[1].immediate()) + } else { + bx.exactudiv(args[0].immediate(), args[1].immediate()) + }, "unchecked_div" => if signed { bx.sdiv(args[0].immediate(), args[1].immediate()) diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index 99707a4a3c0e5..a4e9967daa6e1 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -283,7 +283,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, (1, vec![param(0), param(0)], tcx.intern_tup(&[param(0), tcx.types.bool])), - "unchecked_div" | "unchecked_rem" => + "unchecked_div" | "unchecked_rem" | "exact_div" => (1, vec![param(0), param(0)], param(0)), "unchecked_shl" | "unchecked_shr" => (1, vec![param(0), param(0)], param(0)),