Auto merge of #120594 - saethlin:delayed-debug-asserts, r=oli-obk

Toggle assert_unsafe_precondition in codegen instead of expansion

The goal of this PR is to make some of the unsafe precondition checks in the standard library available in debug builds. Some UI tests are included to verify that it does that.

The diff is large, but most of it is blessing mir-opt tests and I've also split up this PR so it can be reviewed commit-by-commit.

This PR:
1. Adds a new intrinsic, `debug_assertions` which is lowered to a new MIR NullOp, and only to a constant after monomorphization
2. Rewrites `assume_unsafe_precondition` to check the new intrinsic, and be monomorphic.
3. Skips codegen of the `assume` intrinsic in unoptimized builds, because that was silly before but with these checks it's *very* silly
4. The checks with the most overhead are `ptr::read`/`ptr::write` and `NonNull::new_unchecked`. I've simply added `#[cfg(debug_assertions)]` to the checks for `ptr::read`/`ptr::write` because I was unable to come up with any (good) ideas for decreasing their impact. But for `NonNull::new_unchecked` I found that the majority of callers can use a different function, often a safe one.

Yes, this PR slows down the compile time of some programs. But in our benchmark suite it's never more than 1% icount, and the average icount change in debug-full programs is 0.22%. I think that is acceptable for such an improvement in developer experience.

#120539 (comment)

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1983,6 +1983,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     ConstraintCategory::SizedBound,
                 );
             }
+            &Rvalue::NullaryOp(NullOp::DebugAssertions, _) => {}
 
             Rvalue::ShallowInitBox(operand, ty) => {
                 self.check_operand(operand, location);

compiler/rustc_codegen_cranelift/src/base.rs

@@ -767,6 +767,15 @@ fn codegen_stmt<'tcx>(
                         NullOp::OffsetOf(fields) => {
                             layout.offset_of_subfield(fx, fields.iter()).bytes()
                         }
+                        NullOp::DebugAssertions => {
+                            let val = fx.tcx.sess.opts.debug_assertions;
+                            let val = CValue::by_val(
+                                fx.bcx.ins().iconst(types::I8, i64::try_from(val).unwrap()),
+                                fx.layout_of(fx.tcx.types.bool),
+                            );
+                            lval.write_cvalue(fx, val);
+                            return;
+                        }
                     };
                     let val = CValue::by_val(
                         fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(val).unwrap()),

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -672,17 +672,23 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let val = match null_op {
                     mir::NullOp::SizeOf => {
                         assert!(bx.cx().type_is_sized(ty));
-                        layout.size.bytes()
+                        let val = layout.size.bytes();
+                        bx.cx().const_usize(val)
                     }
                     mir::NullOp::AlignOf => {
                         assert!(bx.cx().type_is_sized(ty));
-                        layout.align.abi.bytes()
+                        let val = layout.align.abi.bytes();
+                        bx.cx().const_usize(val)
                     }
                     mir::NullOp::OffsetOf(fields) => {
-                        layout.offset_of_subfield(bx.cx(), fields.iter()).bytes()
+                        let val = layout.offset_of_subfield(bx.cx(), fields.iter()).bytes();
+                        bx.cx().const_usize(val)
+                    }
+                    mir::NullOp::DebugAssertions => {
+                        let val = bx.tcx().sess.opts.debug_assertions;
+                        bx.cx().const_bool(val)
                     }
                 };
-                let val = bx.cx().const_usize(val);
                 let tcx = self.cx.tcx();
                 OperandRef {
                     val: OperandValue::Immediate(val),

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -1,5 +1,6 @@
 use rustc_middle::mir;
 use rustc_middle::mir::NonDivergingIntrinsic;
+use rustc_session::config::OptLevel;
 
 use super::FunctionCx;
 use super::LocalRef;
@@ -67,8 +68,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_coverage(bx, coverage, statement.source_info.scope);
             }
             mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
-                let op_val = self.codegen_operand(bx, op);
-                bx.assume(op_val.immediate());
+                if !matches!(bx.tcx().sess.opts.optimize, OptLevel::No | OptLevel::Less) {
+                    let op_val = self.codegen_operand(bx, op);
+                    bx.assume(op_val.immediate());
+                }
             }
             mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
                 mir::CopyNonOverlapping { ref count, ref src, ref dst },

compiler/rustc_const_eval/src/interpret/step.rs

@@ -246,13 +246,25 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     );
                 }
                 let val = match null_op {
-                    mir::NullOp::SizeOf => layout.size.bytes(),
-                    mir::NullOp::AlignOf => layout.align.abi.bytes(),
+                    mir::NullOp::SizeOf => {
+                        let val = layout.size.bytes();
+                        Scalar::from_target_usize(val, self)
+                    }
+                    mir::NullOp::AlignOf => {
+                        let val = layout.align.abi.bytes();
+                        Scalar::from_target_usize(val, self)
+                    }
                     mir::NullOp::OffsetOf(fields) => {
-                        layout.offset_of_subfield(self, fields.iter()).bytes()
+                        let val = layout.offset_of_subfield(self, fields.iter()).bytes();
+                        Scalar::from_target_usize(val, self)
+                    }
+                    mir::NullOp::DebugAssertions => {
+                        // The checks hidden behind this are always better done by the interpreter
+                        // itself, because it knows the runtime state better.
+                        Scalar::from_bool(false)
                     }
                 };
-                self.write_scalar(Scalar::from_target_usize(val, self), &dest)?;
+                self.write_scalar(val, &dest)?;
             }
 
             ShallowInitBox(ref operand, _) => {

compiler/rustc_const_eval/src/transform/check_consts/check.rs

@@ -544,7 +544,10 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
             Rvalue::Cast(_, _, _) => {}
 
-            Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(_), _) => {}
+            Rvalue::NullaryOp(
+                NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(_) | NullOp::DebugAssertions,
+                _,
+            ) => {}
             Rvalue::ShallowInitBox(_, _) => {}
 
             Rvalue::UnaryOp(_, operand) => {

compiler/rustc_const_eval/src/transform/validate.rs

@@ -1139,7 +1139,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             Rvalue::Repeat(_, _)
             | Rvalue::ThreadLocalRef(_)
             | Rvalue::AddressOf(_, _)
-            | Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, _)
+            | Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::DebugAssertions, _)
             | Rvalue::Discriminant(_) => {}
         }
         self.super_rvalue(rvalue, location);

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -112,7 +112,8 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir
         | sym::forget
         | sym::black_box
         | sym::variant_count
-        | sym::ptr_mask => hir::Unsafety::Normal,
+        | sym::ptr_mask
+        | sym::debug_assertions => hir::Unsafety::Normal,
         _ => hir::Unsafety::Unsafe,
     };
 
@@ -461,6 +462,8 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                 (0, vec![Ty::new_imm_ptr(tcx, Ty::new_unit(tcx))], tcx.types.usize)
             }
 
+            sym::debug_assertions => (0, Vec::new(), tcx.types.bool),
+
             other => {
                 tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span: it.span, name: other });
                 return;

compiler/rustc_middle/src/mir/pretty.rs

@@ -907,6 +907,7 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                     NullOp::SizeOf => write!(fmt, "SizeOf({t})"),
                     NullOp::AlignOf => write!(fmt, "AlignOf({t})"),
                     NullOp::OffsetOf(fields) => write!(fmt, "OffsetOf({t}, {fields:?})"),
+                    NullOp::DebugAssertions => write!(fmt, "cfg!(debug_assertions)"),
                 }
             }
             ThreadLocalRef(did) => ty::tls::with(|tcx| {

compiler/rustc_middle/src/mir/syntax.rs

@@ -1361,6 +1361,8 @@ pub enum NullOp<'tcx> {
     AlignOf,
     /// Returns the offset of a field
     OffsetOf(&'tcx List<(VariantIdx, FieldIdx)>),
+    /// cfg!(debug_assertions), but expanded in codegen
+    DebugAssertions,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]

compiler/rustc_middle/src/mir/tcx.rs

@@ -194,6 +194,7 @@ impl<'tcx> Rvalue<'tcx> {
             Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {
                 tcx.types.usize
             }
+            Rvalue::NullaryOp(NullOp::DebugAssertions, _) => tcx.types.bool,
             Rvalue::Aggregate(ref ak, ref ops) => match **ak {
                 AggregateKind::Array(ty) => Ty::new_array(tcx, ty, ops.len() as u64),
                 AggregateKind::Tuple => {

compiler/rustc_mir_dataflow/src/move_paths/builder.rs

@@ -429,7 +429,10 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
             | Rvalue::AddressOf(..)
             | Rvalue::Discriminant(..)
             | Rvalue::Len(..)
-            | Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {}
+            | Rvalue::NullaryOp(
+                NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..) | NullOp::DebugAssertions,
+                _,
+            ) => {}
         }
     }
 

compiler/rustc_mir_transform/src/const_prop_lint.rs

@@ -634,6 +634,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                     NullOp::OffsetOf(fields) => {
                         op_layout.offset_of_subfield(self, fields.iter()).bytes()
                     }
+                    NullOp::DebugAssertions => return None,
                 };
                 ImmTy::from_scalar(Scalar::from_target_usize(val, self), layout).into()
             }

compiler/rustc_mir_transform/src/gvn.rs

@@ -489,6 +489,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     NullOp::OffsetOf(fields) => {
                         layout.offset_of_subfield(&self.ecx, fields.iter()).bytes()
                     }
+                    NullOp::DebugAssertions => return None,
                 };
                 let usize_layout = self.ecx.layout_of(self.tcx.types.usize).unwrap();
                 let imm = ImmTy::try_from_uint(val, usize_layout)?;

compiler/rustc_mir_transform/src/instsimplify.rs

@@ -2,10 +2,12 @@
 
 use crate::simplify::simplify_duplicate_switch_targets;
 use rustc_middle::mir::*;
+use rustc_middle::ty::layout;
 use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::{self, GenericArgsRef, ParamEnv, Ty, TyCtxt};
 use rustc_span::symbol::Symbol;
 use rustc_target::abi::FieldIdx;
+use rustc_target::spec::abi::Abi;
 
 pub struct InstSimplify;
 
@@ -38,6 +40,7 @@ impl<'tcx> MirPass<'tcx> for InstSimplify {
                 block.terminator.as_mut().unwrap(),
                 &mut block.statements,
             );
+            ctx.simplify_nounwind_call(block.terminator.as_mut().unwrap());
             simplify_duplicate_switch_targets(block.terminator.as_mut().unwrap());
         }
     }
@@ -252,6 +255,28 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
         terminator.kind = TerminatorKind::Goto { target: destination_block };
     }
 
+    fn simplify_nounwind_call(&self, terminator: &mut Terminator<'tcx>) {
+        let TerminatorKind::Call { func, unwind, .. } = &mut terminator.kind else {
+            return;
+        };
+
+        let Some((def_id, _)) = func.const_fn_def() else {
+            return;
+        };
+
+        let body_ty = self.tcx.type_of(def_id).skip_binder();
+        let body_abi = match body_ty.kind() {
+            ty::FnDef(..) => body_ty.fn_sig(self.tcx).abi(),
+            ty::Closure(..) => Abi::RustCall,
+            ty::Coroutine(..) => Abi::Rust,
+            _ => bug!("unexpected body ty: {:?}", body_ty),
+        };
+
+        if !layout::fn_can_unwind(self.tcx, Some(def_id), body_abi) {
+            *unwind = UnwindAction::Unreachable;
+        }
+    }
+
     fn simplify_intrinsic_assert(
         &self,
         terminator: &mut Terminator<'tcx>,

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -21,6 +21,17 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                     sym::unreachable => {
                         terminator.kind = TerminatorKind::Unreachable;
                     }
+                    sym::debug_assertions => {
+                        let target = target.unwrap();
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::NullaryOp(NullOp::DebugAssertions, tcx.types.bool),
+                            ))),
+                        });
+                        terminator.kind = TerminatorKind::Goto { target };
+                    }
                     sym::forget => {
                         if let Some(target) = *target {
                             block.statements.push(Statement {

compiler/rustc_mir_transform/src/promote_consts.rs

@@ -446,6 +446,7 @@ impl<'tcx> Validator<'_, 'tcx> {
                 NullOp::SizeOf => {}
                 NullOp::AlignOf => {}
                 NullOp::OffsetOf(_) => {}
+                NullOp::DebugAssertions => {}
             },
 
             Rvalue::ShallowInitBox(_, _) => return Err(Unpromotable),

compiler/rustc_smir/src/rustc_smir/convert/mir.rs

@@ -257,6 +257,7 @@ impl<'tcx> Stable<'tcx> for mir::NullOp<'tcx> {
             OffsetOf(indices) => stable_mir::mir::NullOp::OffsetOf(
                 indices.iter().map(|idx| idx.stable(tables)).collect(),
             ),
+            DebugAssertions => stable_mir::mir::NullOp::DebugAssertions,
         }
     }
 }

compiler/stable_mir/src/mir/body.rs

@@ -639,6 +639,7 @@ impl Rvalue {
             Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {
                 Ok(Ty::usize_ty())
             }
+            Rvalue::NullaryOp(NullOp::DebugAssertions, _) => Ok(Ty::bool_ty()),
             Rvalue::Aggregate(ak, ops) => match *ak {
                 AggregateKind::Array(ty) => Ty::try_new_array(ty, ops.len() as u64),
                 AggregateKind::Tuple => Ok(Ty::new_tuple(
@@ -1005,6 +1006,8 @@ pub enum NullOp {
     AlignOf,
     /// Returns the offset of a field.
     OffsetOf(Vec<(VariantIdx, FieldIdx)>),
+    /// cfg!(debug_assertions), but at codegen time
+    DebugAssertions,
 }
 
 impl Operand {

library/alloc/src/raw_vec.rs

@@ -207,11 +207,7 @@ impl<T, A: Allocator> RawVec<T, A> {
             // Allocators currently return a `NonNull<[u8]>` whose length
             // matches the size requested. If that ever changes, the capacity
             // here should change to `ptr.len() / mem::size_of::<T>()`.
-            Self {
-                ptr: unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) },
-                cap: unsafe { Cap(capacity) },
-                alloc,
-            }
+            Self { ptr: Unique::from(ptr.cast()), cap: unsafe { Cap(capacity) }, alloc }
         }
     }
 
@@ -239,6 +235,11 @@ impl<T, A: Allocator> RawVec<T, A> {
         self.ptr.as_ptr()
     }
 
+    #[inline]
+    pub fn non_null(&self) -> NonNull<T> {
+        NonNull::from(self.ptr)
+    }
+
     /// Gets the capacity of the allocation.
     ///
     /// This will always be `usize::MAX` if `T` is zero-sized.
@@ -398,7 +399,7 @@ impl<T, A: Allocator> RawVec<T, A> {
         // Allocators currently return a `NonNull<[u8]>` whose length matches
         // the size requested. If that ever changes, the capacity here should
         // change to `ptr.len() / mem::size_of::<T>()`.
-        self.ptr = unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) };
+        self.ptr = Unique::from(ptr.cast());
         self.cap = unsafe { Cap(cap) };
     }
 

library/alloc/src/vec/into_iter.rs

@@ -136,7 +136,7 @@ impl<T, A: Allocator> IntoIter<T, A> {
         // struct and then overwriting &mut self.
         // this creates less assembly
         self.cap = 0;
-        self.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) };
+        self.buf = RawVec::NEW.non_null();
         self.ptr = self.buf;
         self.end = self.buf.as_ptr();
 

library/alloc/src/vec/mod.rs

@@ -2861,16 +2861,16 @@ impl<T, A: Allocator> IntoIterator for Vec<T, A> {
     #[inline]
     fn into_iter(self) -> Self::IntoIter {
         unsafe {
-            let mut me = ManuallyDrop::new(self);
+            let me = ManuallyDrop::new(self);
             let alloc = ManuallyDrop::new(ptr::read(me.allocator()));
-            let begin = me.as_mut_ptr();
+            let buf = me.buf.non_null();
+            let begin = buf.as_ptr();
             let end = if T::IS_ZST {
                 begin.wrapping_byte_add(me.len())
             } else {
                 begin.add(me.len()) as *const T
             };
             let cap = me.buf.capacity();
-            let buf = NonNull::new_unchecked(begin);
             IntoIter { buf, phantom: PhantomData, cap, alloc, ptr: buf, end }
         }
     }

library/core/src/char/convert.rs

@@ -27,7 +27,7 @@ pub(super) const unsafe fn from_u32_unchecked(i: u32) -> char {
     unsafe {
         assert_unsafe_precondition!(
             "invalid value for `char`",
-            (i: u32) => char_try_from_u32(i).is_ok()
+            (i: u32 = i) => char_try_from_u32(i).is_ok()
         );
         transmute(i)
     }

library/core/src/hint.rs

@@ -148,7 +148,7 @@ pub const unsafe fn assert_unchecked(cond: bool) {
     unsafe {
         intrinsics::assert_unsafe_precondition!(
             "hint::assert_unchecked must never be called when the condition is false",
-            (cond: bool) => cond,
+            (cond: bool = cond) => cond,
         );
         crate::intrinsics::assume(cond);
     }