Auto merge of #94597 - nnethercote:ConstAllocation, r=fee1-dead

Introduce `ConstAllocation`.

Currently some `Allocation`s are interned, some are not, and it's very
hard to tell at a use point which is which.

This commit introduces `ConstAllocation` for the known-interned ones,
which makes the division much clearer. `ConstAllocation::inner()` is
used to get the underlying `Allocation`.

In some places it's natural to use an `Allocation`, in some it's natural
to use a `ConstAllocation`, and in some places there's no clear choice.
I've tried to make things look as nice as possible, while generally
favouring `ConstAllocation`, which is the type that embodies more
information. This does require quite a few calls to `inner()`.

The commit also tweaks how `PartialOrd` works for `Interned`. The
previous code was too clever by half, building on `T: Ord` to make the
code shorter. That caused problems with deriving `PartialOrd` and `Ord`
for `ConstAllocation`, so I changed it to build on `T: PartialOrd`,
which is slightly more verbose but much more standard and avoided the
problems.

r? `@fee1-dead`

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -4,7 +4,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::ErrorGuaranteed;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{
-    read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar,
+    read_target_uint, AllocId, ConstAllocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar,
 };
 use rustc_middle::ty::ConstKind;
 use rustc_span::DUMMY_SP;
@@ -202,7 +202,7 @@ pub(crate) fn codegen_const_value<'tcx>(
                             &mut fx.constants_cx,
                             fx.module,
                             alloc_id,
-                            alloc.mutability,
+                            alloc.inner().mutability,
                         );
                         let local_data_id =
                             fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
@@ -257,11 +257,15 @@ pub(crate) fn codegen_const_value<'tcx>(
 
 pub(crate) fn pointer_for_allocation<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    alloc: &'tcx Allocation,
+    alloc: ConstAllocation<'tcx>,
 ) -> crate::pointer::Pointer {
     let alloc_id = fx.tcx.create_memory_alloc(alloc);
-    let data_id =
-        data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability);
+    let data_id = data_id_for_alloc_id(
+        &mut fx.constants_cx,
+        &mut *fx.module,
+        alloc_id,
+        alloc.inner().mutability,
+    );
 
     let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     if fx.clif_comments.enabled() {
@@ -361,7 +365,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                 let data_id = *cx.anon_allocs.entry(alloc_id).or_insert_with(|| {
                     module
                         .declare_anonymous_data(
-                            alloc.mutability == rustc_hir::Mutability::Mut,
+                            alloc.inner().mutability == rustc_hir::Mutability::Mut,
                             false,
                         )
                         .unwrap()
@@ -386,6 +390,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
         }
 
         let mut data_ctx = DataContext::new();
+        let alloc = alloc.inner();
         data_ctx.set_align(alloc.align.bytes());
 
         if let Some(section_name) = section_name {
@@ -429,7 +434,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                     continue;
                 }
                 GlobalAlloc::Memory(target_alloc) => {
-                    data_id_for_alloc_id(cx, module, alloc_id, target_alloc.mutability)
+                    data_id_for_alloc_id(cx, module, alloc_id, target_alloc.inner().mutability)
                 }
                 GlobalAlloc::Static(def_id) => {
                     if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -159,7 +159,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let idx_bytes = match idx_const {
                     ConstValue::ByRef { alloc, offset } => {
                         let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
-                        alloc.get_bytes(fx, alloc_range(offset, size)).unwrap()
+                        alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
                     }
                     _ => unreachable!("{:?}", idx_const),
                 };

compiler/rustc_codegen_gcc/src/common.rs

@@ -13,7 +13,7 @@ use rustc_codegen_ssa::traits::{
 use rustc_middle::mir::Mutability;
 use rustc_middle::ty::ScalarInt;
 use rustc_middle::ty::layout::{TyAndLayout, LayoutOf};
-use rustc_middle::mir::interpret::{Allocation, GlobalAlloc, Scalar};
+use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
 use rustc_span::Symbol;
 use rustc_target::abi::{self, HasDataLayout, Pointer, Size};
 
@@ -230,6 +230,7 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                     match self.tcx.global_alloc(alloc_id) {
                         GlobalAlloc::Memory(alloc) => {
                             let init = const_alloc_to_gcc(self, alloc);
+                            let alloc = alloc.inner();
                             let value =
                                 match alloc.mutability {
                                     Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
@@ -262,21 +263,21 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         }
     }
 
-    fn const_data_from_alloc(&self, alloc: &Allocation) -> Self::Value {
+    fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value {
         const_alloc_to_gcc(self, alloc)
     }
 
-    fn from_const_alloc(&self, layout: TyAndLayout<'tcx>, alloc: &Allocation, offset: Size) -> PlaceRef<'tcx, RValue<'gcc>> {
-        assert_eq!(alloc.align, layout.align.abi);
+    fn from_const_alloc(&self, layout: TyAndLayout<'tcx>, alloc: ConstAllocation<'tcx>, offset: Size) -> PlaceRef<'tcx, RValue<'gcc>> {
+        assert_eq!(alloc.inner().align, layout.align.abi);
         let ty = self.type_ptr_to(layout.gcc_type(self, true));
         let value =
             if layout.size == Size::ZERO {
-                let value = self.const_usize(alloc.align.bytes());
+                let value = self.const_usize(alloc.inner().align.bytes());
                 self.context.new_cast(None, value, ty)
             }
             else {
                 let init = const_alloc_to_gcc(self, alloc);
-                let base_addr = self.static_addr_of(init, alloc.align, None);
+                let base_addr = self.static_addr_of(init, alloc.inner().align, None);
 
                 let array = self.const_bitcast(base_addr, self.type_i8p());
                 let value = self.context.new_array_access(None, array, self.const_usize(offset.bytes())).get_address(None);

compiler/rustc_codegen_gcc/src/consts.rs

@@ -7,7 +7,7 @@ use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_middle::ty::layout::LayoutOf;
-use rustc_middle::mir::interpret::{self, Allocation, ErrorHandled, Scalar as InterpScalar, read_target_uint};
+use rustc_middle::mir::interpret::{self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint};
 use rustc_span::Span;
 use rustc_span::def_id::DefId;
 use rustc_target::abi::{self, Align, HasDataLayout, Primitive, Size, WrappingRange};
@@ -284,7 +284,8 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 }
 
-pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: &Allocation) -> RValue<'gcc> {
+pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAllocation<'tcx>) -> RValue<'gcc> {
+    let alloc = alloc.inner();
     let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
     let dl = cx.data_layout();
     let pointer_size = dl.pointer_size.bytes() as usize;
@@ -338,7 +339,7 @@ pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: &Alloca
     cx.const_struct(&llvals, true)
 }
 
-pub fn codegen_static_initializer<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, def_id: DefId) -> Result<(RValue<'gcc>, &'tcx Allocation), ErrorHandled> {
+pub fn codegen_static_initializer<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, def_id: DefId) -> Result<(RValue<'gcc>, ConstAllocation<'tcx>), ErrorHandled> {
     let alloc = cx.tcx.eval_static_initializer(def_id)?;
     Ok((const_alloc_to_gcc(cx, alloc), alloc))
 }

compiler/rustc_codegen_llvm/src/common.rs

@@ -11,7 +11,7 @@ use rustc_ast::Mutability;
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
 use rustc_middle::bug;
-use rustc_middle::mir::interpret::{Allocation, GlobalAlloc, Scalar};
+use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::ScalarInt;
 use rustc_span::symbol::Symbol;
@@ -249,6 +249,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) {
                     GlobalAlloc::Memory(alloc) => {
                         let init = const_alloc_to_llvm(self, alloc);
+                        let alloc = alloc.inner();
                         let value = match alloc.mutability {
                             Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
                             _ => self.static_addr_of(init, alloc.align, None),
@@ -285,24 +286,25 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         }
     }
 
-    fn const_data_from_alloc(&self, alloc: &Allocation) -> Self::Value {
+    fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value {
         const_alloc_to_llvm(self, alloc)
     }
 
     fn from_const_alloc(
         &self,
         layout: TyAndLayout<'tcx>,
-        alloc: &Allocation,
+        alloc: ConstAllocation<'tcx>,
         offset: Size,
     ) -> PlaceRef<'tcx, &'ll Value> {
-        assert_eq!(alloc.align, layout.align.abi);
+        let alloc_align = alloc.inner().align;
+        assert_eq!(alloc_align, layout.align.abi);
         let llty = self.type_ptr_to(layout.llvm_type(self));
         let llval = if layout.size == Size::ZERO {
-            let llval = self.const_usize(alloc.align.bytes());
+            let llval = self.const_usize(alloc_align.bytes());
             unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
         } else {
             let init = const_alloc_to_llvm(self, alloc);
-            let base_addr = self.static_addr_of(init, alloc.align, None);
+            let base_addr = self.static_addr_of(init, alloc_align, None);
 
             let llval = unsafe {
                 llvm::LLVMRustConstInBoundsGEP2(

compiler/rustc_codegen_llvm/src/consts.rs

@@ -12,7 +12,7 @@ use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::mir::interpret::{
-    read_target_uint, Allocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer,
+    read_target_uint, Allocation, ConstAllocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer,
     Scalar as InterpScalar,
 };
 use rustc_middle::mir::mono::MonoItem;
@@ -25,7 +25,8 @@ use rustc_target::abi::{
 use std::ops::Range;
 use tracing::debug;
 
-pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Value {
+pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value {
+    let alloc = alloc.inner();
     let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
     let dl = cx.data_layout();
     let pointer_size = dl.pointer_size.bytes() as usize;
@@ -127,7 +128,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) ->
 pub fn codegen_static_initializer<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     def_id: DefId,
-) -> Result<(&'ll Value, &'tcx Allocation), ErrorHandled> {
+) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> {
     let alloc = cx.tcx.eval_static_initializer(def_id)?;
     Ok((const_alloc_to_llvm(cx, alloc), alloc))
 }
@@ -370,6 +371,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                 // Error has already been reported
                 return;
             };
+            let alloc = alloc.inner();
 
             let g = self.get_static(def_id);
 

compiler/rustc_codegen_ssa/src/traits/consts.rs

@@ -1,6 +1,6 @@
 use super::BackendTypes;
 use crate::mir::place::PlaceRef;
-use rustc_middle::mir::interpret::{Allocation, Scalar};
+use rustc_middle::mir::interpret::{ConstAllocation, Scalar};
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_span::Symbol;
 use rustc_target::abi::{self, Size};
@@ -26,13 +26,13 @@ pub trait ConstMethods<'tcx>: BackendTypes {
     fn const_to_opt_uint(&self, v: Self::Value) -> Option<u64>;
     fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option<u128>;
 
-    fn const_data_from_alloc(&self, alloc: &Allocation) -> Self::Value;
+    fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value;
 
     fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value;
     fn from_const_alloc(
         &self,
         layout: TyAndLayout<'tcx>,
-        alloc: &Allocation,
+        alloc: ConstAllocation<'tcx>,
         offset: Size,
     ) -> PlaceRef<'tcx, Self::Value>;
 

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -367,7 +367,7 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
                             "the raw bytes of the constant ({}",
                             display_allocation(
                                 *ecx.tcx,
-                                ecx.tcx.global_alloc(alloc_id).unwrap_memory()
+                                ecx.tcx.global_alloc(alloc_id).unwrap_memory().inner()
                             )
                         ));
                     },

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -18,8 +18,8 @@ use rustc_target::abi::{Align, Size};
 use rustc_target::spec::abi::Abi;
 
 use crate::interpret::{
-    self, compile_time_machine, AllocId, Allocation, Frame, ImmTy, InterpCx, InterpResult, OpTy,
-    PlaceTy, Scalar, StackPopUnwind,
+    self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult,
+    OpTy, PlaceTy, Scalar, StackPopUnwind,
 };
 
 use super::error::*;
@@ -475,13 +475,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
     fn before_access_global(
         memory_extra: &MemoryExtra,
         alloc_id: AllocId,
-        allocation: &Allocation,
+        alloc: ConstAllocation<'tcx>,
         static_def_id: Option<DefId>,
         is_write: bool,
     ) -> InterpResult<'tcx> {
+        let alloc = alloc.inner();
         if is_write {
             // Write access. These are never allowed, but we give a targeted error message.
-            if allocation.mutability == Mutability::Not {
+            if alloc.mutability == Mutability::Not {
                 Err(err_ub!(WriteToReadOnly(alloc_id)).into())
             } else {
                 Err(ConstEvalErrKind::ModifiedGlobal.into())
@@ -504,7 +505,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
                 // But make sure we never accept a read from something mutable, that would be
                 // unsound. The reason is that as the content of this allocation may be different
                 // now and at run-time, so if we permit reading now we might return the wrong value.
-                assert_eq!(allocation.mutability, Mutability::Not);
+                assert_eq!(alloc.mutability, Mutability::Not);
                 Ok(())
             }
         }

compiler/rustc_const_eval/src/const_eval/mod.rs

@@ -183,7 +183,7 @@ pub(crate) fn deref_const<'tcx>(
     let mplace = ecx.deref_operand(&op).unwrap();
     if let Some(alloc_id) = mplace.ptr.provenance {
         assert_eq!(
-            tcx.get_global_alloc(alloc_id).unwrap().unwrap_memory().mutability,
+            tcx.get_global_alloc(alloc_id).unwrap().unwrap_memory().inner().mutability,
             Mutability::Not,
             "deref_const cannot be used with mutable allocations as \
             that could allow pattern matching to observe mutable statics",

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -23,7 +23,10 @@ use rustc_middle::ty::{self, layout::TyAndLayout, Ty};
 
 use rustc_ast::Mutability;
 
-use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, ValueVisitor};
+use super::{
+    AllocId, Allocation, ConstAllocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy,
+    ValueVisitor,
+};
 use crate::const_eval;
 
 pub trait CompileTimeMachine<'mir, 'tcx, T> = Machine<
@@ -131,8 +134,8 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
         alloc.mutability = Mutability::Not;
     };
     // link the alloc id to the actual allocation
-    let alloc = tcx.intern_const_alloc(alloc);
     leftover_allocations.extend(alloc.relocations().iter().map(|&(_, alloc_id)| alloc_id));
+    let alloc = tcx.intern_const_alloc(alloc);
     tcx.set_alloc_id_memory(alloc_id, alloc);
     None
 }
@@ -393,7 +396,7 @@ pub fn intern_const_alloc_recursive<
             }
             let alloc = tcx.intern_const_alloc(alloc);
             tcx.set_alloc_id_memory(alloc_id, alloc);
-            for &(_, alloc_id) in alloc.relocations().iter() {
+            for &(_, alloc_id) in alloc.inner().relocations().iter() {
                 if leftover_allocations.insert(alloc_id) {
                     todo.push(alloc_id);
                 }
@@ -425,7 +428,7 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
             &mut InterpCx<'mir, 'tcx, M>,
             &PlaceTy<'tcx, M::PointerTag>,
         ) -> InterpResult<'tcx, ()>,
-    ) -> InterpResult<'tcx, &'tcx Allocation> {
+    ) -> InterpResult<'tcx, ConstAllocation<'tcx>> {
         let dest = self.allocate(layout, MemoryKind::Stack)?;
         f(self, &dest.into())?;
         let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1;

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -56,7 +56,7 @@ crate fn eval_nullary_intrinsic<'tcx>(
         sym::type_name => {
             ensure_monomorphic_enough(tcx, tp_ty)?;
             let alloc = type_name::alloc_type_name(tcx, tp_ty);
-            ConstValue::Slice { data: alloc, start: 0, end: alloc.len() }
+            ConstValue::Slice { data: alloc, start: 0, end: alloc.inner().len() }
         }
         sym::needs_drop => {
             ensure_monomorphic_enough(tcx, tp_ty)?;

compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs

@@ -1,6 +1,6 @@
 use rustc_hir::def_id::CrateNum;
 use rustc_hir::definitions::DisambiguatedDefPathData;
-use rustc_middle::mir::interpret::Allocation;
+use rustc_middle::mir::interpret::{Allocation, ConstAllocation};
 use rustc_middle::ty::{
     self,
     print::{PrettyPrinter, Print, Printer},
@@ -188,7 +188,7 @@ impl Write for AbsolutePathPrinter<'_> {
 }
 
 /// Directly returns an `Allocation` containing an absolute path representation of the given type.
-crate fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> &'tcx Allocation {
+crate fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
     let path = AbsolutePathPrinter { tcx, path: String::new() }.print_type(ty).unwrap().path;
     let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
     tcx.intern_const_alloc(alloc)

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -13,8 +13,9 @@ use rustc_target::abi::Size;
 use rustc_target::spec::abi::Abi;
 
 use super::{
-    AllocId, AllocRange, Allocation, Frame, ImmTy, InterpCx, InterpResult, LocalValue, MemPlace,
-    Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, StackPopUnwind,
+    AllocId, AllocRange, Allocation, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult,
+    LocalValue, MemPlace, Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar,
+    StackPopUnwind,
 };
 
 /// Data returned by Machine::stack_pop,
@@ -252,7 +253,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
     fn before_access_global(
         _memory_extra: &Self::MemoryExtra,
         _alloc_id: AllocId,
-        _allocation: &Allocation,
+        _allocation: ConstAllocation<'tcx>,
         _static_def_id: Option<DefId>,
         _is_write: bool,
     ) -> InterpResult<'tcx> {