diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 4977023a668c3..0e7af71169606 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1875,7 +1875,7 @@ macro_rules! slice_interners { List::empty() } else { self.interners.$field.intern_ref(v, || { - InternedInSet(List::from_arena(&*self.arena, v)) + InternedInSet(List::from_arena(&*self.arena, (), v)) }).0 } })+ diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs index 9747a506fa095..ef7a7a99ff7e9 100644 --- a/compiler/rustc_middle/src/ty/impls_ty.rs +++ b/compiler/rustc_middle/src/ty/impls_ty.rs @@ -11,19 +11,20 @@ use rustc_data_structures::stable_hasher::HashingControls; use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey}; use rustc_query_system::ich::StableHashingContext; use std::cell::RefCell; +use std::ptr; -impl<'a, 'tcx, T> HashStable> for &'tcx ty::List +impl<'a, 'tcx, H, T> HashStable> for &'tcx ty::list::RawList where T: HashStable>, { fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) { thread_local! { - static CACHE: RefCell> = + static CACHE: RefCell> = RefCell::new(Default::default()); } let hash = CACHE.with(|cache| { - let key = (self.as_ptr() as usize, self.len(), hcx.hashing_controls()); + let key = (ptr::from_ref(*self).cast::<()>(), hcx.hashing_controls()); if let Some(&hash) = cache.borrow().get(&key) { return hash; } @@ -40,7 +41,7 @@ where } } -impl<'a, 'tcx, T> ToStableHashKey> for &'tcx ty::List +impl<'a, 'tcx, H, T> ToStableHashKey> for &'tcx ty::list::RawList where T: HashStable>, { @@ -55,16 +56,6 @@ where } } -impl<'a, 'tcx, T> HashStable> for &'tcx ty::ListWithCachedTypeInfo -where - T: HashStable>, -{ - #[inline] - fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) { - self.as_list().hash_stable(hcx, hasher); - } -} - impl<'a> ToStableHashKey> for SimplifiedType { type KeyType = Fingerprint; diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs index 08078673d7911..04e165c8a73af 100644 --- a/compiler/rustc_middle/src/ty/list.rs +++ b/compiler/rustc_middle/src/ty/list.rs @@ -32,31 +32,24 @@ use rustc_data_structures::sync::DynSync; /// - `T` must be `Copy`. This lets `List` be stored in a dropless arena and /// iterators return a `T` rather than a `&T`. /// - `T` must not be zero-sized. +pub type List = RawList<(), T>; + +/// A generic type that can be used to prepend a [`List`] with some header. +/// +/// The header will be ignored for value-based operations like [`PartialEq`], +/// [`Hash`] and [`Encodable`]. #[repr(C)] -pub struct List { - skel: ListSkeleton, +pub struct RawList { + skel: ListSkeleton, opaque: OpaqueListContents, } -/// This type defines the statically known field offsets and alignment of a [`List`]. -/// -/// The alignment of a `List` cannot be determined, because it has an extern type tail. -/// We use `ListSkeleton` instead of `List` whenever computing the alignment is required, -/// for example: -/// - Implementing the [`Aligned`] trait, which is required for [`CopyTaggedPtr`]. -/// - Projecting from [`ListWithCachedTypeInfo`] to `List`, which requires computing the padding -/// between the cached type info and the list, which requires computing the list's alignment. -/// -/// Note that `ListSkeleton` is `Sized`, but **it's size is not correct**, as it is missing the -/// dynamically sized list tail. Do not create a `ListSkeleton` on the stack. -/// -/// FIXME: This can be removed once we properly support `!Sized + Aligned + Thin` types. -/// -/// [`CopyTaggedPtr`]: rustc_data_structures::tagged_ptr::CopyTaggedPtr +/// A [`RawList`] without the unsized tail. This type is used for layout computation +/// and constructing empty lists. #[repr(C)] -struct ListSkeleton { +struct ListSkeleton { + header: H, len: usize, - /// Although this claims to be a zero-length array, in practice `len` /// elements are actually present. data: [T; 0], @@ -68,13 +61,7 @@ extern "C" { type OpaqueListContents; } -impl List { - /// Returns a reference to the (unique, static) empty list. - #[inline(always)] - pub fn empty<'a>() -> &'a List { - ListWithCachedTypeInfo::empty() - } - +impl RawList { #[inline(always)] pub fn len(&self) -> usize { self.skel.len @@ -84,9 +71,7 @@ impl List { pub fn as_slice(&self) -> &[T] { self } -} -impl List { /// Allocates a list from `arena` and copies the contents of `slice` into it. /// /// WARNING: the contents *must be unique*, such that no list with these @@ -97,15 +82,26 @@ impl List { /// (because the empty list exists statically, and is available via /// `empty()`). #[inline] - pub(super) fn from_arena<'tcx>(arena: &'tcx Arena<'tcx>, slice: &[T]) -> &'tcx List { + pub(super) fn from_arena<'tcx>( + arena: &'tcx Arena<'tcx>, + header: H, + slice: &[T], + ) -> &'tcx RawList + where + T: Copy, + { assert!(!mem::needs_drop::()); assert!(mem::size_of::() != 0); assert!(!slice.is_empty()); let (layout, _offset) = - Layout::new::().extend(Layout::for_value::<[T]>(slice)).unwrap(); - let mem = arena.dropless.alloc_raw(layout) as *mut List; + Layout::new::>().extend(Layout::for_value::<[T]>(slice)).unwrap(); + + let mem = arena.dropless.alloc_raw(layout) as *mut RawList; unsafe { + // Write the header + ptr::addr_of_mut!((*mem).skel.header).write(header); + // Write the length ptr::addr_of_mut!((*mem).skel.len).write(slice.len()); @@ -123,17 +119,44 @@ impl List { // // This would be weird, as `self.into_iter` iterates over `T` directly. #[inline(always)] - pub fn iter(&self) -> <&'_ List as IntoIterator>::IntoIter { + pub fn iter(&self) -> <&'_ RawList as IntoIterator>::IntoIter + where + T: Copy, + { self.into_iter() } } -impl fmt::Debug for List { +macro_rules! impl_list_empty { + ($header_ty:ty, $header_init:expr) => { + impl RawList<$header_ty, T> { + /// Returns a reference to the (per header unique, static) empty list. + #[inline(always)] + pub fn empty<'a>() -> &'a RawList<$header_ty, T> { + #[repr(align(64))] + struct MaxAlign; + + static EMPTY: ListSkeleton<$header_ty, MaxAlign> = + ListSkeleton { header: $header_init, len: 0, data: [] }; + + assert!(mem::align_of::() <= mem::align_of::()); + + // SAFETY: `EMPTY` is sufficiently aligned to be an empty list for all + // types with `align_of(T) <= align_of(MaxAlign)`, which we checked above. + unsafe { &*(std::ptr::addr_of!(EMPTY) as *const RawList<$header_ty, T>) } + } + } + }; +} + +impl_list_empty!((), ()); + +impl fmt::Debug for RawList { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } } -impl<'tcx, T: DebugWithInfcx>> DebugWithInfcx> for List { +impl<'tcx, H, T: DebugWithInfcx>> DebugWithInfcx> for RawList { fn fmt>>( this: WithInfcx<'_, Infcx, &Self>, f: &mut core::fmt::Formatter<'_>, @@ -142,40 +165,40 @@ impl<'tcx, T: DebugWithInfcx>> DebugWithInfcx> for Lis } } -impl> Encodable for List { +impl> Encodable for RawList { #[inline] fn encode(&self, s: &mut S) { (**self).encode(s); } } -impl PartialEq for List { +impl PartialEq for RawList { #[inline] - fn eq(&self, other: &List) -> bool { + fn eq(&self, other: &RawList) -> bool { // Pointer equality implies list equality (due to the unique contents // assumption). ptr::eq(self, other) } } -impl Eq for List {} +impl Eq for RawList {} -impl Ord for List +impl Ord for RawList where T: Ord, { - fn cmp(&self, other: &List) -> Ordering { + fn cmp(&self, other: &RawList) -> Ordering { // Pointer equality implies list equality (due to the unique contents // assumption), but the contents must be compared otherwise. if self == other { Ordering::Equal } else { <[T] as Ord>::cmp(&**self, &**other) } } } -impl PartialOrd for List +impl PartialOrd for RawList where T: PartialOrd, { - fn partial_cmp(&self, other: &List) -> Option { + fn partial_cmp(&self, other: &RawList) -> Option { // Pointer equality implies list equality (due to the unique contents // assumption), but the contents must be compared otherwise. if self == other { @@ -186,7 +209,7 @@ where } } -impl Hash for List { +impl Hash for RawList { #[inline] fn hash(&self, s: &mut H) { // Pointer hashing is sufficient (due to the unique contents @@ -195,7 +218,7 @@ impl Hash for List { } } -impl Deref for List { +impl Deref for RawList { type Target = [T]; #[inline(always)] fn deref(&self) -> &[T] { @@ -203,7 +226,7 @@ impl Deref for List { } } -impl AsRef<[T]> for List { +impl AsRef<[T]> for RawList { #[inline(always)] fn as_ref(&self) -> &[T] { let data_ptr = ptr::addr_of!(self.skel.data).cast::(); @@ -215,7 +238,7 @@ impl AsRef<[T]> for List { } } -impl<'a, T: Copy> IntoIterator for &'a List { +impl<'a, H, T: Copy> IntoIterator for &'a RawList { type Item = T; type IntoIter = iter::Copied<<&'a [T] as IntoIterator>::IntoIter>; #[inline(always)] @@ -224,225 +247,56 @@ impl<'a, T: Copy> IntoIterator for &'a List { } } -unsafe impl Sync for List {} +unsafe impl Sync for RawList {} // We need this since `List` uses extern type `OpaqueListContents`. #[cfg(parallel_compiler)] -unsafe impl DynSync for List {} +unsafe impl DynSync for RawList {} // Safety: -// Layouts of `ListSkeleton` and `List` are the same, modulo opaque tail, -// thus aligns of `ListSkeleton` and `List` must be the same. -unsafe impl Aligned for List { - const ALIGN: ptr::Alignment = align_of::>(); +// Layouts of `ListSkeleton` and `RawList` are the same, modulo opaque tail, +// thus aligns of `ListSkeleton` and `RawList` must be the same. +unsafe impl Aligned for RawList { + const ALIGN: ptr::Alignment = align_of::>(); } /// A [`List`] that additionally stores type information inline to speed up /// [`TypeVisitableExt`](super::TypeVisitableExt) operations. -#[repr(C)] -pub struct ListWithCachedTypeInfo { - skel: ListWithCachedTypeInfoSkeleton, - opaque: OpaqueListContents, -} - -/// The additional info that is stored in [`ListWithCachedTypeInfo`]. -#[repr(C)] -pub struct TypeInfo { - flags: TypeFlags, - outer_exclusive_binder: DebruijnIndex, -} - -impl From for TypeInfo { - fn from(computation: FlagComputation) -> TypeInfo { - TypeInfo { - flags: computation.flags, - outer_exclusive_binder: computation.outer_exclusive_binder, - } - } -} - -/// This type is similar to [`ListSkeleton`], but for [`ListWithCachedTypeInfo`]. -/// It is used for computing the alignment of a [`ListWithCachedTypeInfo`]. -#[repr(C)] -struct ListWithCachedTypeInfoSkeleton { - info: TypeInfo, - // N.B.: There may be padding between these two fields. We cannot use `List` directly - // here, because it has an unknown alignment which makes computing the amount of padding - // and therefore projecting from `&ListWithCachedTypeInfo` to `&List` impossible. - list: ListSkeleton, -} +pub type ListWithCachedTypeInfo = RawList; impl ListWithCachedTypeInfo { - #[inline(always)] - pub fn empty<'a>() -> &'a ListWithCachedTypeInfo { - #[repr(align(64))] - struct MaxAlign; - - #[repr(C)] - struct Empty { - info: TypeInfo, - zero: [u8; 2 * mem::align_of::() - mem::size_of::()], - align: MaxAlign, - } - - static EMPTY: Empty = Empty { - info: TypeInfo { flags: TypeFlags::empty(), outer_exclusive_binder: super::INNERMOST }, - zero: [0; 2 * mem::align_of::() - mem::size_of::()], - align: MaxAlign, - }; - - assert!(mem::align_of::() <= mem::align_of::()); - - // The layout of the empty `ListWithCachedTypeInfo` must be one of the following, - // depending on the alignment of `T`: - // - // On 64-bit platforms: - // F = flags (32 bit), B = outer_exclusive_binder (32 bit), LL = len (64 bit) - // align(T) <= 8: FBLL - // align(T) = 16: FB..LL.. - // align(T) = 32: FB......LL...... - // align(T) = 64: FB..............LL.............. - // - // On 32-bit platforms: - // F = flags (32 bit), B = outer_exclusive_binder (32 bit), L = len (32 bit) - // align(T) <= 4: FBL - // align(T) = 8: FBL. - // align(T) = 16: FB..L... - // align(T) = 32: FB......L....... - // align(T) = 64: FB..............L............... - // - // We zero out every possible location of `len` so that `EMPTY` is a valid - // `ListWithCachedTypeInfo` for all `T` with alignment up to 64 bytes. - unsafe { &*(std::ptr::addr_of!(EMPTY) as *const ListWithCachedTypeInfo) } - } - - #[inline] - pub(super) fn from_arena<'tcx>( - arena: &'tcx Arena<'tcx>, - info: TypeInfo, - slice: &[T], - ) -> &'tcx ListWithCachedTypeInfo - where - T: Copy, - { - assert!(!mem::needs_drop::()); - assert!(mem::size_of::() != 0); - assert!(!slice.is_empty()); - - let (list_layout, _offset) = - Layout::new::().extend(Layout::for_value::<[T]>(slice)).unwrap(); - - let (layout, _offset) = Layout::new::().extend(list_layout).unwrap(); - - let mem = arena.dropless.alloc_raw(layout) as *mut ListWithCachedTypeInfo; - unsafe { - // Write the cached type info - ptr::addr_of_mut!((*mem).skel.info).write(info); - - // Write the length - ptr::addr_of_mut!((*mem).skel.list.len).write(slice.len()); - - // Write the elements - ptr::addr_of_mut!((*mem).skel.list.data) - .cast::() - .copy_from_nonoverlapping(slice.as_ptr(), slice.len()); - - &*mem - } - } - - #[inline(always)] - pub fn as_list(&self) -> &List { - self - } - #[inline(always)] pub fn flags(&self) -> TypeFlags { - self.skel.info.flags + self.skel.header.flags } #[inline(always)] pub fn outer_exclusive_binder(&self) -> DebruijnIndex { - self.skel.info.outer_exclusive_binder - } -} - -impl Deref for ListWithCachedTypeInfo { - type Target = List; - #[inline(always)] - fn deref(&self) -> &List { - let list_ptr = ptr::addr_of!(self.skel.list) as *const List; - unsafe { &*list_ptr } - } -} - -impl AsRef<[T]> for ListWithCachedTypeInfo { - #[inline(always)] - fn as_ref(&self) -> &[T] { - (&**self).as_ref() + self.skel.header.outer_exclusive_binder } } -impl fmt::Debug for ListWithCachedTypeInfo { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (**self).fmt(f) - } -} -impl<'tcx, T: DebugWithInfcx>> DebugWithInfcx> - for ListWithCachedTypeInfo -{ - fn fmt>>( - this: WithInfcx<'_, Infcx, &Self>, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - DebugWithInfcx::fmt(this.map(|this| &**this), f) - } -} +impl_list_empty!(TypeInfo, TypeInfo::empty()); -impl> Encodable for ListWithCachedTypeInfo { - #[inline] - fn encode(&self, s: &mut S) { - (**self).encode(s); - } +/// The additional info that is stored in [`ListWithCachedTypeInfo`]. +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TypeInfo { + flags: TypeFlags, + outer_exclusive_binder: DebruijnIndex, } -impl PartialEq for ListWithCachedTypeInfo { - #[inline] - fn eq(&self, other: &ListWithCachedTypeInfo) -> bool { - // Pointer equality implies list equality (due to the unique contents - // assumption). - ptr::eq(self, other) +impl TypeInfo { + const fn empty() -> Self { + Self { flags: TypeFlags::empty(), outer_exclusive_binder: super::INNERMOST } } } -impl Eq for ListWithCachedTypeInfo {} - -impl Hash for ListWithCachedTypeInfo { - #[inline] - fn hash(&self, s: &mut H) { - // Pointer hashing is sufficient (due to the unique contents - // assumption). - ptr::from_ref(self).hash(s) - } -} - -impl<'a, T: Copy> IntoIterator for &'a ListWithCachedTypeInfo { - type Item = T; - type IntoIter = iter::Copied<<&'a [T] as IntoIterator>::IntoIter>; - #[inline(always)] - fn into_iter(self) -> Self::IntoIter { - (**self).into_iter() +impl From for TypeInfo { + fn from(computation: FlagComputation) -> TypeInfo { + TypeInfo { + flags: computation.flags, + outer_exclusive_binder: computation.outer_exclusive_binder, + } } } - -unsafe impl Sync for ListWithCachedTypeInfo {} - -#[cfg(parallel_compiler)] -unsafe impl DynSync for ListWithCachedTypeInfo {} - -// Safety: -// Layouts of `ListWithCachedTypeInfoSkeleton` and `ListWithCachedTypeInfo` -// are the same, modulo opaque tail, thus their aligns must be the same. -unsafe impl Aligned for ListWithCachedTypeInfo { - const ALIGN: ptr::Alignment = align_of::>(); -}