[PERF] Introduce CovariantCopyTaggedPtr and use it in TyKind::TraitObject

compiler/rustc_ast/src/ast.rs

@@ -28,6 +28,7 @@ use rustc_data_structures::packed::Pu128;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::tagged_ptr::Tag;
 use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 pub use rustc_span::AttrId;
 use rustc_span::source_map::{Spanned, respan};
@@ -2269,10 +2270,32 @@ impl TyKind {
 
 /// Syntax used to declare a trait object.
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
+#[repr(u8)]
 pub enum TraitObjectSyntax {
-    Dyn,
-    DynStar,
-    None,
+    // SAFETY: When adding new variants make sure to update the `Tag` impl.
+    Dyn = 0,
+    DynStar = 1,
+    None = 2,
+}
+
+/// SAFETY: `TraitObjectSyntax` only has 3 data-less variants which means
+/// it can be represented with a `u2`. We use `repr(u8)` to guarantee the
+/// discriminants of the variants are no greater than `3`.
+unsafe impl Tag for TraitObjectSyntax {
+    const BITS: u32 = 2;
+
+    fn into_usize(self) -> usize {
+        self as u8 as usize
+    }
+
+    unsafe fn from_usize(tag: usize) -> Self {
+        match tag {
+            0 => TraitObjectSyntax::Dyn,
+            1 => TraitObjectSyntax::DynStar,
+            2 => TraitObjectSyntax::None,
+            _ => unreachable!(),
+        }
+    }
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]

compiler/rustc_ast_lowering/src/lib.rs

@@ -47,6 +47,7 @@ use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::sorted_map::SortedMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::tagged_ptr::CovariantCopyTaggedPtr;
 use rustc_errors::{DiagArgFromDisplay, DiagCtxtHandle, StashKey};
 use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res};
 use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId};
@@ -1157,7 +1158,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 let lifetime_bound = this.elided_dyn_bound(t.span);
                 (bounds, lifetime_bound)
             });
-            let kind = hir::TyKind::TraitObject(bounds, lifetime_bound, TraitObjectSyntax::None);
+            let kind = hir::TyKind::TraitObject(
+                bounds,
+                CovariantCopyTaggedPtr::new(lifetime_bound, TraitObjectSyntax::None),
+            );
             return hir::Ty { kind, span: self.lower_span(t.span), hir_id: self.next_id() };
         }
 
@@ -1308,7 +1312,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         lifetime_bound.unwrap_or_else(|| this.elided_dyn_bound(t.span));
                     (bounds, lifetime_bound)
                 });
-                hir::TyKind::TraitObject(bounds, lifetime_bound, *kind)
+                hir::TyKind::TraitObject(bounds, CovariantCopyTaggedPtr::new(lifetime_bound, *kind))
             }
             TyKind::ImplTrait(def_node_id, bounds) => {
                 let span = t.span;
@@ -2364,8 +2368,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         hir_id = self.next_id();
                         hir::TyKind::TraitObject(
                             arena_vec![self; principal],
-                            self.elided_dyn_bound(span),
-                            TraitObjectSyntax::None,
+                            CovariantCopyTaggedPtr::new(
+                                self.elided_dyn_bound(span),
+                                TraitObjectSyntax::None,
+                            ),
                         )
                     }
                     _ => hir::TyKind::Path(hir::QPath::Resolved(None, path)),

compiler/rustc_borrowck/src/diagnostics/region_errors.rs

@@ -893,7 +893,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 if alias_ty.span.desugaring_kind().is_some() {
                     // Skip `async` desugaring `impl Future`.
                 }
-                if let TyKind::TraitObject(_, lt, _) = alias_ty.kind {
+                if let TyKind::TraitObject(_, lt) = alias_ty.kind {
                     if lt.ident.name == kw::Empty {
                         spans_suggs.push((lt.ident.span.shrink_to_hi(), " + 'a".to_string()));
                     } else {

compiler/rustc_data_structures/src/marker.rs

@@ -73,6 +73,7 @@ impl_dyn_send!(
     [Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend]
     [crate::sync::RwLock<T> where T: DynSend]
     [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool]
+    [crate::tagged_ptr::CovariantCopyTaggedPtr<P, T, CP> where P: Send + crate::tagged_ptr::Pointer<Target: Sized>, T: Send + crate::tagged_ptr::Tag, const CP: bool]
     [rustc_arena::TypedArena<T> where T: DynSend]
     [indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
     [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
@@ -149,6 +150,7 @@ impl_dyn_sync!(
     [crate::sync::WorkerLocal<T> where T: DynSend]
     [crate::intern::Interned<'a, T> where 'a, T: DynSync]
     [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Sync + crate::tagged_ptr::Pointer, T: Sync + crate::tagged_ptr::Tag, const CP: bool]
+    [crate::tagged_ptr::CovariantCopyTaggedPtr<P, T, CP> where P: Sync + crate::tagged_ptr::Pointer<Target: Sized>, T: Sync + crate::tagged_ptr::Tag, const CP: bool]
     [parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
     [parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
     [indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]

compiler/rustc_data_structures/src/tagged_ptr.rs

@@ -23,10 +23,12 @@ use std::sync::Arc;
 use crate::aligned::Aligned;
 
 mod copy;
+mod covariant;
 mod drop;
 mod impl_tag;
 
 pub use copy::CopyTaggedPtr;
+pub use covariant::CovariantCopyTaggedPtr;
 pub use drop::TaggedPtr;
 
 /// This describes the pointer type encapsulated by [`TaggedPtr`] and