Remove unwraps from metadata decoding: introduce BlobDecoder

compiler/rustc_hir/src/definitions.rs

@@ -11,7 +11,7 @@ use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_data_structures::unord::UnordMap;
 use rustc_hashes::Hash64;
 use rustc_index::IndexVec;
-use rustc_macros::{Decodable, Encodable};
+use rustc_macros::{BlobDecodable, Decodable, Encodable};
 use rustc_span::{Symbol, kw, sym};
 use tracing::{debug, instrument};
 
@@ -127,7 +127,7 @@ pub struct Definitions {
 /// A unique identifier that we can use to lookup a definition
 /// precisely. It combines the index of the definition's parent (if
 /// any) with a `DisambiguatedDefPathData`.
-#[derive(Copy, Clone, PartialEq, Debug, Encodable, Decodable)]
+#[derive(Copy, Clone, PartialEq, Debug, Encodable, BlobDecodable)]
 pub struct DefKey {
     /// The parent path.
     pub parent: Option<DefIndex>,
@@ -176,7 +176,7 @@ impl DefKey {
 /// between them. This introduces some artificial ordering dependency
 /// but means that if you have, e.g., two impls for the same type in
 /// the same module, they do get distinct `DefId`s.
-#[derive(Copy, Clone, PartialEq, Debug, Encodable, Decodable)]
+#[derive(Copy, Clone, PartialEq, Debug, Encodable, BlobDecodable)]
 pub struct DisambiguatedDefPathData {
     pub data: DefPathData,
     pub disambiguator: u32,
@@ -270,7 +270,7 @@ impl DefPath {
 }
 
 /// New variants should only be added in synchronization with `enum DefKind`.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, BlobDecodable)]
 pub enum DefPathData {
     // Root: these should only be used for the root nodes, because
     // they are treated specially by the `def_path` function.

compiler/rustc_hir/src/lang_items.rs

@@ -10,7 +10,7 @@
 use rustc_ast::attr::AttributeExt;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_macros::{Decodable, Encodable, HashStable_Generic};
+use rustc_macros::{BlobDecodable, Encodable, HashStable_Generic};
 use rustc_span::{Span, Symbol, kw, sym};
 
 use crate::def_id::DefId;
@@ -75,7 +75,7 @@ macro_rules! language_item_table {
         $( $(#[$attr:meta])* $variant:ident, $module:ident :: $name:ident, $method:ident, $target:expr, $generics:expr; )*
     ) => {
         /// A representation of all the valid lang items in Rust.
-        #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable)]
+        #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Encodable, BlobDecodable)]
         pub enum LangItem {
             $(
                 #[doc = concat!("The `", stringify!($name), "` lang item.")]

compiler/rustc_hir/src/stability.rs

@@ -1,6 +1,6 @@
 use std::num::NonZero;
 
-use rustc_macros::{Decodable, Encodable, HashStable_Generic, PrintAttribute};
+use rustc_macros::{BlobDecodable, Decodable, Encodable, HashStable_Generic, PrintAttribute};
 use rustc_span::{ErrorGuaranteed, Symbol, sym};
 
 use crate::RustcVersion;
@@ -21,7 +21,7 @@ pub const VERSION_PLACEHOLDER: &str = concat!("CURRENT_RUSTC_VERSIO", "N");
 ///
 /// - `#[stable]`
 /// - `#[unstable]`
-#[derive(Encodable, Decodable, Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Encodable, BlobDecodable, Copy, Clone, Debug, PartialEq, Eq, Hash)]
 #[derive(HashStable_Generic, PrintAttribute)]
 pub struct Stability {
     pub level: StabilityLevel,
@@ -103,7 +103,7 @@ impl PartialConstStability {
 }
 
 /// The available stability levels.
-#[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, Hash)]
+#[derive(Encodable, BlobDecodable, PartialEq, Copy, Clone, Debug, Eq, Hash)]
 #[derive(HashStable_Generic, PrintAttribute)]
 pub enum StabilityLevel {
     /// `#[unstable]`
@@ -146,7 +146,7 @@ pub enum StabilityLevel {
 }
 
 /// Rust release in which a feature is stabilized.
-#[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, PartialOrd, Ord, Hash)]
+#[derive(Encodable, BlobDecodable, PartialEq, Copy, Clone, Debug, Eq, PartialOrd, Ord, Hash)]
 #[derive(HashStable_Generic, PrintAttribute)]
 pub enum StableSince {
     /// also stores the original symbol for printing
@@ -172,7 +172,7 @@ impl StabilityLevel {
     }
 }
 
-#[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, Hash)]
+#[derive(Encodable, BlobDecodable, PartialEq, Copy, Clone, Debug, Eq, Hash)]
 #[derive(HashStable_Generic, PrintAttribute)]
 pub enum UnstableReason {
     None,

compiler/rustc_hir/src/version.rs

@@ -4,12 +4,12 @@ use std::sync::OnceLock;
 
 use rustc_error_messages::{DiagArgValue, IntoDiagArg};
 use rustc_macros::{
-    Decodable, Encodable, HashStable_Generic, PrintAttribute, current_rustc_version,
+    BlobDecodable, Encodable, HashStable_Generic, PrintAttribute, current_rustc_version,
 };
 
 use crate::attrs::PrintAttribute;
 
-#[derive(Encodable, Decodable, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Encodable, BlobDecodable, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[derive(HashStable_Generic, PrintAttribute)]
 pub struct RustcVersion {
     pub major: u16,

compiler/rustc_macros/src/lib.rs

@@ -72,14 +72,77 @@ decl_derive!(
     hash_stable::hash_stable_no_context_derive
 );
 
-decl_derive!([Decodable_NoContext] => serialize::decodable_nocontext_derive);
+// Encoding and Decoding derives
+decl_derive!([Decodable_NoContext] =>
+    /// See docs on derive [`Decodable`].
+    ///
+    /// Derives `Decodable<D> for T where D: Decoder`.
+    serialize::decodable_nocontext_derive
+);
 decl_derive!([Encodable_NoContext] => serialize::encodable_nocontext_derive);
-decl_derive!([Decodable] => serialize::decodable_derive);
+decl_derive!([Decodable] =>
+    /// Derives `Decodable<D> for T where D: SpanDecoder`
+    ///
+    /// # Deriving decoding traits
+    ///
+    /// > Some shared docs about decoding traits, since this is likely the first trait you find
+    ///
+    /// The difference between theses derives can be subtle!
+    /// At a high level, there's the `T: Decodable<D>` trait that says some type `T`
+    /// can be decoded using a decoder `D`. There are various decoders!
+    /// The different derives place different *trait* bounds on this type `D`.
+    ///
+    /// Even though this derive, based on its name, seems like the most vanilla one,
+    /// it actually places a pretty strict bound on `D`: `SpanDecoder`.
+    /// It means that types that derive this can contain spans, among other things,
+    /// and still be decoded. The reason this is hard is that at least in metadata,
+    /// spoans can only be decoded later, once some information from the header
+    /// is already decoded to properly deal with spans.
+    ///
+    /// The hierarchy is roughly:
+    ///
+    /// - derive [`Decodable_NoContext`] is the most relaxed bounds that could be placed on `D`,
+    ///   and is only really suited for structs and enums containing primitive types.
+    /// - derive [`BlobDecodable`] may be a better default, than deriving `Decodable`:
+    ///   it places fewer requirements on `D`, while still allowing some complex types to be decoded.
+    /// - derive [`LazyDecodable`]: Only for types containing `Lazy{Array,Table,Value}`.
+    /// - derive [`Decodable`] for structures containing spans. Requires `D: SpanDecoder`
+    /// - derive [`TyDecodable`] for types that require access to the `TyCtxt` while decoding.
+    ///   For example: arena allocated types.
+    serialize::decodable_derive
+);
 decl_derive!([Encodable] => serialize::encodable_derive);
-decl_derive!([TyDecodable] => serialize::type_decodable_derive);
+decl_derive!([TyDecodable] =>
+    /// See docs on derive [`Decodable`].
+    ///
+    /// Derives `Decodable<D> for T where D: TyDecoder`.
+    serialize::type_decodable_derive
+);
 decl_derive!([TyEncodable] => serialize::type_encodable_derive);
-decl_derive!([MetadataDecodable] => serialize::meta_decodable_derive);
-decl_derive!([MetadataEncodable] => serialize::meta_encodable_derive);
+decl_derive!([LazyDecodable] =>
+    /// See docs on derive [`Decodable`].
+    ///
+    /// Derives `Decodable<D> for T where D: LazyDecoder`.
+    /// This constrains the decoder to be specifically the decoder that can decode
+    /// `LazyArray`s, `LazyValue`s amd `LazyTable`s in metadata.
+    /// Therefore, we only need this on things containing LazyArray really.
+    ///
+    /// Most decodable derives mirror an encodable derive.
+    /// [`LazyDecodable`] and [`BlobDecodable`] together roughly mirror [`MetadataEncodable`]
+    serialize::lazy_decodable_derive
+);
+decl_derive!([BlobDecodable] =>
+    /// See docs on derive [`Decodable`].
+    ///
+    /// Derives `Decodable<D> for T where D: BlobDecoder`.
+    serialize::blob_decodable_derive
+);
+decl_derive!([MetadataEncodable] =>
+    /// Most encodable derives mirror a decodable derive.
+    /// [`MetadataEncodable`] is roughly mirrored by the combination of [`LazyDecodable`] and [`BlobDecodable`]
+    serialize::meta_encodable_derive
+);
+
 decl_derive!(
     [TypeFoldable, attributes(type_foldable)] =>
     /// Derives `TypeFoldable` for the annotated `struct` or `enum` (`union` is not supported).

compiler/rustc_macros/src/serialize.rs

@@ -16,14 +16,21 @@ pub(super) fn type_decodable_derive(
     decodable_body(s, decoder_ty)
 }
 
-pub(super) fn meta_decodable_derive(
+pub(super) fn blob_decodable_derive(
     mut s: synstructure::Structure<'_>,
 ) -> proc_macro2::TokenStream {
-    if !s.ast().generics.lifetimes().any(|lt| lt.lifetime.ident == "tcx") {
-        s.add_impl_generic(parse_quote! { 'tcx });
-    }
-    s.add_impl_generic(parse_quote! { '__a });
-    let decoder_ty = quote! { DecodeContext<'__a, 'tcx> };
+    let decoder_ty = quote! { __D };
+    s.add_impl_generic(parse_quote! { #decoder_ty: ::rustc_span::BlobDecoder });
+    s.add_bounds(synstructure::AddBounds::Generics);
+
+    decodable_body(s, decoder_ty)
+}
+
+pub(super) fn lazy_decodable_derive(
+    mut s: synstructure::Structure<'_>,
+) -> proc_macro2::TokenStream {
+    let decoder_ty = quote! { __D };
+    s.add_impl_generic(parse_quote! { #decoder_ty: LazyDecoder });
     s.add_bounds(synstructure::AddBounds::Generics);
 
     decodable_body(s, decoder_ty)

compiler/rustc_metadata/src/creader.rs

@@ -276,8 +276,9 @@ impl CStore {
             .filter_map(|(cnum, data)| data.as_deref().map(|data| (cnum, data)))
     }
 
-    pub fn all_proc_macro_def_ids(&self) -> impl Iterator<Item = DefId> {
-        self.iter_crate_data().flat_map(|(krate, data)| data.proc_macros_for_crate(krate, self))
+    pub fn all_proc_macro_def_ids(&self, tcx: TyCtxt<'_>) -> impl Iterator<Item = DefId> {
+        self.iter_crate_data()
+            .flat_map(move |(krate, data)| data.proc_macros_for_crate(tcx, krate, self))
     }
 
     fn push_dependencies_in_postorder(&self, deps: &mut IndexSet<CrateNum>, cnum: CrateNum) {
@@ -683,6 +684,7 @@ impl CStore {
         };
 
         let crate_metadata = CrateMetadata::new(
+            tcx,
             self,
             metadata,
             crate_root,
@@ -778,7 +780,7 @@ impl CStore {
         self.used_extern_options.insert(name);
         match self.maybe_resolve_crate(tcx, name, dep_kind, origin) {
             Ok(cnum) => {
-                self.set_used_recursively(cnum);
+                self.set_used_recursively(tcx, cnum);
                 Some(cnum)
             }
             Err(err) => {