Encode small spans to avoid interning.

compiler/rustc_hir/src/hir.rs

@@ -3486,7 +3486,7 @@ mod size_asserts {
     rustc_data_structures::static_assert_size!(super::Ty<'static>, 80);
     rustc_data_structures::static_assert_size!(super::GenericBound<'_>, 48);
     rustc_data_structures::static_assert_size!(super::Generics<'static>, 56);
-    rustc_data_structures::static_assert_size!(super::Impl<'static>, 72);
+    rustc_data_structures::static_assert_size!(super::Impl<'static>, 88);
 
     rustc_data_structures::static_assert_size!(super::Item<'static>, 88);
     rustc_data_structures::static_assert_size!(super::TraitItem<'static>, 96);

compiler/rustc_infer/src/infer/mod.rs

@@ -405,7 +405,7 @@ pub enum SubregionOrigin<'tcx> {
 
 // `SubregionOrigin` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(SubregionOrigin<'_>, 32);
+static_assert_size!(SubregionOrigin<'_>, 40);
 
 /// Times when we replace late-bound regions with variables:
 #[derive(Clone, Copy, Debug)]

compiler/rustc_parse/src/parser/mod.rs

@@ -152,7 +152,7 @@ pub struct Parser<'a> {
 // This type is used a lot, e.g. it's cloned when matching many declarative macro rules. Make sure
 // it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(Parser<'_>, 328);
+rustc_data_structures::static_assert_size!(Parser<'_>, 344);
 
 /// Stores span information about a closure.
 #[derive(Clone)]

compiler/rustc_span/src/lib.rs

@@ -15,10 +15,12 @@
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(array_windows)]
+#![feature(const_nonnull_new)]
 #![feature(let_else)]
 #![feature(if_let_guard)]
 #![feature(negative_impls)]
 #![feature(min_specialization)]
+#![feature(strict_provenance)]
 #![feature(rustc_attrs)]
 #![allow(rustc::potential_query_instability)]
 
@@ -604,7 +606,11 @@ impl Span {
 
     /// Returns `self` if `self` is not the dummy span, and `other` otherwise.
     pub fn substitute_dummy(self, other: Span) -> Span {
-        if self.is_dummy() { other } else { self }
+        if self.is_dummy() {
+            other
+        } else {
+            self
+        }
     }
 
     /// Returns `true` if `self` fully encloses `other`.
@@ -635,21 +641,33 @@ impl Span {
     pub fn trim_start(self, other: Span) -> Option<Span> {
         let span = self.data();
         let other = other.data();
-        if span.hi > other.hi { Some(span.with_lo(cmp::max(span.lo, other.hi))) } else { None }
+        if span.hi > other.hi {
+            Some(span.with_lo(cmp::max(span.lo, other.hi)))
+        } else {
+            None
+        }
     }
 
     /// Returns the source span -- this is either the supplied span, or the span for
     /// the macro callsite that expanded to it.
     pub fn source_callsite(self) -> Span {
         let expn_data = self.ctxt().outer_expn_data();
-        if !expn_data.is_root() { expn_data.call_site.source_callsite() } else { self }
+        if !expn_data.is_root() {
+            expn_data.call_site.source_callsite()
+        } else {
+            self
+        }
     }
 
     /// The `Span` for the tokens in the previous macro expansion from which `self` was generated,
     /// if any.
     pub fn parent_callsite(self) -> Option<Span> {
         let expn_data = self.ctxt().outer_expn_data();
-        if !expn_data.is_root() { Some(expn_data.call_site) } else { None }
+        if !expn_data.is_root() {
+            Some(expn_data.call_site)
+        } else {
+            None
+        }
     }
 
     /// Walk down the expansion ancestors to find a span that's contained within `outer`.
@@ -693,10 +711,18 @@ impl Span {
     pub fn source_callee(self) -> Option<ExpnData> {
         fn source_callee(expn_data: ExpnData) -> ExpnData {
             let next_expn_data = expn_data.call_site.ctxt().outer_expn_data();
-            if !next_expn_data.is_root() { source_callee(next_expn_data) } else { expn_data }
+            if !next_expn_data.is_root() {
+                source_callee(next_expn_data)
+            } else {
+                expn_data
+            }
         }
         let expn_data = self.ctxt().outer_expn_data();
-        if !expn_data.is_root() { Some(source_callee(expn_data)) } else { None }
+        if !expn_data.is_root() {
+            Some(source_callee(expn_data))
+        } else {
+            None
+        }
     }
 
     /// Checks if a span is "internal" to a macro in which `#[unstable]`

compiler/rustc_span/src/span_encoding.rs

@@ -11,6 +11,8 @@ use crate::{BytePos, SpanData};
 
 use rustc_data_structures::fx::FxHashSet;
 
+use std::ptr::invalid;
+
 /// A compressed span.
 ///
 /// Whereas [`SpanData`] is 16 bytes, which is a bit too big to stick everywhere, `Span`
@@ -25,17 +27,11 @@ use rustc_data_structures::fx::FxHashSet;
 /// slower because only 80--90% of spans could be stored inline (even less in
 /// very large crates) and so the interner was used a lot more.
 ///
-/// Inline (compressed) format with no parent:
+/// Inline (compressed) format:
 /// - `span.base_or_index == span_data.lo`
 /// - `span.len_or_tag == len == span_data.hi - span_data.lo` (must be `<= MAX_LEN`)
 /// - `span.ctxt == span_data.ctxt` (must be `<= MAX_CTXT`)
 ///
-/// Inline (compressed) format with root context:
-/// - `span.base_or_index == span_data.lo`
-/// - `span.len_or_tag == len == span_data.hi - span_data.lo` (must be `<= MAX_LEN`)
-/// - `span.len_or_tag` has top bit (`PARENT_MASK`) set
-/// - `span.ctxt == span_data.parent` (must be `<= MAX_CTXT`)
-///
 /// Interned format:
 /// - `span.base_or_index == index` (indexes into the interner table)
 /// - `span.len_or_tag == LEN_TAG` (high bit set, all other bits are zero)
@@ -69,18 +65,14 @@ use rustc_data_structures::fx::FxHashSet;
 #[derive(Clone, Copy, Eq, PartialEq, Hash)]
 #[rustc_pass_by_value]
 pub struct Span {
-    pointer: &'static SpanData,
+    pointer_or_code: *const SpanData,
 }
 
+const MAX_LEN: u32 = u16::MAX as u32;
+const MAX_CTXT: u32 = u16::MAX as u32 >> 1;
+
 /// Dummy span, both position and length are zero, syntax context is zero as well.
-pub const DUMMY_SP: Span = Span {
-    pointer: &SpanData {
-        lo: BytePos(0),
-        hi: BytePos(0),
-        ctxt: SyntaxContext::root(),
-        parent: None,
-    },
-};
+pub const DUMMY_SP: Span = Span { pointer_or_code: invalid(1) };
 
 impl Span {
     #[inline]
@@ -94,10 +86,22 @@ impl Span {
             std::mem::swap(&mut lo, &mut hi);
         }
 
-        // Interned format.
-        let pointer =
-            with_span_interner(|interner| interner.intern(SpanData { lo, hi, ctxt, parent }));
-        Span { pointer }
+        let (base, len, ctxt2) = (lo.0, hi.0 - lo.0, ctxt.as_u32());
+
+        if len <= MAX_LEN && ctxt2 <= MAX_CTXT && parent.is_none() {
+            // Inline format.
+            let code: usize =
+                ((base as usize) << 32) | ((len as usize) << 16) | ((ctxt2 as usize) << 1) | 1;
+            let pointer_or_code = invalid(code);
+            Span { pointer_or_code }
+        } else {
+            // Interned format.
+            let pointer =
+                with_span_interner(|interner| interner.intern(SpanData { lo, hi, ctxt, parent }));
+            let pointer = pointer as *const SpanData;
+            debug_assert_eq!(pointer.addr() & 1, 0);
+            Span { pointer_or_code: pointer }
+        }
     }
 
     #[inline]
@@ -113,7 +117,22 @@ impl Span {
     /// This function must not be used outside the incremental engine.
     #[inline]
     pub fn data_untracked(self) -> SpanData {
-        *self.pointer
+        let code = self.pointer_or_code.addr();
+        if code & 1 == 1 {
+            // Inline format.
+            let base = (code >> 32) as u32;
+            let len = (code >> 16) as u16;
+            let ctxt = (code as u16) >> 1;
+            SpanData {
+                lo: BytePos(base),
+                hi: BytePos(base + len as u32),
+                ctxt: SyntaxContext::from_u32(ctxt as u32),
+                parent: None,
+            }
+        } else {
+            // Interned format.
+            unsafe { *self.pointer_or_code }
+        }
     }
 }