Auto merge of #51952 - petrochenkov:transmark, r=<try>

 hygiene: Decouple transparencies from expansion IDs

Fixes #50504 in accordance with [this comment](#50504 (comment)).

src/libproc_macro/lib.rs

@@ -1351,7 +1351,7 @@ pub mod __internal {
     use syntax::parse::token::{self, Token};
     use syntax::tokenstream;
     use syntax_pos::{BytePos, Loc, DUMMY_SP};
-    use syntax_pos::hygiene::{Mark, SyntaxContext, Transparency};
+    use syntax_pos::hygiene::{SyntaxContext, Transparency};
 
     use super::{TokenStream, LexError, Span};
 
@@ -1436,20 +1436,15 @@ pub mod __internal {
 
             // No way to determine def location for a proc macro right now, so use call location.
             let location = cx.current_expansion.mark.expn_info().unwrap().call_site;
-            // Opaque mark was already created by expansion, now create its transparent twin.
-            // We can't use the call-site span literally here, even if it appears to provide
-            // correct name resolution, because it has all the `ExpnInfo` wrong, so the edition
-            // checks, lint macro checks, macro backtraces will all break.
-            let opaque_mark = cx.current_expansion.mark;
-            let transparent_mark = Mark::fresh_cloned(opaque_mark);
-            transparent_mark.set_transparency(Transparency::Transparent);
-
-            let to_span = |mark| Span(location.with_ctxt(SyntaxContext::empty().apply_mark(mark)));
+            let to_span = |transparency| Span(location.with_ctxt(
+                SyntaxContext::empty().apply_mark_with_transparency(cx.current_expansion.mark,
+                                                                    transparency))
+            );
             p.set(ProcMacroSess {
                 parse_sess: cx.parse_sess,
                 data: ProcMacroData {
-                    def_site: to_span(opaque_mark),
-                    call_site: to_span(transparent_mark),
+                    def_site: to_span(Transparency::Opaque),
+                    call_site: to_span(Transparency::Transparent),
                 },
             });
             f()

src/librustc_resolve/lib.rs

@@ -1996,17 +1996,17 @@ impl<'a> Resolver<'a> {
             let mut iter = ctxt.marks().into_iter().rev().peekable();
             let mut result = None;
             // Find the last modern mark from the end if it exists.
-            while let Some(&mark) = iter.peek() {
-                if mark.transparency() == Transparency::Opaque {
+            while let Some(&(mark, transparency)) = iter.peek() {
+                if transparency == Transparency::Opaque {
                     result = Some(mark);
                     iter.next();
                 } else {
                     break;
                 }
             }
             // Then find the last legacy mark from the end if it exists.
-            for mark in iter {
-                if mark.transparency() == Transparency::SemiTransparent {
+            for (mark, transparency) in iter {
+                if transparency == Transparency::SemiTransparent {
                     result = Some(mark);
                 } else {
                     break;

src/librustc_resolve/macros.rs

@@ -24,7 +24,7 @@ use syntax::errors::DiagnosticBuilder;
 use syntax::ext::base::{self, Annotatable, Determinacy, MultiModifier, MultiDecorator};
 use syntax::ext::base::{MacroKind, SyntaxExtension, Resolver as SyntaxResolver};
 use syntax::ext::expand::{self, AstFragment, AstFragmentKind, Invocation, InvocationKind};
-use syntax::ext::hygiene::{self, Mark, Transparency};
+use syntax::ext::hygiene::{self, Mark};
 use syntax::ext::placeholders::placeholder;
 use syntax::ext::tt::macro_rules;
 use syntax::feature_gate::{self, emit_feature_err, GateIssue};
@@ -331,13 +331,8 @@ impl<'a> base::Resolver for Resolver<'a> {
 
         self.unused_macros.remove(&def_id);
         let ext = self.get_macro(def);
-        if ext.is_modern() {
-            let transparency =
-                if ext.is_transparent() { Transparency::Transparent } else { Transparency::Opaque };
-            invoc.expansion_data.mark.set_transparency(transparency);
-        } else if def_id.krate == BUILTIN_MACROS_CRATE {
-            invoc.expansion_data.mark.set_is_builtin(true);
-        }
+        invoc.expansion_data.mark.set_default_transparency(ext.default_transparency());
+        invoc.expansion_data.mark.set_is_builtin(def_id.krate == BUILTIN_MACROS_CRATE);
         Ok(Some(ext))
     }
 

src/libsyntax/ext/base.rs

@@ -17,7 +17,7 @@ use syntax_pos::{Span, MultiSpan, DUMMY_SP};
 use edition::Edition;
 use errors::{DiagnosticBuilder, DiagnosticId};
 use ext::expand::{self, AstFragment, Invocation};
-use ext::hygiene::{self, Mark, SyntaxContext};
+use ext::hygiene::{self, Mark, SyntaxContext, Transparency};
 use fold::{self, Folder};
 use parse::{self, parser, DirectoryOwnership};
 use parse::token;
@@ -673,20 +673,14 @@ impl SyntaxExtension {
         }
     }
 
-    pub fn is_modern(&self) -> bool {
+    pub fn default_transparency(&self) -> Transparency {
         match *self {
-            SyntaxExtension::DeclMacro { .. } |
             SyntaxExtension::ProcMacro { .. } |
             SyntaxExtension::AttrProcMacro(..) |
-            SyntaxExtension::ProcMacroDerive(..) => true,
-            _ => false,
-        }
-    }
-
-    pub fn is_transparent(&self) -> bool {
-        match *self {
-            SyntaxExtension::DeclMacro { is_transparent, .. } => is_transparent,
-            _ => false,
+            SyntaxExtension::ProcMacroDerive(..) |
+            SyntaxExtension::DeclMacro { is_transparent: true, .. } => Transparency::Transparent,
+            SyntaxExtension::DeclMacro { is_transparent: false, .. } => Transparency::Opaque,
+            _ => Transparency::SemiTransparent,
         }
     }
 

src/libsyntax_pos/hygiene.rs

@@ -27,16 +27,17 @@ use std::fmt;
 
 /// A SyntaxContext represents a chain of macro expansions (represented by marks).
 #[derive(Clone, Copy, PartialEq, Eq, Default, PartialOrd, Ord, Hash)]
-pub struct SyntaxContext(pub(super) u32);
+pub struct SyntaxContext(u32);
 
 #[derive(Copy, Clone, Debug)]
-pub struct SyntaxContextData {
-    pub outer_mark: Mark,
-    pub prev_ctxt: SyntaxContext,
+struct SyntaxContextData {
+    outer_mark: Mark,
+    transparency: Transparency,
+    prev_ctxt: SyntaxContext,
     // This context, but with all transparent and semi-transparent marks filtered away.
-    pub opaque: SyntaxContext,
+    opaque: SyntaxContext,
     // This context, but with all transparent marks filtered away.
-    pub opaque_and_semitransparent: SyntaxContext,
+    opaque_and_semitransparent: SyntaxContext,
 }
 
 /// A mark is a unique id associated with a macro expansion.
@@ -46,14 +47,14 @@ pub struct Mark(u32);
 #[derive(Clone, Debug)]
 struct MarkData {
     parent: Mark,
-    transparency: Transparency,
+    default_transparency: Transparency,
     is_builtin: bool,
     expn_info: Option<ExpnInfo>,
 }
 
 /// A property of a macro expansion that determines how identifiers
 /// produced by that expansion are resolved.
-#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
 pub enum Transparency {
     /// Identifier produced by a transparent expansion is always resolved at call-site.
     /// Call-site spans in procedural macros, hygiene opt-out in `macro` should use this.
@@ -81,7 +82,7 @@ impl Mark {
             Mark::fresh_with_data(MarkData {
                 parent,
                 // By default expansions behave like `macro_rules`.
-                transparency: Transparency::SemiTransparent,
+                default_transparency: Transparency::SemiTransparent,
                 is_builtin: false,
                 expn_info: None,
             }, data)
@@ -127,34 +128,32 @@ impl Mark {
         })
     }
 
+    // FIXME: This operation doesn't really make sense when single macro expansion
+    // can produce tokens with different transparencies. Figure out how to avoid it.
     pub fn modern(mut self) -> Mark {
         HygieneData::with(|data| {
-            while data.marks[self.0 as usize].transparency != Transparency::Opaque {
+            while data.marks[self.0 as usize].default_transparency != Transparency::Opaque {
                 self = data.marks[self.0 as usize].parent;
             }
             self
         })
     }
 
     #[inline]
-    pub fn transparency(self) -> Transparency {
-        assert_ne!(self, Mark::root());
-        HygieneData::with(|data| data.marks[self.0 as usize].transparency)
-    }
-
-    #[inline]
-    pub fn set_transparency(self, transparency: Transparency) {
+    pub fn set_default_transparency(self, transparency: Transparency) {
         assert_ne!(self, Mark::root());
-        HygieneData::with(|data| data.marks[self.0 as usize].transparency = transparency)
+        HygieneData::with(|data| data.marks[self.0 as usize].default_transparency = transparency)
     }
 
     #[inline]
     pub fn is_builtin(self) -> bool {
+        assert_ne!(self, Mark::root());
         HygieneData::with(|data| data.marks[self.0 as usize].is_builtin)
     }
 
     #[inline]
     pub fn set_is_builtin(self, is_builtin: bool) {
+        assert_ne!(self, Mark::root());
         HygieneData::with(|data| data.marks[self.0 as usize].is_builtin = is_builtin)
     }
 
@@ -198,26 +197,27 @@ impl Mark {
 }
 
 #[derive(Debug)]
-pub struct HygieneData {
+crate struct HygieneData {
     marks: Vec<MarkData>,
     syntax_contexts: Vec<SyntaxContextData>,
-    markings: HashMap<(SyntaxContext, Mark), SyntaxContext>,
+    markings: HashMap<(SyntaxContext, Mark, Transparency), SyntaxContext>,
     default_edition: Edition,
 }
 
 impl HygieneData {
-    pub fn new() -> Self {
+    crate fn new() -> Self {
         HygieneData {
             marks: vec![MarkData {
                 parent: Mark::root(),
                 // If the root is opaque, then loops searching for an opaque mark
                 // will automatically stop after reaching it.
-                transparency: Transparency::Opaque,
+                default_transparency: Transparency::Opaque,
                 is_builtin: true,
                 expn_info: None,
             }],
             syntax_contexts: vec![SyntaxContextData {
                 outer_mark: Mark::root(),
+                transparency: Transparency::Opaque,
                 prev_ctxt: SyntaxContext(0),
                 opaque: SyntaxContext(0),
                 opaque_and_semitransparent: SyntaxContext(0),
@@ -249,6 +249,14 @@ impl SyntaxContext {
         SyntaxContext(0)
     }
 
+    crate fn as_u32(self) -> u32 {
+        self.0
+    }
+
+    crate fn from_u32(raw: u32) -> SyntaxContext {
+        SyntaxContext(raw)
+    }
+
     // Allocate a new SyntaxContext with the given ExpnInfo. This is used when
     // deserializing Spans from the incr. comp. cache.
     // FIXME(mw): This method does not restore MarkData::parent or
@@ -259,7 +267,7 @@ impl SyntaxContext {
         HygieneData::with(|data| {
             data.marks.push(MarkData {
                 parent: Mark::root(),
-                transparency: Transparency::SemiTransparent,
+                default_transparency: Transparency::SemiTransparent,
                 is_builtin: false,
                 expn_info: Some(expansion_info),
             });
@@ -268,6 +276,7 @@ impl SyntaxContext {
 
             data.syntax_contexts.push(SyntaxContextData {
                 outer_mark: mark,
+                transparency: Transparency::SemiTransparent,
                 prev_ctxt: SyntaxContext::empty(),
                 opaque: SyntaxContext::empty(),
                 opaque_and_semitransparent: SyntaxContext::empty(),
@@ -276,22 +285,31 @@ impl SyntaxContext {
         })
     }
 
-    /// Extend a syntax context with a given mark
     pub fn apply_mark(self, mark: Mark) -> SyntaxContext {
-        if mark.transparency() == Transparency::Opaque {
-            return self.apply_mark_internal(mark);
+        assert_ne!(mark, Mark::root());
+        self.apply_mark_with_transparency(
+            mark, HygieneData::with(|data| data.marks[mark.0 as usize].default_transparency)
+        )
+    }
+
+    /// Extend a syntax context with a given mark and transparency
+    pub fn apply_mark_with_transparency(self, mark: Mark, transparency: Transparency)
+                                        -> SyntaxContext {
+        assert_ne!(mark, Mark::root());
+        if transparency == Transparency::Opaque {
+            return self.apply_mark_internal(mark, transparency);
         }
 
         let call_site_ctxt =
             mark.expn_info().map_or(SyntaxContext::empty(), |info| info.call_site.ctxt());
-        let call_site_ctxt = if mark.transparency() == Transparency::SemiTransparent {
+        let call_site_ctxt = if transparency == Transparency::SemiTransparent {
             call_site_ctxt.modern()
         } else {
             call_site_ctxt.modern_and_legacy()
         };
 
         if call_site_ctxt == SyntaxContext::empty() {
-            return self.apply_mark_internal(mark);
+            return self.apply_mark_internal(mark, transparency);
         }
 
         // Otherwise, `mark` is a macros 1.0 definition and the call site is in a
@@ -304,27 +322,26 @@ impl SyntaxContext {
         //
         // See the example at `test/run-pass/hygiene/legacy_interaction.rs`.
         let mut ctxt = call_site_ctxt;
-        for mark in self.marks() {
-            ctxt = ctxt.apply_mark_internal(mark);
+        for (mark, transparency) in self.marks() {
+            ctxt = ctxt.apply_mark_internal(mark, transparency);
         }
-        ctxt.apply_mark_internal(mark)
+        ctxt.apply_mark_internal(mark, transparency)
     }
 
-    fn apply_mark_internal(self, mark: Mark) -> SyntaxContext {
+    fn apply_mark_internal(self, mark: Mark, transparency: Transparency) -> SyntaxContext {
         HygieneData::with(|data| {
             let syntax_contexts = &mut data.syntax_contexts;
-            let transparency = data.marks[mark.0 as usize].transparency;
-
             let mut opaque = syntax_contexts[self.0 as usize].opaque;
             let mut opaque_and_semitransparent =
                 syntax_contexts[self.0 as usize].opaque_and_semitransparent;
 
             if transparency >= Transparency::Opaque {
                 let prev_ctxt = opaque;
-                opaque = *data.markings.entry((prev_ctxt, mark)).or_insert_with(|| {
+                opaque = *data.markings.entry((prev_ctxt, mark, transparency)).or_insert_with(|| {
                     let new_opaque = SyntaxContext(syntax_contexts.len() as u32);
                     syntax_contexts.push(SyntaxContextData {
                         outer_mark: mark,
+                        transparency,
                         prev_ctxt,
                         opaque: new_opaque,
                         opaque_and_semitransparent: new_opaque,
@@ -336,11 +353,12 @@ impl SyntaxContext {
             if transparency >= Transparency::SemiTransparent {
                 let prev_ctxt = opaque_and_semitransparent;
                 opaque_and_semitransparent =
-                        *data.markings.entry((prev_ctxt, mark)).or_insert_with(|| {
+                        *data.markings.entry((prev_ctxt, mark, transparency)).or_insert_with(|| {
                     let new_opaque_and_semitransparent =
                         SyntaxContext(syntax_contexts.len() as u32);
                     syntax_contexts.push(SyntaxContextData {
                         outer_mark: mark,
+                        transparency,
                         prev_ctxt,
                         opaque,
                         opaque_and_semitransparent: new_opaque_and_semitransparent,
@@ -350,11 +368,12 @@ impl SyntaxContext {
             }
 
             let prev_ctxt = self;
-            *data.markings.entry((prev_ctxt, mark)).or_insert_with(|| {
+            *data.markings.entry((prev_ctxt, mark, transparency)).or_insert_with(|| {
                 let new_opaque_and_semitransparent_and_transparent =
                     SyntaxContext(syntax_contexts.len() as u32);
                 syntax_contexts.push(SyntaxContextData {
                     outer_mark: mark,
+                    transparency,
                     prev_ctxt,
                     opaque,
                     opaque_and_semitransparent,
@@ -388,12 +407,13 @@ impl SyntaxContext {
         })
     }
 
-    pub fn marks(mut self) -> Vec<Mark> {
+    pub fn marks(mut self) -> Vec<(Mark, Transparency)> {
         HygieneData::with(|data| {
             let mut marks = Vec::new();
             while self != SyntaxContext::empty() {
-                marks.push(data.syntax_contexts[self.0 as usize].outer_mark);
-                self = data.syntax_contexts[self.0 as usize].prev_ctxt;
+                let ctxt_data = &data.syntax_contexts[self.0 as usize];
+                marks.push((ctxt_data.outer_mark, ctxt_data.transparency));
+                self = ctxt_data.prev_ctxt;
             }
             marks.reverse();
             marks