Simplify hygiene::Mark application, and

remove variant `Token::SubstNt` in favor of `quoted::TokenTree::MetaVar`.
rust-lang · Jun 26, 2017 · d4488b7 · d4488b7
1 parent fc9ccfd
commit d4488b7
Show file tree

Hide file tree

Showing 26 changed files with 160 additions and 172 deletions.
diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs
@@ -87,6 +87,8 @@ pub mod __internal {
     use std::rc::Rc;
 
     use syntax::ast;
+    use syntax::ext::base::ExtCtxt;
+    use syntax::ext::hygiene::Mark;
     use syntax::ptr::P;
     use syntax::parse::{self, token, ParseSess};
     use syntax::tokenstream::{TokenTree, TokenStream as TokenStream_};
@@ -107,7 +109,7 @@ pub mod __internal {
     }
 
     pub fn token_stream_parse_items(stream: TokenStream) -> Result<Vec<P<ast::Item>>, LexError> {
-        with_parse_sess(move |sess| {
+        with_sess(move |(sess, _)| {
             let mut parser = parse::stream_to_parser(sess, stream.inner);
             let mut items = Vec::new();
 
@@ -140,13 +142,14 @@ pub mod __internal {
 
     // Emulate scoped_thread_local!() here essentially
     thread_local! {
-        static CURRENT_SESS: Cell<*const ParseSess> = Cell::new(0 as *const _);
+        static CURRENT_SESS: Cell<(*const ParseSess, Mark)> =
+            Cell::new((0 as *const _, Mark::root()));
     }
 
-    pub fn set_parse_sess<F, R>(sess: &ParseSess, f: F) -> R
+    pub fn set_sess<F, R>(cx: &ExtCtxt, f: F) -> R
         where F: FnOnce() -> R
     {
-        struct Reset { prev: *const ParseSess }
+        struct Reset { prev: (*const ParseSess, Mark) }
 
         impl Drop for Reset {
             fn drop(&mut self) {
@@ -156,18 +159,18 @@ pub mod __internal {
 
         CURRENT_SESS.with(|p| {
             let _reset = Reset { prev: p.get() };
-            p.set(sess);
+            p.set((cx.parse_sess, cx.current_expansion.mark));
             f()
         })
     }
 
-    pub fn with_parse_sess<F, R>(f: F) -> R
-        where F: FnOnce(&ParseSess) -> R
+    pub fn with_sess<F, R>(f: F) -> R
+        where F: FnOnce((&ParseSess, Mark)) -> R
     {
         let p = CURRENT_SESS.with(|p| p.get());
-        assert!(!p.is_null(), "proc_macro::__internal::with_parse_sess() called \
-                               before set_parse_sess()!");
-        f(unsafe { &*p })
+        assert!(!p.0.is_null(), "proc_macro::__internal::with_sess() called \
+                                 before set_parse_sess()!");
+        f(unsafe { (&*p.0, p.1) })
     }
 }
 
@@ -181,10 +184,11 @@ impl FromStr for TokenStream {
     type Err = LexError;
 
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
-        __internal::with_parse_sess(|sess| {
+        __internal::with_sess(|(sess, mark)| {
             let src = src.to_string();
             let name = "<proc-macro source code>".to_string();
-            let stream = parse::parse_stream_from_source_str(name, src, sess);
+            let call_site = mark.expn_info().unwrap().call_site;
+            let stream = parse::parse_stream_from_source_str(name, src, sess, Some(call_site));
             Ok(__internal::token_stream_wrap(stream))
         })
     }

diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs
@@ -283,8 +283,7 @@ fn hash_token<'a, 'gcx, 'tcx, W: StableHasherResult>(token: &token::Token,
         }
 
         token::Token::Ident(ident) |
-        token::Token::Lifetime(ident) |
-        token::Token::SubstNt(ident) => ident.name.hash_stable(hcx, hasher),
+        token::Token::Lifetime(ident) => ident.name.hash_stable(hcx, hasher),
 
         token::Token::Interpolated(ref non_terminal) => {
             // FIXME(mw): This could be implemented properly. It's just a

diff --git a/src/librustc_metadata/cstore_impl.rs b/src/librustc_metadata/cstore_impl.rs
@@ -372,7 +372,7 @@ impl CrateStore for cstore::CStore {
 
         let filemap = sess.parse_sess.codemap().new_filemap(source_name, def.body);
         let local_span = Span { lo: filemap.start_pos, hi: filemap.end_pos, ctxt: NO_EXPANSION };
-        let body = filemap_to_stream(&sess.parse_sess, filemap);
+        let body = filemap_to_stream(&sess.parse_sess, filemap, None);
 
         // Mark the attrs as used
         let attrs = data.get_item_attrs(id.index, &self.dep_graph);

diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs
@@ -319,7 +319,7 @@ impl<'a> Classifier<'a> {
             token::Lifetime(..) => Class::Lifetime,
 
             token::Underscore | token::Eof | token::Interpolated(..) |
-            token::SubstNt(..) | token::Tilde | token::At => Class::None,
+            token::Tilde | token::At => Class::None,
         };
 
         // Anything that didn't return above is the simple case where we the

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
@@ -903,17 +903,3 @@ pub fn get_exprs_from_tts(cx: &mut ExtCtxt,
     }
     Some(es)
 }
-
-pub struct ChangeSpan {
-    pub span: Span
-}
-
-impl Folder for ChangeSpan {
-    fn new_span(&mut self, _sp: Span) -> Span {
-        self.span
-    }
-
-    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
-        fold::noop_fold_mac(mac, self)
-    }
-}
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
@@ -16,7 +16,7 @@ use config::{is_test_or_bench, StripUnconfigured};
 use errors::FatalError;
 use ext::base::*;
 use ext::derive::{add_derived_markers, collect_derives};
-use ext::hygiene::Mark;
+use ext::hygiene::{Mark, SyntaxContext};
 use ext::placeholders::{placeholder, PlaceholderExpander};
 use feature_gate::{self, Features, is_builtin_attr};
 use fold;
@@ -470,15 +470,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             Ok(())
         };
 
-        let marked_tts = noop_fold_tts(mac.node.stream(), &mut Marker(mark));
         let opt_expanded = match *ext {
             SyntaxExtension::DeclMacro(ref expand, def_span) => {
                 if let Err(msg) = validate_and_set_expn_info(def_span.map(|(_, s)| s),
                                                              false) {
                     self.cx.span_err(path.span, &msg);
                     return kind.dummy(span);
                 }
-                kind.make_from(expand.expand(self.cx, span, marked_tts))
+                kind.make_from(expand.expand(self.cx, span, mac.node.stream()))
             }
 
             NormalTT(ref expandfun, def_info, allow_internal_unstable) => {
@@ -487,7 +486,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     self.cx.span_err(path.span, &msg);
                     return kind.dummy(span);
                 }
-                kind.make_from(expandfun.expand(self.cx, span, marked_tts))
+                kind.make_from(expandfun.expand(self.cx, span, mac.node.stream()))
             }
 
             IdentTT(ref expander, tt_span, allow_internal_unstable) => {
@@ -506,7 +505,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     }
                 });
 
-                let input: Vec<_> = marked_tts.into_trees().collect();
+                let input: Vec<_> = mac.node.stream().into_trees().collect();
                 kind.make_from(expander.expand(self.cx, span, ident, input))
             }
 
@@ -541,21 +540,17 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     },
                 });
 
-                let tok_result = expandfun.expand(self.cx, span, marked_tts);
+                let tok_result = expandfun.expand(self.cx, span, mac.node.stream());
                 Some(self.parse_expansion(tok_result, kind, path, span))
             }
         };
 
-        let expanded = if let Some(expanded) = opt_expanded {
-            expanded
-        } else {
+        unwrap_or!(opt_expanded, {
             let msg = format!("non-{kind} macro in {kind} position: {name}",
                               name = path.segments[0].identifier.name, kind = kind.name());
             self.cx.span_err(path.span, &msg);
-            return kind.dummy(span);
-        };
-
-        expanded.fold_with(&mut Marker(mark))
+            kind.dummy(span)
+        })
     }
 
     /// Expand a derive invocation. Returns the result of expansion.
@@ -621,8 +616,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             }
         };
         parser.ensure_complete_parse(path, kind.name(), span);
-        // FIXME better span info
-        expansion.fold_with(&mut ChangeSpan { span: span })
+        expansion
     }
 }
 
@@ -673,7 +667,9 @@ impl<'a> Parser<'a> {
         if self.token != token::Eof {
             let msg = format!("macro expansion ignores token `{}` and any following",
                               self.this_token_to_string());
-            let mut err = self.diagnostic().struct_span_err(self.span, &msg);
+            let mut def_site_span = self.span;
+            def_site_span.ctxt = SyntaxContext::empty(); // Avoid emitting backtrace info twice.
+            let mut err = self.diagnostic().struct_span_err(def_site_span, &msg);
             let msg = format!("caused by the macro expansion here; the usage \
                                of `{}!` is likely invalid in {} context",
                                macro_path, kind_name);
@@ -787,12 +783,12 @@ fn stream_for_item(item: &Annotatable, parse_sess: &ParseSess) -> TokenStream {
         Annotatable::TraitItem(ref ti) => pprust::trait_item_to_string(ti),
         Annotatable::ImplItem(ref ii) => pprust::impl_item_to_string(ii),
     };
-    string_to_stream(text, parse_sess)
+    string_to_stream(text, parse_sess, item.span())
 }
 
-fn string_to_stream(text: String, parse_sess: &ParseSess) -> TokenStream {
+fn string_to_stream(text: String, parse_sess: &ParseSess, span: Span) -> TokenStream {
     let filename = String::from("<macro expansion>");
-    filemap_to_stream(parse_sess, parse_sess.codemap().new_filemap(filename, text))
+    filemap_to_stream(parse_sess, parse_sess.codemap().new_filemap(filename, text), Some(span))
 }
 
 impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
@@ -1070,7 +1066,7 @@ impl<'feat> ExpansionConfig<'feat> {
 }
 
 // A Marker adds the given mark to the syntax context.
-struct Marker(Mark);
+pub struct Marker(pub Mark);
 
 impl Folder for Marker {
     fn fold_ident(&mut self, mut ident: Ident) -> Ident {

diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
@@ -364,7 +364,7 @@ pub mod rt {
 
         fn parse_tts(&self, s: String) -> Vec<TokenTree> {
             let source_name = "<quote expansion>".to_owned();
-            parse::parse_stream_from_source_str(source_name, s, self.parse_sess())
+            parse::parse_stream_from_source_str(source_name, s, self.parse_sess(), None)
                 .into_trees().collect()
         }
     }
@@ -700,7 +700,7 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
         token::Underscore   => "Underscore",
         token::Eof          => "Eof",
 
-        token::Whitespace | token::SubstNt(_) | token::Comment | token::Shebang(_) => {
+        token::Whitespace | token::Comment | token::Shebang(_) => {
             panic!("unhandled token in quote!");
         }
     };

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
@@ -158,15 +158,10 @@ pub type NamedParseResult = ParseResult<HashMap<Ident, Rc<NamedMatch>>>;
 pub fn count_names(ms: &[TokenTree]) -> usize {
     ms.iter().fold(0, |count, elt| {
         count + match *elt {
-            TokenTree::Sequence(_, ref seq) => {
-                seq.num_captures
-            }
-            TokenTree::Delimited(_, ref delim) => {
-                count_names(&delim.tts)
-            }
-            TokenTree::MetaVarDecl(..) => {
-                1
-            }
+            TokenTree::Sequence(_, ref seq) => seq.num_captures,
+            TokenTree::Delimited(_, ref delim) => count_names(&delim.tts),
+            TokenTree::MetaVar(..) => 0,
+            TokenTree::MetaVarDecl(..) => 1,
             TokenTree::Token(..) => 0,
         }
     })
@@ -244,7 +239,7 @@ fn nameize<I: Iterator<Item=NamedMatch>>(sess: &ParseSess, ms: &[TokenTree], mut
                     }
                 }
             }
-            TokenTree::Token(..) => (),
+            TokenTree::MetaVar(..) | TokenTree::Token(..) => (),
         }
 
         Ok(())
@@ -409,12 +404,11 @@ fn inner_parse_loop(sess: &ParseSess,
                     ei.idx = 0;
                     cur_eis.push(ei);
                 }
-                TokenTree::Token(_, ref t) => {
-                    if token_name_eq(t, token) {
-                        ei.idx += 1;
-                        next_eis.push(ei);
-                    }
+                TokenTree::Token(_, ref t) if token_name_eq(t, token) => {
+                    ei.idx += 1;
+                    next_eis.push(ei);
                 }
+                TokenTree::Token(..) | TokenTree::MetaVar(..) => {}
             }
         }
     }

diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
@@ -120,7 +120,7 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt,
                     _ => cx.span_bug(sp, "malformed macro rhs"),
                 };
                 // rhs has holes ( `$id` and `$(...)` that need filled)
-                let tts = transcribe(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs);
+                let tts = transcribe(cx, Some(named_matches), rhs);
 
                 if cx.trace_macros() {
                     trace_macros_note(cx, sp, format!("to `{}`", tts));
@@ -292,7 +292,7 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool {
     use self::quoted::TokenTree;
     for tt in tts {
         match *tt {
-            TokenTree::Token(..) | TokenTree::MetaVarDecl(..) => (),
+            TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (),
             TokenTree::Delimited(_, ref del) => if !check_lhs_no_empty_seq(sess, &del.tts) {
                 return false;
             },
@@ -372,7 +372,7 @@ impl FirstSets {
             let mut first = TokenSet::empty();
             for tt in tts.iter().rev() {
                 match *tt {
-                    TokenTree::Token(..) | TokenTree::MetaVarDecl(..) => {
+                    TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
                         first.replace_with(tt.clone());
                     }
                     TokenTree::Delimited(span, ref delimited) => {
@@ -432,7 +432,7 @@ impl FirstSets {
         for tt in tts.iter() {
             assert!(first.maybe_empty);
             match *tt {
-                TokenTree::Token(..) | TokenTree::MetaVarDecl(..) => {
+                TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
                     first.add_one(tt.clone());
                     return first;
                 }
@@ -602,7 +602,7 @@ fn check_matcher_core(sess: &ParseSess,
         // First, update `last` so that it corresponds to the set
         // of NT tokens that might end the sequence `... token`.
         match *token {
-            TokenTree::Token(..) | TokenTree::MetaVarDecl(..) => {
+            TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
                 let can_be_followed_by_any;
                 if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, token) {
                     let msg = format!("invalid fragment specifier `{}`", bad_frag);
@@ -872,6 +872,7 @@ fn is_legal_fragment_specifier(sess: &ParseSess,
 fn quoted_tt_to_string(tt: &quoted::TokenTree) -> String {
     match *tt {
         quoted::TokenTree::Token(_, ref tok) => ::print::pprust::token_to_string(tok),
+        quoted::TokenTree::MetaVar(_, name) => format!("${}", name),
         quoted::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind),
         _ => panic!("unexpected quoted::TokenTree::{{Sequence or Delimited}} \
                      in follow set checker"),