[experiment] expand: Stop using nonterminals when passing items to proc macro attributes

src/librustc_expand/base.rs

@@ -4,7 +4,7 @@ use rustc_attr::{self as attr, Deprecation, HasAttrs, Stability};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{DiagnosticBuilder, DiagnosticId};
-use rustc_parse::{self, parser, DirectoryOwnership, MACRO_ARGUMENTS};
+use rustc_parse::{self, nt_to_tokenstream, parser, DirectoryOwnership, MACRO_ARGUMENTS};
 use rustc_session::parse::ParseSess;
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::{AstPass, ExpnData, ExpnId, ExpnKind};
@@ -118,6 +118,28 @@ impl Annotatable {
         }
     }
 
+    crate fn into_tokenstream(self, sess: &ParseSess) -> TokenStream {
+        let span = self.span();
+        let nt = match self {
+            Annotatable::Item(item) => token::NtItem(item),
+            Annotatable::TraitItem(item)
+            | Annotatable::ImplItem(item)
+            | Annotatable::ForeignItem(item) => {
+                token::NtItem(P(item.and_then(ast::AssocItem::into_item)))
+            }
+            Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
+            Annotatable::Expr(expr) => token::NtExpr(expr),
+            Annotatable::Arm(..)
+            | Annotatable::Field(..)
+            | Annotatable::FieldPat(..)
+            | Annotatable::GenericParam(..)
+            | Annotatable::Param(..)
+            | Annotatable::StructField(..)
+            | Annotatable::Variant(..) => panic!("unexpected annotatable"),
+        };
+        nt_to_tokenstream(&nt, sess, span)
+    }
+
     pub fn expect_item(self) -> P<ast::Item> {
         match self {
             Annotatable::Item(i) => i,

src/librustc_expand/expand.rs

@@ -7,7 +7,6 @@ use crate::proc_macro::collect_derives;
 
 use rustc_ast_pretty::pprust;
 use rustc_attr::{self as attr, is_builtin_attr, HasAttrs};
-use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, FatalError, PResult};
 use rustc_feature::Features;
 use rustc_parse::configure;
@@ -25,7 +24,7 @@ use syntax::ast::{ItemKind, MacArgs, MacStmtStyle, StmtKind};
 use syntax::mut_visit::*;
 use syntax::ptr::P;
 use syntax::token;
-use syntax::tokenstream::{TokenStream, TokenTree};
+use syntax::tokenstream::TokenStream;
 use syntax::util::map_in_place::MapInPlace;
 use syntax::visit::{self, AssocCtxt, Visitor};
 
@@ -668,36 +667,13 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 SyntaxExtensionKind::Attr(expander) => {
                     self.gate_proc_macro_input(&item);
                     self.gate_proc_macro_attr_item(span, &item);
-                    // `Annotatable` can be converted into tokens directly, but we are packing it
-                    // into a nonterminal as a piece of AST to make the produced token stream
-                    // look nicer in pretty-printed form. This may be no longer necessary.
-                    let item_tok = TokenTree::token(
-                        token::Interpolated(Lrc::new(match item {
-                            Annotatable::Item(item) => token::NtItem(item),
-                            Annotatable::TraitItem(item)
-                            | Annotatable::ImplItem(item)
-                            | Annotatable::ForeignItem(item) => {
-                                token::NtItem(P(item.and_then(ast::AssocItem::into_item)))
-                            }
-                            Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
-                            Annotatable::Expr(expr) => token::NtExpr(expr),
-                            Annotatable::Arm(..)
-                            | Annotatable::Field(..)
-                            | Annotatable::FieldPat(..)
-                            | Annotatable::GenericParam(..)
-                            | Annotatable::Param(..)
-                            | Annotatable::StructField(..)
-                            | Annotatable::Variant(..) => panic!("unexpected annotatable"),
-                        })),
-                        DUMMY_SP,
-                    )
-                    .into();
+                    let tokens = item.into_tokenstream(self.cx.parse_sess);
                     let item = attr.unwrap_normal_item();
                     if let MacArgs::Eq(..) = item.args {
                         self.cx.span_err(span, "key-value macro attributes are not supported");
                     }
                     let tok_result =
-                        expander.expand(self.cx, span, item.args.inner_tokens(), item_tok);
+                        expander.expand(self.cx, span, item.args.inner_tokens(), tokens);
                     self.parse_ast_fragment(tok_result, fragment_kind, &item.path, span)
                 }
                 SyntaxExtensionKind::LegacyAttr(expander) => {

src/test/ui/async-await/issues/issue-60674.stdout

@@ -1,3 +1,3 @@
-async fn f(mut x: u8) { }
-async fn g((mut x, y, mut z): (u8, u8, u8)) { }
-async fn g(mut x: u8, (a, mut b, c): (u8, u8, u8), y: u8) { }
+async fn f (mut x : u8) { }
+async fn g ((mut x, y, mut z) : (u8, u8, u8)) { }
+async fn g (mut x : u8, (a, mut b, c) : (u8, u8, u8), y : u8) { }

src/test/ui/proc-macro/auxiliary/attr-args.rs

@@ -15,7 +15,7 @@ pub fn attr_with_args(args: TokenStream, input: TokenStream) -> TokenStream {
 
     let input = input.to_string();
 
-    assert_eq!(input, "fn foo() { }");
+    assert_eq!(input, "fn foo () { }");
 
     r#"
         fn foo() -> &'static str { "Hello, world!" }

src/test/ui/proc-macro/auxiliary/attr-cfg.rs

@@ -11,12 +11,10 @@ use proc_macro::TokenStream;
 pub fn attr_cfg(args: TokenStream, input: TokenStream) -> TokenStream {
     let input_str = input.to_string();
 
-    assert_eq!(input_str, "fn outer() -> u8 {
-    #[cfg(foo)]
-    fn inner() -> u8 { 1 }
-    #[cfg(bar)]
-    fn inner() -> u8 { 2 }
-    inner()
+    assert_eq!(input_str, "fn outer () -> u8
+{
+    # [cfg (foo)] fn inner () -> u8 { 1 } # [cfg (bar)] fn inner () -> u8
+    { 2 } inner ()
 }");
 
     input

src/test/ui/proc-macro/auxiliary/attr-on-trait.rs

@@ -10,6 +10,6 @@ use proc_macro::TokenStream;
 #[proc_macro_attribute]
 pub fn foo(attr: TokenStream, item: TokenStream) -> TokenStream {
     drop(attr);
-    assert_eq!(item.to_string(), "fn foo() { }");
+    assert_eq!(item.to_string(), "fn foo () { }");
     "fn foo(&self);".parse().unwrap()
 }

src/test/ui/proc-macro/auxiliary/attr-stmt-expr-rpass.rs

@@ -10,28 +10,28 @@ use proc_macro::TokenStream;
 #[proc_macro_attribute]
 pub fn expect_let(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "let string = \"Hello, world!\";");
+    assert_eq!(item.to_string(), "let string = \"Hello, world!\" ;");
     item
 }
 
 #[proc_macro_attribute]
 pub fn expect_print_stmt(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "println!(\"{}\", string);");
+    assert_eq!(item.to_string(), "println ! (\"{}\", string) ;");
     item
 }
 
 #[proc_macro_attribute]
 pub fn expect_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "print_str(\"string\")");
+    assert_eq!(item.to_string(), "print_str (\"string\")");
     item
 }
 
 #[proc_macro_attribute]
 pub fn expect_print_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "println!(\"{}\", string)");
+    assert_eq!(item.to_string(), "println ! (\"{}\", string)");
     item
 }
 

src/test/ui/proc-macro/auxiliary/attr-stmt-expr.rs

@@ -10,28 +10,28 @@ use proc_macro::TokenStream;
 #[proc_macro_attribute]
 pub fn expect_let(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "let string = \"Hello, world!\";");
+    assert_eq!(item.to_string(), "let string = \"Hello, world!\" ;");
     item
 }
 
 #[proc_macro_attribute]
 pub fn expect_print_stmt(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "println!(\"{}\", string);");
+    assert_eq!(item.to_string(), "println ! (\"{}\", string) ;");
     item
 }
 
 #[proc_macro_attribute]
 pub fn expect_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "print_str(\"string\")");
+    assert_eq!(item.to_string(), "print_str (\"string\")");
     item
 }
 
 #[proc_macro_attribute]
 pub fn expect_print_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "println!(\"{}\", string)");
+    assert_eq!(item.to_string(), "println ! (\"{}\", string)");
     item
 }
 

src/test/ui/proc-macro/dollar-crate-issue-57089.stdout

@@ -38,8 +38,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
         span: #3 bytes(LO..HI),
     },
 ]
-PRINT-ATTR INPUT (DISPLAY): struct A(crate::S);
-PRINT-ATTR RE-COLLECTED (DISPLAY): struct A ($crate :: S) ;
+PRINT-ATTR INPUT (DISPLAY): struct A ($crate :: S) ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",

src/test/ui/proc-macro/dollar-crate-issue-62325.stdout

@@ -1,5 +1,4 @@
-PRINT-ATTR INPUT (DISPLAY): struct A(identity!(crate :: S));
-PRINT-ATTR RE-COLLECTED (DISPLAY): struct A (identity ! ($crate :: S)) ;
+PRINT-ATTR INPUT (DISPLAY): struct A (identity ! ($crate :: S)) ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",
@@ -54,8 +53,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
         span: #3 bytes(LO..HI),
     },
 ]
-PRINT-ATTR INPUT (DISPLAY): struct B(identity!(::dollar_crate_external :: S));
-PRINT-ATTR RE-COLLECTED (DISPLAY): struct B (identity ! ($crate :: S)) ;
+PRINT-ATTR INPUT (DISPLAY): struct B (identity ! ($crate :: S)) ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",

src/test/ui/proc-macro/dollar-crate.stdout

@@ -38,8 +38,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
         span: #3 bytes(LO..HI),
     },
 ]
-PRINT-ATTR INPUT (DISPLAY): struct A(crate::S);
-PRINT-ATTR RE-COLLECTED (DISPLAY): struct A ($crate :: S) ;
+PRINT-ATTR INPUT (DISPLAY): struct A ($crate :: S) ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",
@@ -160,8 +159,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
         span: #13 bytes(LO..HI),
     },
 ]
-PRINT-ATTR INPUT (DISPLAY): struct A(::dollar_crate_external::S);
-PRINT-ATTR RE-COLLECTED (DISPLAY): struct A ($crate :: S) ;
+PRINT-ATTR INPUT (DISPLAY): struct A ($crate :: S) ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",

src/test/ui/rfc-2565-param-attrs/auxiliary/param-attrs.rs

@@ -17,28 +17,28 @@ macro_rules! checker {
     }
 }
 
-checker!(attr_extern, r#"extern "C" {
-    fn ffi(#[a1] arg1: i32, #[a2] ...);
-}"#);
-checker!(attr_extern_cvar, r#"unsafe extern "C" fn cvar(arg1: i32, #[a1] mut args: ...) { }"#);
-checker!(attr_alias, "type Alias = fn(#[a1] u8, #[a2] ...);");
-checker!(attr_free, "fn free(#[a1] arg1: u8) { let lam = |#[a2] W(x), #[a3] y| (); }");
-checker!(attr_inherent_1, "fn inherent1(#[a1] self, #[a2] arg1: u8) { }");
-checker!(attr_inherent_2, "fn inherent2(#[a1] &self, #[a2] arg1: u8) { }");
-checker!(attr_inherent_3, "fn inherent3<'a>(#[a1] &'a mut self, #[a2] arg1: u8) { }");
-checker!(attr_inherent_4, "fn inherent4<'a>(#[a1] self: Box<Self>, #[a2] arg1: u8) { }");
-checker!(attr_inherent_issue_64682, "fn inherent5(#[a1] #[a2] arg1: u8, #[a3] arg2: u8) { }");
-checker!(attr_trait_1, "fn trait1(#[a1] self, #[a2] arg1: u8);");
-checker!(attr_trait_2, "fn trait2(#[a1] &self, #[a2] arg1: u8);");
-checker!(attr_trait_3, "fn trait3<'a>(#[a1] &'a mut self, #[a2] arg1: u8);");
-checker!(attr_trait_4, "fn trait4<'a>(#[a1] self: Box<Self>, #[a2] arg1: u8, #[a3] Vec<u8>);");
-checker!(attr_trait_issue_64682, "fn trait5(#[a1] #[a2] arg1: u8, #[a3] arg2: u8);");
-checker!(rename_params, r#"impl Foo {
-    fn hello(#[angery(true)] a: i32, #[a2] b: i32, #[what = "how"] c: u32) { }
-    fn hello2(#[a1] #[a2] a: i32, #[what = "how"] b: i32,
-              #[angery(true)] c: u32) {
-    }
-    fn hello_self(#[a1] #[a2] &self, #[a1] #[a2] a: i32,
-                  #[what = "how"] b: i32, #[angery(true)] c: u32) {
-    }
+checker!(attr_extern, r#"extern "C" { fn ffi (# [a1] arg1 : i32, # [a2] ...) ; }"#);
+checker!(attr_extern_cvar, r#"unsafe extern "C" fn cvar (arg1 : i32, # [a1] mut args : ...) { }"#);
+checker!(attr_alias, "type Alias = fn (# [a1] u8, # [a2] ...) ;");
+checker!(attr_free, "fn free (# [a1] arg1 : u8) { let lam = | # [a2] W (x), # [a3] y | () ; }");
+checker!(attr_inherent_1, "fn inherent1 (# [a1] self, # [a2] arg1 : u8) { }");
+checker!(attr_inherent_2, "fn inherent2 (# [a1] & self, # [a2] arg1 : u8) { }");
+checker!(attr_inherent_3, "fn inherent3 < 'a > (# [a1] & 'a mut self, # [a2] arg1 : u8) { }");
+checker!(attr_inherent_4, "fn inherent4 < 'a > (# [a1] self : Box < Self >, # [a2] arg1 : u8) { }");
+checker!(attr_inherent_issue_64682, "fn inherent5 (# [a1] # [a2] arg1 : u8, # [a3] arg2 : u8) { }");
+checker!(attr_trait_1, "fn trait1 (# [a1] self, # [a2] arg1 : u8) ;");
+checker!(attr_trait_2, "fn trait2 (# [a1] & self, # [a2] arg1 : u8) ;");
+checker!(attr_trait_3, "fn trait3 < 'a > (# [a1] & 'a mut self, # [a2] arg1 : u8) ;");
+checker!(attr_trait_4, r#"fn trait4 < 'a >
+(# [a1] self : Box < Self >, # [a2] arg1 : u8, # [a3] Vec < u8 >) ;"#);
+checker!(attr_trait_issue_64682, "fn trait5 (# [a1] # [a2] arg1 : u8, # [a3] arg2 : u8) ;");
+checker!(rename_params, r#"impl Foo
+{
+    fn hello
+    (# [angery (true)] a : i32, # [a2] b : i32, # [what = "how"] c : u32) { }
+    fn hello2
+    (# [a1] # [a2] a : i32, # [what = "how"] b : i32, # [angery (true)] c :
+     u32) { } fn hello_self
+    (# [a1] # [a2] & self, # [a1] # [a2] a : i32, # [what = "how"] b : i32, #
+     [angery (true)] c : u32) { }
 }"#);