rustc: Tweak custom attribute capabilities

This commit starts to lay some groundwork for the stabilization of custom
attribute invocations and general procedural macros. It applies a number of
changes discussed on [internals] as well as a [recent issue][issue], namely:

* The path used to specify a custom attribute must be of length one and cannot
  be a global path. This'll help future-proof us against any ambiguities and
  give us more time to settle the precise syntax. In the meantime though a bare
  identifier can be used and imported to invoke a custom attribute macro. A new
  feature gate, `proc_macro_path_invoc`, was added to gate multi-segment paths
  and absolute paths.

* The set of items which can be annotated by a custom procedural attribute has
  been restricted. Statements, expressions, and modules are disallowed behind
  two new feature gates: `proc_macro_expr` and `proc_macro_mod`.

* The input to procedural macro attributes has been restricted and adjusted.
  Today an invocation like `#[foo(bar)]` will receive `(bar)` as the input token
  stream, but after this PR it will only receive `bar` (the delimiters were
  removed). Invocations like `#[foo]` are still allowed and will be invoked in
  the same way as `#[foo()]`. This is a **breaking change** for all nightly
  users as the syntax coming in to procedural macros will be tweaked slightly.

* Procedural macros (`foo!()` style) can only be expanded to item-like items by
  default. A separate feature gate, `proc_macro_non_items`, is required to
  expand to items like expressions, statements, etc.

Closes #50038

[internals]: https://internals.rust-lang.org/t/help-stabilize-a-subset-of-macros-2-0/7252
[issue]: #50038

src/librustc_resolve/macros.rs

@@ -397,6 +397,18 @@ impl<'a> Resolver<'a> {
 
     fn resolve_macro_to_def(&mut self, scope: Mark, path: &ast::Path, kind: MacroKind, force: bool)
                             -> Result<Def, Determinacy> {
+        if path.segments.len() > 1 {
+            if !self.session.features_untracked().proc_macro_path_invoc {
+                emit_feature_err(
+                    &self.session.parse_sess,
+                    "proc_macro_path_invoc",
+                    path.span,
+                    GateIssue::Language,
+                    "paths of length greater than one in macro invocations are \
+                     currently unstable",
+                );
+            }
+        }
         let def = self.resolve_macro_to_def_inner(scope, path, kind, force);
         if def != Err(Determinacy::Undetermined) {
             // Do not report duplicated errors on every undetermined resolution.

src/libsyntax/ext/expand.rs

@@ -514,6 +514,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 Some(kind.expect_from_annotatables(items))
             }
             AttrProcMacro(ref mac) => {
+                self.gate_proc_macro_attr_item(attr.span, &item);
                 let item_tok = TokenTree::Token(DUMMY_SP, Token::interpolated(match item {
                     Annotatable::Item(item) => token::NtItem(item),
                     Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
@@ -522,7 +523,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
                     Annotatable::Expr(expr) => token::NtExpr(expr),
                 })).into();
-                let tok_result = mac.expand(self.cx, attr.span, attr.tokens, item_tok);
+                let input = self.extract_proc_macro_attr_input(attr.tokens, attr.span);
+                let tok_result = mac.expand(self.cx, attr.span, input, item_tok);
                 self.parse_expansion(tok_result, kind, &attr.path, attr.span)
             }
             ProcMacroDerive(..) | BuiltinDerive(..) => {
@@ -539,6 +541,49 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         }
     }
 
+    fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
+        let mut trees = tokens.trees();
+        match trees.next() {
+            Some(TokenTree::Delimited(_, delim)) => {
+                if trees.next().is_none() {
+                    return delim.tts.into()
+                }
+            }
+            Some(TokenTree::Token(..)) => {}
+            None => return TokenStream::empty(),
+        }
+        self.cx.span_err(span, "custom attribute invocations must be \
+            of the form #[foo] or #[foo(..)], the macro name must only be \
+            followed by a delimiter token");
+        TokenStream::empty()
+    }
+
+    fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) {
+        let (kind, gate) = match *item {
+            Annotatable::Item(ref item) => {
+                match item.node {
+                    ItemKind::Mod(_) if self.cx.ecfg.proc_macro_mod() => return,
+                    ItemKind::Mod(_) => ("modules", "proc_macro_mod"),
+                    _ => return,
+                }
+            }
+            Annotatable::TraitItem(_) => return,
+            Annotatable::ImplItem(_) => return,
+            Annotatable::ForeignItem(_) => return,
+            Annotatable::Stmt(_) |
+            Annotatable::Expr(_) if self.cx.ecfg.proc_macro_expr() => return,
+            Annotatable::Stmt(_) => ("statements", "proc_macro_expr"),
+            Annotatable::Expr(_) => ("expressions", "proc_macro_expr"),
+        };
+        emit_feature_err(
+            self.cx.parse_sess,
+            gate,
+            span,
+            GateIssue::Language,
+            &format!("custom attributes cannot be applied to {}", kind),
+        );
+    }
+
     /// Expand a macro invocation. Returns the result of expansion.
     fn expand_bang_invoc(&mut self,
                          invoc: Invocation,
@@ -665,6 +710,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     self.cx.trace_macros_diag();
                     kind.dummy(span)
                 } else {
+                    self.gate_proc_macro_expansion_kind(span, kind);
                     invoc.expansion_data.mark.set_expn_info(ExpnInfo {
                         call_site: span,
                         callee: NameAndSpan {
@@ -695,6 +741,30 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         }
     }
 
+    fn gate_proc_macro_expansion_kind(&self, span: Span, kind: ExpansionKind) {
+        let kind = match kind {
+            ExpansionKind::Expr => "expressions",
+            ExpansionKind::OptExpr => "expressions",
+            ExpansionKind::Pat => "patterns",
+            ExpansionKind::Ty => "types",
+            ExpansionKind::Stmts => "statements",
+            ExpansionKind::Items => return,
+            ExpansionKind::TraitItems => return,
+            ExpansionKind::ImplItems => return,
+            ExpansionKind::ForeignItems => return,
+        };
+        if self.cx.ecfg.proc_macro_non_items() {
+            return
+        }
+        emit_feature_err(
+            self.cx.parse_sess,
+            "proc_macro_non_items",
+            span,
+            GateIssue::Language,
+            &format!("procedural macros cannot be expanded to {}", kind),
+        );
+    }
+
     /// Expand a derive invocation. Returns the result of expansion.
     fn expand_derive_invoc(&mut self,
                            invoc: Invocation,
@@ -1370,6 +1440,9 @@ impl<'feat> ExpansionConfig<'feat> {
         fn enable_custom_derive = custom_derive,
         fn proc_macro_enabled = proc_macro,
         fn macros_in_extern_enabled = macros_in_extern,
+        fn proc_macro_mod = proc_macro_mod,
+        fn proc_macro_expr = proc_macro_expr,
+        fn proc_macro_non_items = proc_macro_non_items,
     }
 }
 

src/libsyntax/feature_gate.rs

@@ -451,6 +451,15 @@ declare_features! (
     (active, mmx_target_feature, "1.27.0", None, None),
     (active, sse4a_target_feature, "1.27.0", None, None),
     (active, tbm_target_feature, "1.27.0", None, None),
+
+    // Allows macro invocations of the form `#[foo::bar]`
+    (active, proc_macro_path_invoc, "1.27.0", None, None),
+
+    // Allows macro invocations on modules expressions and statements and
+    // procedural macros to expand to non-items.
+    (active, proc_macro_mod, "1.27.0", None, None),
+    (active, proc_macro_expr, "1.27.0", None, None),
+    (active, proc_macro_non_items, "1.27.0", None, None),
 );
 
 declare_features! (

src/test/compile-fail-fulldeps/proc-macro/attr-invalid-exprs.rs

@@ -13,7 +13,7 @@
 
 //! Attributes producing expressions in invalid locations
 
-#![feature(proc_macro, stmt_expr_attributes)]
+#![feature(proc_macro, stmt_expr_attributes, proc_macro_expr)]
 
 extern crate attr_stmt_expr;
 use attr_stmt_expr::{duplicate, no_output};

src/test/compile-fail-fulldeps/proc-macro/attr-stmt-expr.rs

@@ -11,7 +11,7 @@
 // aux-build:attr-stmt-expr.rs
 // ignore-stage1
 
-#![feature(proc_macro)]
+#![feature(proc_macro, proc_macro_expr)]
 
 extern crate attr_stmt_expr;
 use attr_stmt_expr::{expect_let, expect_print_stmt, expect_expr, expect_print_expr};

src/test/compile-fail-fulldeps/proc-macro/attributes-included.rs

@@ -11,7 +11,7 @@
 // aux-build:attributes-included.rs
 // ignore-stage1
 
-#![feature(proc_macro, rustc_attrs)]
+#![feature(proc_macro, rustc_attrs, proc_macro_path_invoc)]
 #![warn(unused)]
 
 extern crate attributes_included;

src/test/compile-fail-fulldeps/proc-macro/auxiliary/proc-macro-gates.rs

@@ -0,0 +1,29 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// no-prefer-dynamic
+// force-host
+
+#![crate_type = "proc-macro"]
+#![feature(proc_macro)]
+
+extern crate proc_macro;
+
+use proc_macro::*;
+
+#[proc_macro]
+pub fn m(a: TokenStream) -> TokenStream {
+    a
+}
+
+#[proc_macro_attribute]
+pub fn a(_a: TokenStream, b: TokenStream) -> TokenStream {
+    b
+}

src/test/compile-fail-fulldeps/proc-macro/lints_in_proc_macros.rs

@@ -11,7 +11,7 @@
 // aux-build:bang_proc_macro2.rs
 // ignore-stage1
 
-#![feature(proc_macro)]
+#![feature(proc_macro, proc_macro_non_items)]
 #![allow(unused_macros)]
 
 extern crate bang_proc_macro2;

src/test/compile-fail-fulldeps/proc-macro/macro-use-bang.rs

@@ -10,7 +10,7 @@
 
 // aux-build:bang_proc_macro.rs
 
-#![feature(proc_macro)]
+#![feature(proc_macro, proc_macro_non_items)]
 
 #[macro_use]
 extern crate bang_proc_macro;

src/test/compile-fail-fulldeps/proc-macro/proc-macro-gates.rs

@@ -0,0 +1,54 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:proc-macro-gates.rs
+// gate-test-proc_macro_non_items
+// gate-test-proc_macro_path_invoc
+// gate-test-proc_macro_mod line
+// gate-test-proc_macro_expr
+// gate-test-proc_macro_mod
+
+#![feature(proc_macro, stmt_expr_attributes)]
+
+extern crate proc_macro_gates as foo;
+
+use foo::*;
+
+#[foo::a] //~ ERROR: paths of length greater than one
+fn _test() {}
+
+#[a] //~ ERROR: custom attributes cannot be applied to modules
+mod _test2 {}
+
+#[a = y] //~ ERROR: must only be followed by a delimiter token
+fn _test3() {}
+
+#[a = ] //~ ERROR: must only be followed by a delimiter token
+fn _test4() {}
+
+#[a () = ] //~ ERROR: must only be followed by a delimiter token
+fn _test5() {}
+
+fn main() {
+    #[a] //~ ERROR: custom attributes cannot be applied to statements
+    let _x = 2;
+    let _x = #[a] 2;
+    //~^ ERROR: custom attributes cannot be applied to expressions
+
+    let _x: m!(u32) = 3;
+    //~^ ERROR: procedural macros cannot be expanded to types
+    if let m!(Some(_x)) = Some(3) {
+    //~^ ERROR: procedural macros cannot be expanded to patterns
+    }
+    let _x = m!(3);
+    //~^ ERROR: procedural macros cannot be expanded to expressions
+    m!(let _x = 3;);
+    //~^ ERROR: procedural macros cannot be expanded to statements
+}

src/test/compile-fail-fulldeps/proc-macro/proc-macro-gates2.rs

@@ -0,0 +1,35 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:proc-macro-gates.rs
+
+#![feature(proc_macro, stmt_expr_attributes)]
+
+extern crate proc_macro_gates as foo;
+
+use foo::*;
+
+// NB. these errors aren't the best errors right now, but they're definitely
+// intended to be errors. Somehow using a custom attribute in these positions
+// should either require a feature gate or not be allowed on stable.
+
+fn _test6<#[a] T>() {}
+//~^ ERROR: unknown to the compiler
+
+fn _test7() {
+    match 1 {
+        #[a] //~ ERROR: unknown to the compiler
+        0 => {}
+        _ => {}
+    }
+}
+
+fn main() {
+}

src/test/compile-fail/extern-macro.rs

@@ -10,7 +10,7 @@
 
 // #41719
 
-#![feature(use_extern_macros)]
+#![feature(use_extern_macros, proc_macro_path_invoc)]
 
 fn main() {
     enum Foo {}

src/test/compile-fail/macro-with-seps-err-msg.rs

@@ -10,6 +10,8 @@
 
 // gate-test-use_extern_macros
 
+#![feature(proc_macro_path_invoc)]
+
 fn main() {
     globnar::brotz!(); //~ ERROR non-ident macro paths are experimental
     #[derive(foo::Bar)] struct T; //~ ERROR non-ident macro paths are experimental

src/test/compile-fail/macros-nonfatal-errors.rs

@@ -13,6 +13,7 @@
 
 #![feature(asm)]
 #![feature(trace_macros, concat_idents)]
+#![feature(proc_macro_path_invoc)]
 
 #[derive(Default)] //~ ERROR
 enum OrDeriveThis {}

src/test/compile-fail/privacy/associated-item-privacy-inherent.rs

@@ -10,6 +10,7 @@
 
 #![feature(decl_macro, associated_type_defaults)]
 #![allow(unused, private_in_public)]
+#![feature(proc_macro_path_invoc)]
 
 mod priv_nominal {
     pub struct Pub;

src/test/compile-fail/privacy/associated-item-privacy-trait.rs

@@ -10,6 +10,7 @@
 
 // ignore-tidy-linelength
 
+#![feature(proc_macro_path_invoc)]
 #![feature(decl_macro, associated_type_defaults)]
 #![allow(unused, private_in_public)]
 

src/test/compile-fail/privacy/associated-item-privacy-type-binding.rs

@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#![feature(proc_macro_path_invoc)]
 #![feature(decl_macro, associated_type_defaults)]
 #![allow(unused, private_in_public)]
 

src/test/compile-fail/private-inferred-type-3.rs

@@ -18,6 +18,7 @@
 // error-pattern:type `fn(u8) -> ext::PubTupleStruct {ext::PubTupleStruct::{{constructor}}}` is priv
 // error-pattern:type `for<'r> fn(&'r ext::Pub<u8>) {<ext::Pub<u8>>::priv_method}` is private
 
+#![feature(proc_macro_path_invoc)]
 #![feature(decl_macro)]
 
 extern crate private_inferred_type as ext;

src/test/compile-fail/private-inferred-type.rs

@@ -11,6 +11,7 @@
 #![feature(associated_consts)]
 #![feature(decl_macro)]
 #![allow(private_in_public)]
+#![feature(proc_macro_path_invoc)]
 
 mod m {
     fn priv_fn() {}

src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs

@@ -11,7 +11,7 @@
 // no-prefer-dynamic
 
 #![crate_type = "proc-macro"]
-#![feature(proc_macro)]
+#![feature(proc_macro, proc_macro_non_items)]
 
 extern crate proc_macro;