Auto merge of #62086 - petrochenkov:builtout, r=eddyb

Define built-in macros through libcore

This PR defines built-in macros through libcore using a scheme similar to lang items (attribute `#[rustc_builtin_macro]`).
All the macro properties (stability, visibility, etc.) are taken from the source code in libcore, with exception of the expander function transforming input tokens/AST into output tokens/AST, which is still provided by the compiler.

The macros are made available to user code through the standard library prelude (`{core,std}::prelude::v1`), so they are still always in scope.
As a result **built-in macros now have stable absolute addresses in the library**, like `core::prelude::v1::line!()`, this is an insta-stable change.

Right now `prelude::v1` is the only publicly available absolute address for these macros, but eventually they can be moved into more appropriate locations with library team approval (e.g. `Clone` derive -> `core::clone::Clone`).

Now when built-in macros have canonical definitions they can be imported or reexported without issues (#61687).

Other changes:
- You can now define a derive macro with a name matching one of the built-in derives (#52269). This was an artificial restriction that could be worked around with import renaming anyway.

Known regressions:
- Empty library crate with a crate-level `#![test]` attribute no longer compiles without `--test`. Previously it didn't compile *with* `--test` or with the bin crate type.

Fixes #61687
Fixes #61804
r? @eddyb

src/libcore/macros.rs

@@ -657,13 +657,13 @@ macro_rules! uninit_array {
     );
 }
 
-/// Built-in macros to the compiler itself.
+/// Definitions of built-in macros.
 ///
-/// These macros do not have any corresponding definition with a `macro_rules!`
-/// macro, but are documented here. Their implementations can be found hardcoded
-/// into libsyntax itself.
-#[cfg(rustdoc)]
-mod builtin {
+/// Most of the macro properties (stability, visibility, etc.) are taken from the source code here,
+/// with exception of expansion functions transforming macro inputs into outputs,
+/// those functions are provided by the compiler.
+#[cfg(not(bootstrap))]
+pub(crate) mod builtin {
 
     /// Causes compilation to fail with the given error message when encountered.
     ///
@@ -950,7 +950,7 @@ mod builtin {
 
     /// Same as `column`, but less likely to be shadowed.
     #[unstable(feature = "__rust_unstable_column", issue = "0",
-               reason = "internal implementation detail of the `column` macro")]
+               reason = "internal implementation detail of the `panic` macro")]
     #[rustc_builtin_macro]
     #[rustc_macro_transparency = "semitransparent"]
     pub macro __rust_unstable_column() { /* compiler built-in */ }

src/libcore/prelude/v1.rs

@@ -44,3 +44,50 @@ pub use crate::option::Option::{self, Some, None};
 #[stable(feature = "core_prelude", since = "1.4.0")]
 #[doc(no_inline)]
 pub use crate::result::Result::{self, Ok, Err};
+
+// Re-exported built-in macros
+#[cfg(not(bootstrap))]
+#[stable(feature = "builtin_macro_prelude", since = "1.38.0")]
+#[allow(deprecated)]
+#[doc(no_inline)]
+pub use crate::macros::builtin::{
+    Clone,
+    Copy,
+    Debug,
+    Decodable,
+    Default,
+    Encodable,
+    Eq,
+    Hash,
+    Ord,
+    PartialEq,
+    PartialOrd,
+    RustcDecodable,
+    RustcEncodable,
+    __rust_unstable_column,
+    asm,
+    assert,
+    bench,
+    cfg,
+    column,
+    compile_error,
+    concat,
+    concat_idents,
+    env,
+    file,
+    format_args,
+    format_args_nl,
+    global_allocator,
+    global_asm,
+    include,
+    include_bytes,
+    include_str,
+    line,
+    log_syntax,
+    module_path,
+    option_env,
+    stringify,
+    test,
+    test_case,
+    trace_macros,
+};

src/libcore/task/wake.rs

@@ -217,7 +217,7 @@ impl fmt::Debug for Context<'_> {
 /// This handle encapsulates a [`RawWaker`] instance, which defines the
 /// executor-specific wakeup behavior.
 ///
-/// Implements [`Clone`], [`Send`], and [`Sync`].
+/// Implements [`Clone`], [`trait@Send`], and [`trait@Sync`].
 ///
 /// [`RawWaker`]: struct.RawWaker.html
 #[repr(transparent)]

src/librustc/hir/def_id.rs

@@ -12,10 +12,6 @@ newtype_index! {
 
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum CrateNum {
-    /// Virtual crate for builtin macros
-    // FIXME(jseyfried): this is also used for custom derives until proc-macro crates get
-    // `CrateNum`s.
-    BuiltinMacros,
     /// A special CrateNum that we use for the tcx.rcache when decoding from
     /// the incr. comp. cache.
     ReservedForIncrCompCache,
@@ -26,7 +22,6 @@ impl ::std::fmt::Debug for CrateNum {
     fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
         match self {
             CrateNum::Index(id) => write!(fmt, "crate{}", id.private),
-            CrateNum::BuiltinMacros => write!(fmt, "builtin macros crate"),
             CrateNum::ReservedForIncrCompCache => write!(fmt, "crate for decoding incr comp cache"),
         }
     }
@@ -86,7 +81,6 @@ impl fmt::Display for CrateNum {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             CrateNum::Index(id) => fmt::Display::fmt(&id.private, f),
-            CrateNum::BuiltinMacros => write!(f, "builtin macros crate"),
             CrateNum::ReservedForIncrCompCache => write!(f, "crate for decoding incr comp cache"),
         }
     }

src/librustc/hir/lowering.rs

@@ -4009,8 +4009,7 @@ impl<'a> LoweringContext<'a> {
         let mut vis = self.lower_visibility(&i.vis, None);
         let attrs = self.lower_attrs(&i.attrs);
         if let ItemKind::MacroDef(ref def) = i.node {
-            if !def.legacy || attr::contains_name(&i.attrs, sym::macro_export) ||
-                              attr::contains_name(&i.attrs, sym::rustc_builtin_macro) {
+            if !def.legacy || attr::contains_name(&i.attrs, sym::macro_export) {
                 let body = self.lower_token_stream(def.stream());
                 let hir_id = self.lower_node_id(i.id);
                 self.exported_macros.push(hir::MacroDef {

src/librustc_interface/passes.rs

@@ -372,7 +372,10 @@ fn configure_and_expand_inner<'a>(
         crate_loader,
         &resolver_arenas,
     );
-    syntax_ext::register_builtins(&mut resolver, plugin_info.syntax_exts, sess.edition());
+    syntax_ext::register_builtin_macros(&mut resolver, sess.edition());
+    syntax_ext::plugin_macro_defs::inject(
+        &mut krate, &mut resolver, plugin_info.syntax_exts, sess.edition()
+    );
 
     // Expand all macros
     sess.profiler(|p| p.start_activity("macro expansion"));

src/librustc_resolve/build_reduced_graph.rs

@@ -13,7 +13,7 @@ use crate::{resolve_error, resolve_struct_error, ResolutionError, Determinacy};
 
 use rustc::bug;
 use rustc::hir::def::{self, *};
-use rustc::hir::def_id::{CrateNum, CRATE_DEF_INDEX, LOCAL_CRATE, DefId};
+use rustc::hir::def_id::{CRATE_DEF_INDEX, LOCAL_CRATE, DefId};
 use rustc::ty;
 use rustc::middle::cstore::CrateStore;
 use rustc_metadata::cstore::LoadedMacro;
@@ -31,7 +31,6 @@ use syntax::ast::{self, Block, ForeignItem, ForeignItemKind, Item, ItemKind, Nod
 use syntax::ast::{MetaItemKind, StmtKind, TraitItem, TraitItemKind, Variant};
 use syntax::ext::base::SyntaxExtension;
 use syntax::ext::hygiene::ExpnId;
-use syntax::ext::tt::macro_rules;
 use syntax::feature_gate::is_builtin_attr;
 use syntax::parse::token::{self, Token};
 use syntax::span_err;
@@ -748,21 +747,24 @@ impl<'a> Resolver<'a> {
         };
         if let Some(id) = self.definitions.as_local_node_id(def_id) {
             self.local_macro_def_scopes[&id]
-        } else if def_id.krate == CrateNum::BuiltinMacros {
+        } else if self.is_builtin_macro(Some(def_id)) {
             self.injected_crate.unwrap_or(self.graph_root)
         } else {
             let module_def_id = ty::DefIdTree::parent(&*self, def_id).unwrap();
             self.get_module(module_def_id)
         }
     }
 
-    pub fn get_macro(&mut self, res: Res) -> Option<Lrc<SyntaxExtension>> {
-        let def_id = match res {
-            Res::Def(DefKind::Macro(..), def_id) => def_id,
+    crate fn get_macro(&mut self, res: Res) -> Option<Lrc<SyntaxExtension>> {
+        match res {
+            Res::Def(DefKind::Macro(..), def_id) => self.get_macro_by_def_id(def_id),
             Res::NonMacroAttr(attr_kind) =>
-                return Some(self.non_macro_attr(attr_kind == NonMacroAttrKind::Tool)),
-            _ => return None,
-        };
+                Some(self.non_macro_attr(attr_kind == NonMacroAttrKind::Tool)),
+            _ => None,
+        }
+    }
+
+    crate fn get_macro_by_def_id(&mut self, def_id: DefId) -> Option<Lrc<SyntaxExtension>> {
         if let Some(ext) = self.macro_map.get(&def_id) {
             return Some(ext.clone());
         }
@@ -772,10 +774,7 @@ impl<'a> Resolver<'a> {
             LoadedMacro::ProcMacro(ext) => return Some(ext),
         };
 
-        let ext = Lrc::new(macro_rules::compile(&self.session.parse_sess,
-                                               &self.session.features_untracked(),
-                                               &macro_def,
-                                               self.cstore.crate_edition_untracked(def_id.krate)));
+        let ext = self.compile_macro(&macro_def, self.cstore.crate_edition_untracked(def_id.krate));
         self.macro_map.insert(def_id, ext.clone());
         Some(ext)
     }

src/librustc_resolve/diagnostics.rs

@@ -35,17 +35,12 @@ enum AssocSuggestion {
 
 struct TypoSuggestion {
     candidate: Symbol,
-
-    /// The kind of the binding ("crate", "module", etc.)
-    kind: &'static str,
-
-    /// An appropriate article to refer to the binding ("a", "an", etc.)
-    article: &'static str,
+    res: Res,
 }
 
 impl TypoSuggestion {
     fn from_res(candidate: Symbol, res: Res) -> TypoSuggestion {
-        TypoSuggestion { candidate, kind: res.descr(), article: res.article() }
+        TypoSuggestion { candidate, res }
     }
 }
 
@@ -59,7 +54,9 @@ fn add_typo_suggestion(
     err: &mut DiagnosticBuilder<'_>, suggestion: Option<TypoSuggestion>, span: Span
 ) -> bool {
     if let Some(suggestion) = suggestion {
-        let msg = format!("{} {} with a similar name exists", suggestion.article, suggestion.kind);
+        let msg = format!(
+            "{} {} with a similar name exists", suggestion.res.article(), suggestion.res.descr()
+        );
         err.span_suggestion(
             span, &msg, suggestion.candidate.to_string(), Applicability::MaybeIncorrect
         );
@@ -566,7 +563,7 @@ impl<'a> Resolver<'a> {
         filter_fn: &impl Fn(Res) -> bool,
     ) -> Option<TypoSuggestion> {
         let mut suggestions = Vec::new();
-        self.visit_scopes(scope_set, parent_scope, ident, |this, scope, _| {
+        self.visit_scopes(scope_set, parent_scope, ident, |this, scope, use_prelude, _| {
             match scope {
                 Scope::DeriveHelpers => {
                     let res = Res::NonMacroAttr(NonMacroAttrKind::DeriveHelper);
@@ -611,16 +608,6 @@ impl<'a> Resolver<'a> {
                         }
                     }));
                 }
-                Scope::BuiltinMacros => {
-                    suggestions.extend(this.builtin_macros.iter().filter_map(|(name, binding)| {
-                        let res = binding.res();
-                        if filter_fn(res) {
-                            Some(TypoSuggestion::from_res(*name, res))
-                        } else {
-                            None
-                        }
-                    }));
-                }
                 Scope::BuiltinAttrs => {
                     let res = Res::NonMacroAttr(NonMacroAttrKind::Builtin);
                     if filter_fn(res) {
@@ -656,7 +643,11 @@ impl<'a> Resolver<'a> {
                 }
                 Scope::StdLibPrelude => {
                     if let Some(prelude) = this.prelude {
-                        add_module_candidates(prelude, &mut suggestions, filter_fn);
+                        let mut tmp_suggestions = Vec::new();
+                        add_module_candidates(prelude, &mut tmp_suggestions, filter_fn);
+                        suggestions.extend(tmp_suggestions.into_iter().filter(|s| {
+                            use_prelude || this.is_builtin_macro(s.res.opt_def_id())
+                        }));
                     }
                 }
                 Scope::BuiltinTypes => {
@@ -733,11 +724,7 @@ impl<'a> Resolver<'a> {
                                         );
 
                                         if filter_fn(crate_mod) {
-                                            Some(TypoSuggestion {
-                                                candidate: ident.name,
-                                                article: "a",
-                                                kind: "crate",
-                                            })
+                                            Some(TypoSuggestion::from_res(ident.name, crate_mod))
                                         } else {
                                             None
                                         }

src/librustc_resolve/lib.rs

@@ -114,7 +114,6 @@ enum Scope<'a> {
     CrateRoot,
     Module(Module<'a>),
     MacroUsePrelude,
-    BuiltinMacros,
     BuiltinAttrs,
     LegacyPluginHelpers,
     ExternPrelude,
@@ -1679,7 +1678,7 @@ pub struct Resolver<'a> {
 
     crate_loader: &'a mut CrateLoader<'a>,
     macro_names: FxHashSet<Ident>,
-    builtin_macros: FxHashMap<Name, &'a NameBinding<'a>>,
+    builtin_macros: FxHashMap<Name, SyntaxExtension>,
     macro_use_prelude: FxHashMap<Name, &'a NameBinding<'a>>,
     pub all_macros: FxHashMap<Name, Res>,
     macro_map: FxHashMap<DefId, Lrc<SyntaxExtension>>,
@@ -2021,7 +2020,7 @@ impl<'a> Resolver<'a> {
 
             crate_loader,
             macro_names: FxHashSet::default(),
-            builtin_macros: FxHashMap::default(),
+            builtin_macros: Default::default(),
             macro_use_prelude: FxHashMap::default(),
             all_macros: FxHashMap::default(),
             macro_map: FxHashMap::default(),
@@ -2068,6 +2067,11 @@ impl<'a> Resolver<'a> {
         f(self, MacroNS);
     }
 
+    fn is_builtin_macro(&mut self, def_id: Option<DefId>) -> bool {
+        def_id.and_then(|def_id| self.get_macro_by_def_id(def_id))
+              .map_or(false, |ext| ext.is_builtin)
+    }
+
     fn macro_def(&self, mut ctxt: SyntaxContext) -> DefId {
         loop {
             match self.macro_defs.get(&ctxt.outer_expn()) {
@@ -2146,7 +2150,7 @@ impl<'a> Resolver<'a> {
         scope_set: ScopeSet,
         parent_scope: &ParentScope<'a>,
         ident: Ident,
-        mut visitor: impl FnMut(&mut Self, Scope<'a>, Ident) -> Option<T>,
+        mut visitor: impl FnMut(&mut Self, Scope<'a>, /*use_prelude*/ bool, Ident) -> Option<T>,
     ) -> Option<T> {
         // General principles:
         // 1. Not controlled (user-defined) names should have higher priority than controlled names
@@ -2185,8 +2189,8 @@ impl<'a> Resolver<'a> {
         // 4. `macro_use` prelude (open, the open part is from macro expansions, not controlled).
         // 4a. User-defined prelude from macro-use
         //    (open, the open part is from macro expansions, not controlled).
-        // 4b. Standard library prelude is currently implemented as `macro-use` (closed, controlled)
-        // 5. Language prelude: builtin macros (closed, controlled, except for legacy plugins).
+        // 4b. "Standard library prelude" part implemented through `macro-use` (closed, controlled).
+        // 4c. Standard library prelude (de-facto closed, controlled).
         // 6. Language prelude: builtin attributes (closed, controlled).
         // 4-6. Legacy plugin helpers (open, not controlled). Similar to derive helpers,
         //    but introduced by legacy plugins using `register_attribute`. Priority is somewhere
@@ -2214,17 +2218,16 @@ impl<'a> Resolver<'a> {
                 Scope::CrateRoot => true,
                 Scope::Module(..) => true,
                 Scope::MacroUsePrelude => use_prelude || rust_2015,
-                Scope::BuiltinMacros => true,
                 Scope::BuiltinAttrs => true,
                 Scope::LegacyPluginHelpers => use_prelude || rust_2015,
                 Scope::ExternPrelude => use_prelude || is_absolute_path,
                 Scope::ToolPrelude => use_prelude,
-                Scope::StdLibPrelude => use_prelude,
+                Scope::StdLibPrelude => use_prelude || ns == MacroNS,
                 Scope::BuiltinTypes => true,
             };
 
             if visit {
-                if let break_result @ Some(..) = visitor(self, scope, ident) {
+                if let break_result @ Some(..) = visitor(self, scope, use_prelude, ident) {
                     return break_result;
                 }
             }
@@ -2263,7 +2266,6 @@ impl<'a> Resolver<'a> {
                     }
                 }
                 Scope::MacroUsePrelude => Scope::StdLibPrelude,
-                Scope::BuiltinMacros => Scope::BuiltinAttrs,
                 Scope::BuiltinAttrs => Scope::LegacyPluginHelpers,
                 Scope::LegacyPluginHelpers => break, // nowhere else to search
                 Scope::ExternPrelude if is_absolute_path => break,
@@ -2272,7 +2274,7 @@ impl<'a> Resolver<'a> {
                 Scope::StdLibPrelude => match ns {
                     TypeNS => Scope::BuiltinTypes,
                     ValueNS => break, // nowhere else to search
-                    MacroNS => Scope::BuiltinMacros,
+                    MacroNS => Scope::BuiltinAttrs,
                 }
                 Scope::BuiltinTypes => break, // nowhere else to search
             };