From aabb90d25489b02bd87eb2429479379f80ce5564 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 29 Apr 2024 08:38:05 +1000 Subject: [PATCH 01/14] rustc_expand: clean up attributes. Sort them, and remove the unused ones (`lint_reasons` and `proc_macro_span`). --- compiler/rustc_expand/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index 3c9d2e8706835..13e32a6a1e620 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -1,19 +1,19 @@ +// tidy-alphabetical-start +#![allow(internal_features)] +#![allow(rustc::diagnostic_outside_of_impl)] #![doc(rust_logo)] -#![feature(rustdoc_internals)] #![feature(array_windows)] #![feature(associated_type_defaults)] #![feature(if_let_guard)] #![feature(let_chains)] -#![feature(lint_reasons)] #![feature(macro_metavar_expr)] #![feature(map_try_insert)] #![feature(proc_macro_diagnostic)] #![feature(proc_macro_internals)] -#![feature(proc_macro_span)] +#![feature(rustdoc_internals)] #![feature(try_blocks)] #![feature(yeet_expr)] -#![allow(rustc::diagnostic_outside_of_impl)] -#![allow(internal_features)] +// tidy-alphabetical-end extern crate proc_macro as pm; From d8178569784a78f6e52eeb281c106de801bb398a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 30 Apr 2024 08:48:27 +1000 Subject: [PATCH 02/14] Remove an unnecessary re-export of `rustc_span::hygiene`. --- compiler/rustc_expand/src/expand.rs | 2 +- compiler/rustc_expand/src/lib.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 503c9170cab53..64a17eae6741c 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -4,7 +4,6 @@ use crate::errors::{ IncompleteParse, RecursionLimitReached, RemoveExprNotSupported, RemoveNodeNotSupported, UnsupportedKeyValue, WrongFragmentKind, }; -use crate::hygiene::SyntaxContext; use crate::mbe::diagnostics::annotate_err_with_kind; use crate::module::{mod_dir_path, parse_external_mod, DirOwnership, ParsedExternalMod}; use crate::placeholders::{placeholder, PlaceholderExpander}; @@ -32,6 +31,7 @@ use rustc_session::lint::builtin::{UNUSED_ATTRIBUTES, UNUSED_DOC_COMMENTS}; use rustc_session::lint::BuiltinLintDiag; use rustc_session::parse::feature_err; use rustc_session::{Limit, Session}; +use rustc_span::hygiene::SyntaxContext; use rustc_span::symbol::{sym, Ident}; use rustc_span::{ErrorGuaranteed, FileName, LocalExpnId, Span}; diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index 13e32a6a1e620..2e52a7f9a7da8 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -21,7 +21,6 @@ mod placeholders; mod proc_macro_server; pub use mbe::macro_rules::compile_declarative_macro; -pub(crate) use rustc_span::hygiene; pub mod base; pub mod build; #[macro_use] From 7c6d36345f8abe55c4fa57e6d194219fcc954152 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 30 Apr 2024 08:49:58 +1000 Subject: [PATCH 03/14] Remove an unnecessary `#[macro_use]`. --- compiler/rustc_expand/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index 2e52a7f9a7da8..6216965f7bfee 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -23,7 +23,6 @@ mod proc_macro_server; pub use mbe::macro_rules::compile_declarative_macro; pub mod base; pub mod build; -#[macro_use] pub mod config; pub mod errors; pub mod expand; From 3f055893b40a854441daa23aa192220bd614bd84 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 30 Apr 2024 10:08:52 +1000 Subject: [PATCH 04/14] Remove unused `ExpCtxt` methods. --- compiler/rustc_expand/src/build.rs | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs index 83f120525bc5f..1b6e191c2eb09 100644 --- a/compiler/rustc_expand/src/build.rs +++ b/compiler/rustc_expand/src/build.rs @@ -175,20 +175,6 @@ impl<'a> ExtCtxt<'a> { ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, kind: ast::StmtKind::Expr(expr) } } - pub fn stmt_let_pat(&self, sp: Span, pat: P, ex: P) -> ast::Stmt { - let local = P(ast::Local { - pat, - ty: None, - id: ast::DUMMY_NODE_ID, - kind: LocalKind::Init(ex), - span: sp, - colon_sp: None, - attrs: AttrVec::new(), - tokens: None, - }); - self.stmt_local(local, sp) - } - pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: Ident, ex: P) -> ast::Stmt { self.stmt_let_ty(sp, mutbl, ident, None, ex) } @@ -278,10 +264,6 @@ impl<'a> ExtCtxt<'a> { self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) } - pub fn expr_field(&self, span: Span, expr: P, field: Ident) -> P { - self.expr(span, ast::ExprKind::Field(expr, field)) - } - pub fn expr_macro_call(&self, span: Span, call: P) -> P { self.expr(span, ast::ExprKind::MacCall(call)) } @@ -394,11 +376,6 @@ impl<'a> ExtCtxt<'a> { self.expr(span, ast::ExprKind::Lit(lit)) } - pub fn expr_char(&self, span: Span, ch: char) -> P { - let lit = token::Lit::new(token::Char, literal::escape_char_symbol(ch), None); - self.expr(span, ast::ExprKind::Lit(lit)) - } - pub fn expr_byte_str(&self, span: Span, bytes: Vec) -> P { let lit = token::Lit::new(token::ByteStr, literal::escape_byte_str_symbol(&bytes), None); self.expr(span, ast::ExprKind::Lit(lit)) @@ -414,10 +391,6 @@ impl<'a> ExtCtxt<'a> { self.expr_addr_of(sp, self.expr_array(sp, exprs)) } - pub fn expr_cast(&self, sp: Span, expr: P, ty: P) -> P { - self.expr(sp, ast::ExprKind::Cast(expr, ty)) - } - pub fn expr_some(&self, sp: Span, expr: P) -> P { let some = self.std_path(&[sym::option, sym::Option, sym::Some]); self.expr_call_global(sp, some, thin_vec![expr]) From 189a8a692549f240d31b34138c8fc26dd8b6d118 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 30 Apr 2024 10:20:05 +1000 Subject: [PATCH 05/14] De-`pub` some `rustc_expand` modules. --- compiler/rustc_expand/src/lib.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index 6216965f7bfee..c2c3b777699ce 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -17,25 +17,23 @@ extern crate proc_macro as pm; +mod build; +mod errors; +// FIXME(Nilstrieb) Translate macro_rules diagnostics +#[allow(rustc::untranslatable_diagnostic)] +mod mbe; mod placeholders; mod proc_macro_server; pub use mbe::macro_rules::compile_declarative_macro; pub mod base; -pub mod build; pub mod config; -pub mod errors; pub mod expand; pub mod module; - // FIXME(Nilstrieb) Translate proc_macro diagnostics #[allow(rustc::untranslatable_diagnostic)] pub mod proc_macro; -// FIXME(Nilstrieb) Translate macro_rules diagnostics -#[allow(rustc::untranslatable_diagnostic)] -pub(crate) mod mbe; - // HACK(Centril, #64197): These shouldn't really be here. // Rather, they should be with their respective modules which are defined in other crates. // However, since for now constructing a `ParseSess` sorta requires `config` from this crate, From e809df6ed42f4c2c5cd37b8711f88f61c38332c8 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 30 Apr 2024 10:52:42 +1000 Subject: [PATCH 06/14] Remove unnecessary re-export of `MacroKind`. --- compiler/rustc_expand/src/base.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index 6fe74edbd70a8..ed4cb32d2dbec 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -21,7 +21,7 @@ use rustc_session::config::CollapseMacroDebuginfo; use rustc_session::{parse::ParseSess, Limit, Session}; use rustc_span::def_id::{CrateNum, DefId, LocalDefId}; use rustc_span::edition::Edition; -use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId}; +use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId, MacroKind}; use rustc_span::source_map::SourceMap; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{FileName, Span, DUMMY_SP}; @@ -32,8 +32,6 @@ use std::path::{Path, PathBuf}; use std::rc::Rc; use thin_vec::ThinVec; -pub(crate) use rustc_span::hygiene::MacroKind; - // When adding new variants, make sure to // adjust the `visit_*` / `flat_map_*` calls in `InvocationCollector` // to use `assign_id!` From d7f5319b6db4cac89b7989ecfbb1f511a95826e6 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 30 Apr 2024 12:03:08 +1000 Subject: [PATCH 07/14] Inline and remove three `DummyResult` methods. They each have a single call site. --- compiler/rustc_expand/src/base.rs | 54 +++++++++++++------------------ 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index ed4cb32d2dbec..12868a666056d 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -571,35 +571,6 @@ impl DummyResult { tokens: None, }) } - - /// A plain dummy pattern. - pub fn raw_pat(sp: Span) -> ast::Pat { - ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp, tokens: None } - } - - /// A plain dummy type. - pub fn raw_ty(sp: Span) -> P { - // FIXME(nnethercote): you might expect `ast::TyKind::Dummy` to be used here, but some - // values produced here end up being lowered to HIR, which `ast::TyKind::Dummy` does not - // support, so we use an empty tuple instead. - P(ast::Ty { - id: ast::DUMMY_NODE_ID, - kind: ast::TyKind::Tup(ThinVec::new()), - span: sp, - tokens: None, - }) - } - - /// A plain dummy crate. - pub fn raw_crate() -> ast::Crate { - ast::Crate { - attrs: Default::default(), - items: Default::default(), - spans: Default::default(), - id: ast::DUMMY_NODE_ID, - is_placeholder: Default::default(), - } - } } impl MacResult for DummyResult { @@ -608,7 +579,12 @@ impl MacResult for DummyResult { } fn make_pat(self: Box) -> Option> { - Some(P(DummyResult::raw_pat(self.span))) + Some(P(ast::Pat { + id: ast::DUMMY_NODE_ID, + kind: PatKind::Wild, + span: self.span, + tokens: None, + })) } fn make_items(self: Box) -> Option; 1]>> { @@ -636,7 +612,15 @@ impl MacResult for DummyResult { } fn make_ty(self: Box) -> Option> { - Some(DummyResult::raw_ty(self.span)) + // FIXME(nnethercote): you might expect `ast::TyKind::Dummy` to be used here, but some + // values produced here end up being lowered to HIR, which `ast::TyKind::Dummy` does not + // support, so we use an empty tuple instead. + Some(P(ast::Ty { + id: ast::DUMMY_NODE_ID, + kind: ast::TyKind::Tup(ThinVec::new()), + span: self.span, + tokens: None, + })) } fn make_arms(self: Box) -> Option> { @@ -668,7 +652,13 @@ impl MacResult for DummyResult { } fn make_crate(self: Box) -> Option { - Some(DummyResult::raw_crate()) + Some(ast::Crate { + attrs: Default::default(), + items: Default::default(), + spans: Default::default(), + id: ast::DUMMY_NODE_ID, + is_placeholder: Default::default(), + }) } } From c9c964fc379966a9fd932e1b5ac8a3327e424cd3 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 1 May 2024 09:27:20 +1000 Subject: [PATCH 08/14] Fix some comment formatting. --- compiler/rustc_expand/src/config.rs | 9 +++++---- compiler/rustc_expand/src/expand.rs | 5 +++-- compiler/rustc_expand/src/mbe/diagnostics.rs | 3 ++- compiler/rustc_expand/src/mbe/macro_rules.rs | 11 ++++++----- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index b554a5ccd1961..897420a11cdfb 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -99,10 +99,11 @@ pub fn features(sess: &Session, krate_attrs: &[Attribute], crate_name: Symbol) - // If the declared feature is unstable, record it. if let Some(f) = UNSTABLE_FEATURES.iter().find(|f| name == f.feature.name) { (f.set_enabled)(&mut features); - // When the ICE comes from core, alloc or std (approximation of the standard library), there's a chance - // that the person hitting the ICE may be using -Zbuild-std or similar with an untested target. - // The bug is probably in the standard library and not the compiler in that case, but that doesn't - // really matter - we want a bug report. + // When the ICE comes from core, alloc or std (approximation of the standard + // library), there's a chance that the person hitting the ICE may be using + // -Zbuild-std or similar with an untested target. The bug is probably in the + // standard library and not the compiler in that case, but that doesn't really + // matter - we want a bug report. if features.internal(name) && ![sym::core, sym::alloc, sym::std].contains(&crate_name) { diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 64a17eae6741c..35722c949093e 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -505,8 +505,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> { let (fragment, collected_invocations) = self.collect_invocations(fragment, &derive_placeholders); - // We choose to expand any derive invocations associated with this macro invocation - // *before* any macro invocations collected from the output fragment + // We choose to expand any derive invocations associated with this macro + // invocation *before* any macro invocations collected from the output + // fragment. derive_invocations.extend(collected_invocations); (fragment, derive_invocations) } diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs index 5d6c3e8aa3a4a..464361cb4020b 100644 --- a/compiler/rustc_expand/src/mbe/diagnostics.rs +++ b/compiler/rustc_expand/src/mbe/diagnostics.rs @@ -28,7 +28,8 @@ pub(super) fn failed_to_match_macro<'cx>( ) -> Box { let psess = &cx.sess.psess; - // An error occurred, try the expansion again, tracking the expansion closely for better diagnostics. + // An error occurred, try the expansion again, tracking the expansion closely for better + // diagnostics. let mut tracker = CollectTrackerAndEmitter::new(cx, sp); let try_success_result = try_match_macro(psess, name, &arg, lhses, &mut tracker); diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index df2643ad1ddea..470bde232d723 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -157,8 +157,8 @@ pub(super) trait Tracker<'matcher> { /// This is called before trying to match next MatcherLoc on the current token. fn before_match_loc(&mut self, _parser: &TtParser, _matcher: &'matcher MatcherLoc) {} - /// This is called after an arm has been parsed, either successfully or unsuccessfully. When this is called, - /// `before_match_loc` was called at least once (with a `MatcherLoc::Eof`). + /// This is called after an arm has been parsed, either successfully or unsuccessfully. When + /// this is called, `before_match_loc` was called at least once (with a `MatcherLoc::Eof`). fn after_arm(&mut self, _result: &NamedParseResult) {} /// For tracing. @@ -169,7 +169,8 @@ pub(super) trait Tracker<'matcher> { } } -/// A noop tracker that is used in the hot path of the expansion, has zero overhead thanks to monomorphization. +/// A noop tracker that is used in the hot path of the expansion, has zero overhead thanks to +/// monomorphization. pub(super) struct NoopTracker; impl<'matcher> Tracker<'matcher> for NoopTracker { @@ -492,7 +493,7 @@ pub fn compile_declarative_macro( .pop() .unwrap(); // We don't handle errors here, the driver will abort - // after parsing/expansion. we can report every error in every macro this way. + // after parsing/expansion. We can report every error in every macro this way. check_emission(check_lhs_nt_follows(sess, def, &tt)); return tt; } @@ -528,7 +529,7 @@ pub fn compile_declarative_macro( check_emission(check_rhs(sess, rhs)); } - // don't abort iteration early, so that errors for multiple lhses can be reported + // Don't abort iteration early, so that errors for multiple lhses can be reported. for lhs in &lhses { check_emission(check_lhs_no_empty_seq(sess, slice::from_ref(lhs))); } From 79c4d0202f0c5c44520a424cae34af5e2d30a2cc Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 1 May 2024 09:39:07 +1000 Subject: [PATCH 09/14] Remove unnecessary `pub`s. --- compiler/rustc_expand/src/expand.rs | 10 +++++----- compiler/rustc_expand/src/mbe.rs | 11 ++++++----- compiler/rustc_expand/src/placeholders.rs | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 35722c949093e..84fc92a1df5cf 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -87,7 +87,7 @@ macro_rules! ast_fragments { } impl AstFragment { - pub fn add_placeholders(&mut self, placeholders: &[NodeId]) { + fn add_placeholders(&mut self, placeholders: &[NodeId]) { if placeholders.is_empty() { return; } @@ -100,14 +100,14 @@ macro_rules! ast_fragments { } } - pub fn make_opt_expr(self) -> Option> { + pub(crate) fn make_opt_expr(self) -> Option> { match self { AstFragment::OptExpr(expr) => expr, _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), } } - pub fn make_method_receiver_expr(self) -> P { + pub(crate) fn make_method_receiver_expr(self) -> P { match self { AstFragment::MethodReceiverExpr(expr) => expr, _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), @@ -125,7 +125,7 @@ macro_rules! ast_fragments { T::fragment_to_output(self) } - pub fn mut_visit_with(&mut self, vis: &mut F) { + pub(crate) fn mut_visit_with(&mut self, vis: &mut F) { match self { AstFragment::OptExpr(opt_expr) => { visit_clobber(opt_expr, |opt_expr| { @@ -958,7 +958,7 @@ pub fn parse_ast_fragment<'a>( }) } -pub fn ensure_complete_parse<'a>( +pub(crate) fn ensure_complete_parse<'a>( parser: &Parser<'a>, macro_path: &ast::Path, kind_name: &str, diff --git a/compiler/rustc_expand/src/mbe.rs b/compiler/rustc_expand/src/mbe.rs index bd6a9b7cb07da..a805c4fcf7b97 100644 --- a/compiler/rustc_expand/src/mbe.rs +++ b/compiler/rustc_expand/src/mbe.rs @@ -4,12 +4,13 @@ //! official terminology: "declarative macros". pub(crate) mod diagnostics; -pub(crate) mod macro_check; -pub(crate) mod macro_parser; pub(crate) mod macro_rules; -pub(crate) mod metavar_expr; -pub(crate) mod quoted; -pub(crate) mod transcribe; + +mod macro_check; +mod macro_parser; +mod metavar_expr; +mod quoted; +mod transcribe; use metavar_expr::MetaVarExpr; use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind}; diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs index 581d71875bd07..7026425e167c8 100644 --- a/compiler/rustc_expand/src/placeholders.rs +++ b/compiler/rustc_expand/src/placeholders.rs @@ -9,7 +9,7 @@ use rustc_span::DUMMY_SP; use smallvec::{smallvec, SmallVec}; use thin_vec::ThinVec; -pub fn placeholder( +pub(crate) fn placeholder( kind: AstFragmentKind, id: ast::NodeId, vis: Option, From 3b6978196da24dd249230532153adc0e662678ac Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 1 May 2024 10:25:55 +1000 Subject: [PATCH 10/14] Tweak `fully_expand_fragment` loop. Control flow never gets past the end of the `ExpandResult::Retry` match arm, due to the `span_bug` and the `continue`. Therefore, the code after the match can only be reached from the `ExpandResult::Ready` arm. This commit moves that code after the match into the `ExpandResult::Ready` arm, avoiding the need for the `continue` in the `ExpandResult::Retry` arm. --- compiler/rustc_expand/src/expand.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 84fc92a1df5cf..64f21b747a16b 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -471,7 +471,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { self.cx.force_mode = force; let fragment_kind = invoc.fragment_kind; - let (expanded_fragment, new_invocations) = match self.expand_invoc(invoc, &ext.kind) { + match self.expand_invoc(invoc, &ext.kind) { ExpandResult::Ready(fragment) => { let mut derive_invocations = Vec::new(); let derive_placeholders = self @@ -503,13 +503,19 @@ impl<'a, 'b> MacroExpander<'a, 'b> { }) .unwrap_or_default(); - let (fragment, collected_invocations) = + let (expanded_fragment, collected_invocations) = self.collect_invocations(fragment, &derive_placeholders); // We choose to expand any derive invocations associated with this macro // invocation *before* any macro invocations collected from the output // fragment. derive_invocations.extend(collected_invocations); - (fragment, derive_invocations) + + progress = true; + if expanded_fragments.len() < depth { + expanded_fragments.push(Vec::new()); + } + expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); + invocations.extend(derive_invocations.into_iter().rev()); } ExpandResult::Retry(invoc) => { if force { @@ -520,17 +526,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } else { // Cannot expand, will retry this invocation later. undetermined_invocations.push((invoc, Some(ext))); - continue; } } - }; - - progress = true; - if expanded_fragments.len() < depth { - expanded_fragments.push(Vec::new()); } - expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); - invocations.extend(new_invocations.into_iter().rev()); } self.cx.current_expansion = orig_expansion_data; From 1c15b6ae9c8745cb775b80a48d655c2234135f71 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 1 May 2024 10:50:05 +1000 Subject: [PATCH 11/14] Replace a hard-to-read line. Too clever by half, IMO. --- compiler/rustc_expand/src/expand.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 64f21b747a16b..2b93732e28be3 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -432,7 +432,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> { break; } invocations = mem::take(&mut undetermined_invocations); - force = !mem::replace(&mut progress, false); + force = !progress; + progress = false; if force && self.monotonic { self.cx.dcx().span_delayed_bug( invocations.last().unwrap().0.span(), From ae7e32880af6b4f2de26ee8842e9995896de21a0 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 1 May 2024 10:57:08 +1000 Subject: [PATCH 12/14] Introduce `Invocation::span_mut`. Alongside the existing `Invocation::span`. --- compiler/rustc_expand/src/expand.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 2b93732e28be3..f61cebc02562a 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -372,6 +372,14 @@ impl Invocation { InvocationKind::Derive { path, .. } => path.span, } } + + fn span_mut(&mut self) -> &mut Span { + match &mut self.kind { + InvocationKind::Bang { span, .. } => span, + InvocationKind::Attr { attr, .. } => &mut attr.span, + InvocationKind::Derive { path, .. } => &mut path.span, + } + } } pub struct MacroExpander<'a, 'b> { @@ -590,11 +598,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { for (invoc, _) in invocations.iter_mut() { let expn_id = invoc.expansion_data.id; let parent_def = self.cx.resolver.invocation_parent(expn_id); - let span = match &mut invoc.kind { - InvocationKind::Bang { span, .. } => span, - InvocationKind::Attr { attr, .. } => &mut attr.span, - InvocationKind::Derive { path, .. } => &mut path.span, - }; + let span = invoc.span_mut(); *span = span.with_parent(Some(parent_def)); } } From 5ac017e772a28b86170eab7a056feb9e42d557c2 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 2 May 2024 10:17:10 +1000 Subject: [PATCH 13/14] Type annotate `repeats`. Because the type is not obvious, and this clarifies things. --- compiler/rustc_expand/src/mbe/transcribe.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index dad83984c8b15..48fdde07f42eb 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -117,7 +117,7 @@ pub(super) fn transcribe<'a>( // As we descend in the RHS, we will need to be able to match nested sequences of matchers. // `repeats` keeps track of where we are in matching at each level, with the last element being // the most deeply nested sequence. This is used as a stack. - let mut repeats = Vec::new(); + let mut repeats: Vec<(usize, usize)> = Vec::new(); // `result` contains resulting token stream from the TokenTree we just finished processing. At // the end, this will contain the full result of transcription, but at arbitrary points during From 3a3a15d753649c1701953ecaee6e814497903d73 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 2 May 2024 10:35:23 +1000 Subject: [PATCH 14/14] Refactor `Frame`. It is currently an enum and the `tts` and `idx` fields are repeated across the two variants. This commit splits it into a struct `Frame` and an enum `FrameKind`, to factor out the duplication. The commit also renames `Frame::new` as `Frame::new_delimited` and adds `Frame::new_sequence`. I.e. both variants now have a constructor. --- compiler/rustc_expand/src/mbe/transcribe.rs | 98 +++++++++++---------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 48fdde07f42eb..011aa95c8a116 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -39,26 +39,32 @@ impl MutVisitor for Marker { } /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). -enum Frame<'a> { - Delimited { - tts: &'a [mbe::TokenTree], - idx: usize, - delim: Delimiter, - span: DelimSpan, - spacing: DelimSpacing, - }, - Sequence { - tts: &'a [mbe::TokenTree], - idx: usize, - sep: Option, - kleene_op: KleeneOp, - }, +struct Frame<'a> { + tts: &'a [mbe::TokenTree], + idx: usize, + kind: FrameKind, +} + +enum FrameKind { + Delimited { delim: Delimiter, span: DelimSpan, spacing: DelimSpacing }, + Sequence { sep: Option, kleene_op: KleeneOp }, } impl<'a> Frame<'a> { - /// Construct a new frame around the delimited set of tokens. - fn new(src: &'a mbe::Delimited, span: DelimSpan, spacing: DelimSpacing) -> Frame<'a> { - Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span, spacing } + fn new_delimited(src: &'a mbe::Delimited, span: DelimSpan, spacing: DelimSpacing) -> Frame<'a> { + Frame { + tts: &src.tts, + idx: 0, + kind: FrameKind::Delimited { delim: src.delim, span, spacing }, + } + } + + fn new_sequence( + src: &'a mbe::SequenceRepetition, + sep: Option, + kleene_op: KleeneOp, + ) -> Frame<'a> { + Frame { tts: &src.tts, idx: 0, kind: FrameKind::Sequence { sep, kleene_op } } } } @@ -66,13 +72,9 @@ impl<'a> Iterator for Frame<'a> { type Item = &'a mbe::TokenTree; fn next(&mut self) -> Option<&'a mbe::TokenTree> { - match self { - Frame::Delimited { tts, idx, .. } | Frame::Sequence { tts, idx, .. } => { - let res = tts.get(*idx); - *idx += 1; - res - } - } + let res = self.tts.get(self.idx); + self.idx += 1; + res } } @@ -111,8 +113,11 @@ pub(super) fn transcribe<'a>( // We descend into the RHS (`src`), expanding things as we go. This stack contains the things // we have yet to expand/are still expanding. We start the stack off with the whole RHS. The // choice of spacing values doesn't matter. - let mut stack: SmallVec<[Frame<'_>; 1]> = - smallvec![Frame::new(src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))]; + let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new_delimited( + src, + src_span, + DelimSpacing::new(Spacing::Alone, Spacing::Alone) + )]; // As we descend in the RHS, we will need to be able to match nested sequences of matchers. // `repeats` keeps track of where we are in matching at each level, with the last element being @@ -142,11 +147,12 @@ pub(super) fn transcribe<'a>( // Otherwise, if we have just reached the end of a sequence and we can keep repeating, // go back to the beginning of the sequence. - if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { + let frame = stack.last_mut().unwrap(); + if let FrameKind::Sequence { sep, .. } = &frame.kind { let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); *repeat_idx += 1; if repeat_idx < repeat_len { - *idx = 0; + frame.idx = 0; if let Some(sep) = sep { result.push(TokenTree::Token(sep.clone(), Spacing::Alone)); } @@ -157,16 +163,16 @@ pub(super) fn transcribe<'a>( // We are done with the top of the stack. Pop it. Depending on what it was, we do // different things. Note that the outermost item must be the delimited, wrapped RHS // that was passed in originally to `transcribe`. - match stack.pop().unwrap() { + match stack.pop().unwrap().kind { // Done with a sequence. Pop from repeats. - Frame::Sequence { .. } => { + FrameKind::Sequence { .. } => { repeats.pop(); } // We are done processing a Delimited. If this is the top-level delimited, we are // done. Otherwise, we unwind the result_stack to append what we have produced to // any previous results. - Frame::Delimited { delim, span, mut spacing, .. } => { + FrameKind::Delimited { delim, span, mut spacing, .. } => { // Hack to force-insert a space after `]` in certain case. // See discussion of the `hex-literal` crate in #114571. if delim == Delimiter::Bracket { @@ -192,7 +198,7 @@ pub(super) fn transcribe<'a>( // We are descending into a sequence. We first make sure that the matchers in the RHS // and the matches in `interp` have the same shape. Otherwise, either the caller or the // macro writer has made a mistake. - seq @ mbe::TokenTree::Sequence(_, delimited) => { + seq @ mbe::TokenTree::Sequence(_, seq_rep) => { match lockstep_iter_size(seq, interp, &repeats) { LockstepIterSize::Unconstrained => { return Err(cx @@ -233,12 +239,11 @@ pub(super) fn transcribe<'a>( // The first time we encounter the sequence we push it to the stack. It // then gets reused (see the beginning of the loop) until we are done // repeating. - stack.push(Frame::Sequence { - idx: 0, - sep: seq.separator.clone(), - tts: &delimited.tts, - kleene_op: seq.kleene.op, - }); + stack.push(Frame::new_sequence( + seq_rep, + seq.separator.clone(), + seq.kleene.op, + )); } } } @@ -294,13 +299,7 @@ pub(super) fn transcribe<'a>( // the previous results (from outside the Delimited). mbe::TokenTree::Delimited(mut span, spacing, delimited) => { mut_visit::visit_delim_span(&mut span, &mut marker); - stack.push(Frame::Delimited { - tts: &delimited.tts, - delim: delimited.delim, - idx: 0, - span, - spacing: *spacing, - }); + stack.push(Frame::new_delimited(delimited, span, *spacing)); result_stack.push(mem::take(&mut result)); } @@ -358,10 +357,13 @@ fn maybe_use_metavar_location( ) -> TokenTree { let undelimited_seq = matches!( stack.last(), - Some(Frame::Sequence { + Some(Frame { tts: [_], - sep: None, - kleene_op: KleeneOp::ZeroOrMore | KleeneOp::OneOrMore, + kind: FrameKind::Sequence { + sep: None, + kleene_op: KleeneOp::ZeroOrMore | KleeneOp::OneOrMore, + .. + }, .. }) );