From f2bbb97e1c67c21e7fc351a473d1aac698c4f840 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 27 Nov 2025 13:50:52 +0100 Subject: [PATCH] proc-macro-srv: Fix `::fmt` impl producing trailing whitespace --- .../src/legacy_protocol/msg/flat.rs | 21 ++- crates/proc-macro-srv-cli/src/main_loop.rs | 19 +- .../proc-macro-test/imp/src/lib.rs | 5 + crates/proc-macro-srv/src/lib.rs | 29 +++ crates/proc-macro-srv/src/tests/mod.rs | 176 +++++++++++++++++- crates/proc-macro-srv/src/token_stream.rs | 31 ++- 6 files changed, 251 insertions(+), 30 deletions(-) diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs index 7f19506048de..d22e3f1899b2 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs @@ -303,6 +303,7 @@ impl FlatTree { pub fn to_tokenstream_unresolved>( self, version: u32, + span_join: impl Fn(T::Span, T::Span) -> T::Span, ) -> proc_macro_srv::TokenStream { Reader:: { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { @@ -326,13 +327,14 @@ impl FlatTree { span_data_table: &(), version, } - .read_tokenstream() + .read_tokenstream(span_join) } pub fn to_tokenstream_resolved( self, version: u32, span_data_table: &SpanDataIndexMap, + span_join: impl Fn(Span, Span) -> Span, ) -> proc_macro_srv::TokenStream { Reader:: { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { @@ -356,7 +358,7 @@ impl FlatTree { span_data_table, version, } - .read_tokenstream() + .read_tokenstream(span_join) } } @@ -842,7 +844,10 @@ impl Reader<'_, T> { #[cfg(feature = "sysroot-abi")] impl Reader<'_, T> { - pub(crate) fn read_tokenstream(self) -> proc_macro_srv::TokenStream { + pub(crate) fn read_tokenstream( + self, + span_join: impl Fn(T::Span, T::Span) -> T::Span, + ) -> proc_macro_srv::TokenStream { let mut res: Vec>> = vec![None; self.subtree.len()]; let read_span = |id| T::span_for_token_id(self.span_data_table, id); for i in (0..self.subtree.len()).rev() { @@ -935,6 +940,8 @@ impl Reader<'_, T> { } }) .collect::>(); + let open = read_span(repr.open); + let close = read_span(repr.close); let g = proc_macro_srv::Group { delimiter: match repr.kind { tt::DelimiterKind::Parenthesis => proc_macro_srv::Delimiter::Parenthesis, @@ -944,10 +951,10 @@ impl Reader<'_, T> { }, stream: if stream.is_empty() { None } else { Some(TokenStream::new(stream)) }, span: proc_macro_srv::DelimSpan { - open: read_span(repr.open), - close: read_span(repr.close), - // FIXME - entire: read_span(repr.close), + open, + close, + // FIXME: The protocol does not yet encode entire spans ... + entire: span_join(open, close), }, }; res[i] = Some(g); diff --git a/crates/proc-macro-srv-cli/src/main_loop.rs b/crates/proc-macro-srv-cli/src/main_loop.rs index e4456d2e971e..df54f38cbccb 100644 --- a/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/crates/proc-macro-srv-cli/src/main_loop.rs @@ -90,10 +90,10 @@ fn run_() -> io::Result<()> { let call_site = SpanId(call_site as u32); let mixed_site = SpanId(mixed_site as u32); - let macro_body = - macro_body.to_tokenstream_unresolved::(CURRENT_API_VERSION); + let macro_body = macro_body + .to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b); let attributes = attributes.map(|it| { - it.to_tokenstream_unresolved::(CURRENT_API_VERSION) + it.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b) }); srv.expand( @@ -124,10 +124,17 @@ fn run_() -> io::Result<()> { let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = macro_body - .to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table); + let macro_body = macro_body.to_tokenstream_resolved( + CURRENT_API_VERSION, + &span_data_table, + |a, b| srv.join_spans(a, b).unwrap_or(b), + ); let attributes = attributes.map(|it| { - it.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table) + it.to_tokenstream_resolved( + CURRENT_API_VERSION, + &span_data_table, + |a, b| srv.join_spans(a, b).unwrap_or(b), + ) }); srv.expand( lib, diff --git a/crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs b/crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs index 2a72e50f911e..b4fac26d6e72 100644 --- a/crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs +++ b/crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs @@ -94,6 +94,11 @@ pub fn attr_error(args: TokenStream, item: TokenStream) -> TokenStream { format!("compile_error!(\"#[attr_error({})] {}\");", args, item).parse().unwrap() } +#[proc_macro_derive(DeriveReemit, attributes(helper))] +pub fn derive_reemit(item: TokenStream) -> TokenStream { + item +} + #[proc_macro_derive(DeriveEmpty)] pub fn derive_empty(_item: TokenStream) -> TokenStream { TokenStream::default() diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index aff4dc50378c..a96cf2bdb22d 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -81,6 +81,35 @@ impl<'env> ProcMacroSrv<'env> { temp_dir: TempDir::with_prefix("proc-macro-srv").unwrap(), } } + + pub fn join_spans(&self, first: Span, second: Span) -> Option { + // We can't modify the span range for fixup spans, those are meaningful to fixup, so just + // prefer the non-fixup span. + if first.anchor.ast_id == span::FIXUP_ERASED_FILE_AST_ID_MARKER { + return Some(second); + } + if second.anchor.ast_id == span::FIXUP_ERASED_FILE_AST_ID_MARKER { + return Some(first); + } + // FIXME: Once we can talk back to the client, implement a "long join" request for anchors + // that differ in [AstId]s as joining those spans requires resolving the AstIds. + if first.anchor != second.anchor { + return None; + } + // Differing context, we can't merge these so prefer the one that's root + if first.ctx != second.ctx { + if first.ctx.is_root() { + return Some(second); + } else if second.ctx.is_root() { + return Some(first); + } + } + Some(Span { + range: first.range.cover(second.range), + anchor: second.anchor, + ctx: second.ctx, + }) + } } const EXPANDER_STACK_SIZE: usize = 8 * 1024 * 1024; diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 1e2e8da60cde..4f4fbeae1c60 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -52,6 +52,165 @@ fn test_derive_empty() { ); } +#[test] +fn test_derive_reemit_helpers() { + assert_expand( + "DeriveReemit", + r#" +#[helper(build_fn(private, name = "partial_build"))] +pub struct Foo { + /// The domain where this federated instance is running + #[helper(setter(into))] + pub(crate) domain: String, +} +"#, + expect![[r#" + PUNCT 1 # [joint] + GROUP [] 1 1 1 + IDENT 1 helper + GROUP () 1 1 1 + IDENT 1 build_fn + GROUP () 1 1 1 + IDENT 1 private + PUNCT 1 , [alone] + IDENT 1 name + PUNCT 1 = [alone] + LITER 1 Str partial_build + IDENT 1 pub + IDENT 1 struct + IDENT 1 Foo + GROUP {} 1 1 1 + PUNCT 1 # [alone] + GROUP [] 1 1 1 + IDENT 1 doc + PUNCT 1 = [alone] + LITER 1 Str / The domain where this federated instance is running + PUNCT 1 # [joint] + GROUP [] 1 1 1 + IDENT 1 helper + GROUP () 1 1 1 + IDENT 1 setter + GROUP () 1 1 1 + IDENT 1 into + IDENT 1 pub + GROUP () 1 1 1 + IDENT 1 crate + IDENT 1 domain + PUNCT 1 : [alone] + IDENT 1 String + PUNCT 1 , [alone] + + + PUNCT 1 # [joint] + GROUP [] 1 1 1 + IDENT 1 helper + GROUP () 1 1 1 + IDENT 1 build_fn + GROUP () 1 1 1 + IDENT 1 private + PUNCT 1 , [alone] + IDENT 1 name + PUNCT 1 = [alone] + LITER 1 Str partial_build + IDENT 1 pub + IDENT 1 struct + IDENT 1 Foo + GROUP {} 1 1 1 + PUNCT 1 # [alone] + GROUP [] 1 1 1 + IDENT 1 doc + PUNCT 1 = [alone] + LITER 1 Str / The domain where this federated instance is running + PUNCT 1 # [joint] + GROUP [] 1 1 1 + IDENT 1 helper + GROUP () 1 1 1 + IDENT 1 setter + GROUP () 1 1 1 + IDENT 1 into + IDENT 1 pub + GROUP () 1 1 1 + IDENT 1 crate + IDENT 1 domain + PUNCT 1 : [alone] + IDENT 1 String + PUNCT 1 , [alone] + "#]], + expect![[r#" + PUNCT 42:Root[0000, 0]@1..2#ROOT2024 # [joint] + GROUP [] 42:Root[0000, 0]@2..3#ROOT2024 42:Root[0000, 0]@52..53#ROOT2024 42:Root[0000, 0]@2..53#ROOT2024 + IDENT 42:Root[0000, 0]@3..9#ROOT2024 helper + GROUP () 42:Root[0000, 0]@9..10#ROOT2024 42:Root[0000, 0]@51..52#ROOT2024 42:Root[0000, 0]@9..52#ROOT2024 + IDENT 42:Root[0000, 0]@10..18#ROOT2024 build_fn + GROUP () 42:Root[0000, 0]@18..19#ROOT2024 42:Root[0000, 0]@50..51#ROOT2024 42:Root[0000, 0]@18..51#ROOT2024 + IDENT 42:Root[0000, 0]@19..26#ROOT2024 private + PUNCT 42:Root[0000, 0]@26..27#ROOT2024 , [alone] + IDENT 42:Root[0000, 0]@28..32#ROOT2024 name + PUNCT 42:Root[0000, 0]@33..34#ROOT2024 = [alone] + LITER 42:Root[0000, 0]@35..50#ROOT2024 Str partial_build + IDENT 42:Root[0000, 0]@54..57#ROOT2024 pub + IDENT 42:Root[0000, 0]@58..64#ROOT2024 struct + IDENT 42:Root[0000, 0]@65..68#ROOT2024 Foo + GROUP {} 42:Root[0000, 0]@69..70#ROOT2024 42:Root[0000, 0]@190..191#ROOT2024 42:Root[0000, 0]@69..191#ROOT2024 + PUNCT 42:Root[0000, 0]@0..0#ROOT2024 # [alone] + GROUP [] 42:Root[0000, 0]@0..0#ROOT2024 42:Root[0000, 0]@0..0#ROOT2024 42:Root[0000, 0]@0..0#ROOT2024 + IDENT 42:Root[0000, 0]@0..0#ROOT2024 doc + PUNCT 42:Root[0000, 0]@0..0#ROOT2024 = [alone] + LITER 42:Root[0000, 0]@75..130#ROOT2024 Str / The domain where this federated instance is running + PUNCT 42:Root[0000, 0]@135..136#ROOT2024 # [joint] + GROUP [] 42:Root[0000, 0]@136..137#ROOT2024 42:Root[0000, 0]@157..158#ROOT2024 42:Root[0000, 0]@136..158#ROOT2024 + IDENT 42:Root[0000, 0]@137..143#ROOT2024 helper + GROUP () 42:Root[0000, 0]@143..144#ROOT2024 42:Root[0000, 0]@156..157#ROOT2024 42:Root[0000, 0]@143..157#ROOT2024 + IDENT 42:Root[0000, 0]@144..150#ROOT2024 setter + GROUP () 42:Root[0000, 0]@150..151#ROOT2024 42:Root[0000, 0]@155..156#ROOT2024 42:Root[0000, 0]@150..156#ROOT2024 + IDENT 42:Root[0000, 0]@151..155#ROOT2024 into + IDENT 42:Root[0000, 0]@163..166#ROOT2024 pub + GROUP () 42:Root[0000, 0]@166..167#ROOT2024 42:Root[0000, 0]@172..173#ROOT2024 42:Root[0000, 0]@166..173#ROOT2024 + IDENT 42:Root[0000, 0]@167..172#ROOT2024 crate + IDENT 42:Root[0000, 0]@174..180#ROOT2024 domain + PUNCT 42:Root[0000, 0]@180..181#ROOT2024 : [alone] + IDENT 42:Root[0000, 0]@182..188#ROOT2024 String + PUNCT 42:Root[0000, 0]@188..189#ROOT2024 , [alone] + + + PUNCT 42:Root[0000, 0]@1..2#ROOT2024 # [joint] + GROUP [] 42:Root[0000, 0]@2..3#ROOT2024 42:Root[0000, 0]@52..53#ROOT2024 42:Root[0000, 0]@2..53#ROOT2024 + IDENT 42:Root[0000, 0]@3..9#ROOT2024 helper + GROUP () 42:Root[0000, 0]@9..10#ROOT2024 42:Root[0000, 0]@51..52#ROOT2024 42:Root[0000, 0]@9..52#ROOT2024 + IDENT 42:Root[0000, 0]@10..18#ROOT2024 build_fn + GROUP () 42:Root[0000, 0]@18..19#ROOT2024 42:Root[0000, 0]@50..51#ROOT2024 42:Root[0000, 0]@18..51#ROOT2024 + IDENT 42:Root[0000, 0]@19..26#ROOT2024 private + PUNCT 42:Root[0000, 0]@26..27#ROOT2024 , [alone] + IDENT 42:Root[0000, 0]@28..32#ROOT2024 name + PUNCT 42:Root[0000, 0]@33..34#ROOT2024 = [alone] + LITER 42:Root[0000, 0]@35..50#ROOT2024 Str partial_build + IDENT 42:Root[0000, 0]@54..57#ROOT2024 pub + IDENT 42:Root[0000, 0]@58..64#ROOT2024 struct + IDENT 42:Root[0000, 0]@65..68#ROOT2024 Foo + GROUP {} 42:Root[0000, 0]@69..70#ROOT2024 42:Root[0000, 0]@190..191#ROOT2024 42:Root[0000, 0]@69..191#ROOT2024 + PUNCT 42:Root[0000, 0]@0..0#ROOT2024 # [alone] + GROUP [] 42:Root[0000, 0]@0..0#ROOT2024 42:Root[0000, 0]@0..0#ROOT2024 42:Root[0000, 0]@0..0#ROOT2024 + IDENT 42:Root[0000, 0]@0..0#ROOT2024 doc + PUNCT 42:Root[0000, 0]@0..0#ROOT2024 = [alone] + LITER 42:Root[0000, 0]@75..130#ROOT2024 Str / The domain where this federated instance is running + PUNCT 42:Root[0000, 0]@135..136#ROOT2024 # [joint] + GROUP [] 42:Root[0000, 0]@136..137#ROOT2024 42:Root[0000, 0]@157..158#ROOT2024 42:Root[0000, 0]@136..158#ROOT2024 + IDENT 42:Root[0000, 0]@137..143#ROOT2024 helper + GROUP () 42:Root[0000, 0]@143..144#ROOT2024 42:Root[0000, 0]@156..157#ROOT2024 42:Root[0000, 0]@143..157#ROOT2024 + IDENT 42:Root[0000, 0]@144..150#ROOT2024 setter + GROUP () 42:Root[0000, 0]@150..151#ROOT2024 42:Root[0000, 0]@155..156#ROOT2024 42:Root[0000, 0]@150..156#ROOT2024 + IDENT 42:Root[0000, 0]@151..155#ROOT2024 into + IDENT 42:Root[0000, 0]@163..166#ROOT2024 pub + GROUP () 42:Root[0000, 0]@166..167#ROOT2024 42:Root[0000, 0]@172..173#ROOT2024 42:Root[0000, 0]@166..173#ROOT2024 + IDENT 42:Root[0000, 0]@167..172#ROOT2024 crate + IDENT 42:Root[0000, 0]@174..180#ROOT2024 domain + PUNCT 42:Root[0000, 0]@180..181#ROOT2024 : [alone] + IDENT 42:Root[0000, 0]@182..188#ROOT2024 String + PUNCT 42:Root[0000, 0]@188..189#ROOT2024 , [alone] + "#]], + ); +} + #[test] fn test_derive_error() { assert_expand( @@ -69,7 +228,7 @@ fn test_derive_error() { IDENT 1 compile_error PUNCT 1 ! [joint] GROUP () 1 1 1 - LITER 1 Str #[derive(DeriveError)] struct S {field 58 u32 } + LITER 1 Str #[derive(DeriveError)] struct S {field 58 u32} PUNCT 1 ; [alone] "#]], expect![[r#" @@ -83,9 +242,9 @@ fn test_derive_error() { IDENT 42:Root[0000, 0]@0..13#ROOT2024 compile_error PUNCT 42:Root[0000, 0]@13..14#ROOT2024 ! [joint] - GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@64..65#ROOT2024 42:Root[0000, 0]@14..65#ROOT2024 - LITER 42:Root[0000, 0]@15..64#ROOT2024 Str #[derive(DeriveError)] struct S {field 58 u32 } - PUNCT 42:Root[0000, 0]@65..66#ROOT2024 ; [alone] + GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@63..64#ROOT2024 42:Root[0000, 0]@14..64#ROOT2024 + LITER 42:Root[0000, 0]@15..63#ROOT2024 Str #[derive(DeriveError)] struct S {field 58 u32} + PUNCT 42:Root[0000, 0]@64..65#ROOT2024 ; [alone] "#]], ); } @@ -472,7 +631,7 @@ fn test_attr_macro() { IDENT 1 compile_error PUNCT 1 ! [joint] GROUP () 1 1 1 - LITER 1 Str #[attr_error(some arguments )] mod m {} + LITER 1 Str #[attr_error(some arguments)] mod m {} PUNCT 1 ; [alone] "#]], expect![[r#" @@ -487,9 +646,9 @@ fn test_attr_macro() { IDENT 42:Root[0000, 0]@0..13#ROOT2024 compile_error PUNCT 42:Root[0000, 0]@13..14#ROOT2024 ! [joint] - GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@56..57#ROOT2024 42:Root[0000, 0]@14..57#ROOT2024 - LITER 42:Root[0000, 0]@15..56#ROOT2024 Str #[attr_error(some arguments )] mod m {} - PUNCT 42:Root[0000, 0]@57..58#ROOT2024 ; [alone] + GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@55..56#ROOT2024 42:Root[0000, 0]@14..56#ROOT2024 + LITER 42:Root[0000, 0]@15..55#ROOT2024 Str #[attr_error(some arguments)] mod m {} + PUNCT 42:Root[0000, 0]@56..57#ROOT2024 ; [alone] "#]], ); } @@ -535,6 +694,7 @@ fn list_test_macros() { attr_noop [Attr] attr_panic [Attr] attr_error [Attr] + DeriveReemit [CustomDerive] DeriveEmpty [CustomDerive] DerivePanic [CustomDerive] DeriveError [CustomDerive]"#]] diff --git a/crates/proc-macro-srv/src/token_stream.rs b/crates/proc-macro-srv/src/token_stream.rs index 628d6942392c..931ed12c99ff 100644 --- a/crates/proc-macro-srv/src/token_stream.rs +++ b/crates/proc-macro-srv/src/token_stream.rs @@ -1,7 +1,7 @@ //! The proc-macro server token stream implementation. use core::fmt; -use std::sync::Arc; +use std::{mem, sync::Arc}; use intern::Symbol; use proc_macro::Delimiter; @@ -431,14 +431,22 @@ impl TokenStream { impl fmt::Display for TokenStream { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut emit_whitespace = false; for tt in self.0.iter() { - display_token_tree(tt, f)?; + display_token_tree(tt, &mut emit_whitespace, f)?; } Ok(()) } } -fn display_token_tree(tt: &TokenTree, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +fn display_token_tree( + tt: &TokenTree, + emit_whitespace: &mut bool, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + if mem::take(emit_whitespace) { + write!(f, " ")?; + } match tt { TokenTree::Group(Group { delimiter, stream, span: _ }) => { write!( @@ -466,13 +474,15 @@ fn display_token_tree(tt: &TokenTree, f: &mut std::fmt::Formatter<'_>) -> )?; } TokenTree::Punct(Punct { ch, joint, span: _ }) => { - write!(f, "{ch}{}", if *joint { "" } else { " " })? + *emit_whitespace = !*joint; + write!(f, "{ch}")?; } TokenTree::Ident(Ident { sym, is_raw, span: _ }) => { if *is_raw { write!(f, "r#")?; } - write!(f, "{sym} ")?; + write!(f, "{sym}")?; + *emit_whitespace = true; } TokenTree::Literal(lit) => { display_fmt_literal(lit, f)?; @@ -485,9 +495,7 @@ fn display_token_tree(tt: &TokenTree, f: &mut std::fmt::Formatter<'_>) -> | LitKind::CStrRaw(_) => true, _ => false, }; - if !joint { - write!(f, " ")?; - } + *emit_whitespace = !joint; } } Ok(()) @@ -739,7 +747,12 @@ mod tests { #[test] fn roundtrip() { let token_stream = TokenStream::from_str("struct T {\"string\"}", ()).unwrap(); - token_stream.to_string(); assert_eq!(token_stream.to_string(), "struct T {\"string\"}"); } + + #[test] + fn ident_ts_no_trailing_whitespace_to_string() { + let token_stream = TokenStream::from_str("this_is_an_ident", ()).unwrap(); + assert_eq!(token_stream.to_string(), "this_is_an_ident"); + } }