Skip to content

proc-macro-srv: Fix <TokenStream as Display>::fmt impl producing trailing whitespace #21145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 27, 2025
+251 −30
Merged

proc-macro-srv: Fix <TokenStream as Display>::fmt impl producing trailing whitespace #21145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions crates/proc-macro-api/src/legacy_protocol/msg/flat.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ impl FlatTree {
pub fn to_tokenstream_unresolved<T: SpanTransformer<Table = ()>>(
self,
version: u32,
span_join: impl Fn(T::Span, T::Span) -> T::Span,
) -> proc_macro_srv::TokenStream<T::Span> {
Reader::<T> {
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
Expand All @@ -326,13 +327,14 @@ impl FlatTree {
span_data_table: &(),
version,
}
.read_tokenstream()
.read_tokenstream(span_join)
}

pub fn to_tokenstream_resolved(
self,
version: u32,
span_data_table: &SpanDataIndexMap,
span_join: impl Fn(Span, Span) -> Span,
) -> proc_macro_srv::TokenStream<Span> {
Reader::<Span> {
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
Expand All @@ -356,7 +358,7 @@ impl FlatTree {
span_data_table,
version,
}
.read_tokenstream()
.read_tokenstream(span_join)
}
}

Expand Down Expand Up @@ -842,7 +844,10 @@ impl<T: SpanTransformer> Reader<'_, T> {

#[cfg(feature = "sysroot-abi")]
impl<T: SpanTransformer> Reader<'_, T> {
pub(crate) fn read_tokenstream(self) -> proc_macro_srv::TokenStream<T::Span> {
pub(crate) fn read_tokenstream(
self,
span_join: impl Fn(T::Span, T::Span) -> T::Span,
) -> proc_macro_srv::TokenStream<T::Span> {
let mut res: Vec<Option<proc_macro_srv::Group<T::Span>>> = vec![None; self.subtree.len()];
let read_span = |id| T::span_for_token_id(self.span_data_table, id);
for i in (0..self.subtree.len()).rev() {
Expand Down Expand Up @@ -935,6 +940,8 @@ impl<T: SpanTransformer> Reader<'_, T> {
}
})
.collect::<Vec<_>>();
let open = read_span(repr.open);
let close = read_span(repr.close);
let g = proc_macro_srv::Group {
delimiter: match repr.kind {
tt::DelimiterKind::Parenthesis => proc_macro_srv::Delimiter::Parenthesis,
Expand All @@ -944,10 +951,10 @@ impl<T: SpanTransformer> Reader<'_, T> {
},
stream: if stream.is_empty() { None } else { Some(TokenStream::new(stream)) },
span: proc_macro_srv::DelimSpan {
open: read_span(repr.open),
close: read_span(repr.close),
// FIXME
entire: read_span(repr.close),
open,
close,
// FIXME: The protocol does not yet encode entire spans ...
entire: span_join(open, close),
},
};
res[i] = Some(g);
Expand Down
19 changes: 13 additions & 6 deletions crates/proc-macro-srv-cli/src/main_loop.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,10 @@ fn run_<C: Codec>() -> io::Result<()> {
let call_site = SpanId(call_site as u32);
let mixed_site = SpanId(mixed_site as u32);

let macro_body =
macro_body.to_tokenstream_unresolved::<SpanTrans>(CURRENT_API_VERSION);
let macro_body = macro_body
.to_tokenstream_unresolved::<SpanTrans>(CURRENT_API_VERSION, |_, b| b);
let attributes = attributes.map(|it| {
it.to_tokenstream_unresolved::<SpanTrans>(CURRENT_API_VERSION)
it.to_tokenstream_unresolved::<SpanTrans>(CURRENT_API_VERSION, |_, b| b)
});

srv.expand(
Expand Down Expand Up @@ -124,10 +124,17 @@ fn run_<C: Codec>() -> io::Result<()> {
let call_site = span_data_table[call_site];
let mixed_site = span_data_table[mixed_site];

let macro_body = macro_body
.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table);
let macro_body = macro_body.to_tokenstream_resolved(
CURRENT_API_VERSION,
&span_data_table,
|a, b| srv.join_spans(a, b).unwrap_or(b),
);
let attributes = attributes.map(|it| {
it.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table)
it.to_tokenstream_resolved(
CURRENT_API_VERSION,
&span_data_table,
|a, b| srv.join_spans(a, b).unwrap_or(b),
)
});
srv.expand(
lib,
Expand Down
5 changes: 5 additions & 0 deletions crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,11 @@ pub fn attr_error(args: TokenStream, item: TokenStream) -> TokenStream {
format!("compile_error!(\"#[attr_error({})] {}\");", args, item).parse().unwrap()
}

#[proc_macro_derive(DeriveReemit, attributes(helper))]
pub fn derive_reemit(item: TokenStream) -> TokenStream {
item
}

#[proc_macro_derive(DeriveEmpty)]
pub fn derive_empty(_item: TokenStream) -> TokenStream {
TokenStream::default()
Expand Down
29 changes: 29 additions & 0 deletions crates/proc-macro-srv/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,35 @@ impl<'env> ProcMacroSrv<'env> {
temp_dir: TempDir::with_prefix("proc-macro-srv").unwrap(),
}
}

pub fn join_spans(&self, first: Span, second: Span) -> Option<Span> {
// We can't modify the span range for fixup spans, those are meaningful to fixup, so just
// prefer the non-fixup span.
if first.anchor.ast_id == span::FIXUP_ERASED_FILE_AST_ID_MARKER {
return Some(second);
}
if second.anchor.ast_id == span::FIXUP_ERASED_FILE_AST_ID_MARKER {
return Some(first);
}
// FIXME: Once we can talk back to the client, implement a "long join" request for anchors
// that differ in [AstId]s as joining those spans requires resolving the AstIds.
if first.anchor != second.anchor {
return None;
}
// Differing context, we can't merge these so prefer the one that's root
if first.ctx != second.ctx {
if first.ctx.is_root() {
return Some(second);
} else if second.ctx.is_root() {
return Some(first);
}
}
Some(Span {
range: first.range.cover(second.range),
anchor: second.anchor,
ctx: second.ctx,
})
}
}

const EXPANDER_STACK_SIZE: usize = 8 * 1024 * 1024;
Expand Down
176 changes: 168 additions & 8 deletions crates/proc-macro-srv/src/tests/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,165 @@ fn test_derive_empty() {
);
}

#[test]
fn test_derive_reemit_helpers() {
assert_expand(
"DeriveReemit",
r#"
#[helper(build_fn(private, name = "partial_build"))]
pub struct Foo {
/// The domain where this federated instance is running
#[helper(setter(into))]
pub(crate) domain: String,
}
"#,
expect![[r#"
PUNCT 1 # [joint]
GROUP [] 1 1 1
IDENT 1 helper
GROUP () 1 1 1
IDENT 1 build_fn
GROUP () 1 1 1
IDENT 1 private
PUNCT 1 , [alone]
IDENT 1 name
PUNCT 1 = [alone]
LITER 1 Str partial_build
IDENT 1 pub
IDENT 1 struct
IDENT 1 Foo
GROUP {} 1 1 1
PUNCT 1 # [alone]
GROUP [] 1 1 1
IDENT 1 doc
PUNCT 1 = [alone]
LITER 1 Str / The domain where this federated instance is running
PUNCT 1 # [joint]
GROUP [] 1 1 1
IDENT 1 helper
GROUP () 1 1 1
IDENT 1 setter
GROUP () 1 1 1
IDENT 1 into
IDENT 1 pub
GROUP () 1 1 1
IDENT 1 crate
IDENT 1 domain
PUNCT 1 : [alone]
IDENT 1 String
PUNCT 1 , [alone]


PUNCT 1 # [joint]
GROUP [] 1 1 1
IDENT 1 helper
GROUP () 1 1 1
IDENT 1 build_fn
GROUP () 1 1 1
IDENT 1 private
PUNCT 1 , [alone]
IDENT 1 name
PUNCT 1 = [alone]
LITER 1 Str partial_build
IDENT 1 pub
IDENT 1 struct
IDENT 1 Foo
GROUP {} 1 1 1
PUNCT 1 # [alone]
GROUP [] 1 1 1
IDENT 1 doc
PUNCT 1 = [alone]
LITER 1 Str / The domain where this federated instance is running
PUNCT 1 # [joint]
GROUP [] 1 1 1
IDENT 1 helper
GROUP () 1 1 1
IDENT 1 setter
GROUP () 1 1 1
IDENT 1 into
IDENT 1 pub
GROUP () 1 1 1
IDENT 1 crate
IDENT 1 domain
PUNCT 1 : [alone]
IDENT 1 String
PUNCT 1 , [alone]
"#]],
expect![[r#"
PUNCT 42:Root[0000, 0]@1..2#ROOT2024 # [joint]
GROUP [] 42:Root[0000, 0]@2..3#ROOT2024 42:Root[0000, 0]@52..53#ROOT2024 42:Root[0000, 0]@2..53#ROOT2024
IDENT 42:Root[0000, 0]@3..9#ROOT2024 helper
GROUP () 42:Root[0000, 0]@9..10#ROOT2024 42:Root[0000, 0]@51..52#ROOT2024 42:Root[0000, 0]@9..52#ROOT2024
IDENT 42:Root[0000, 0]@10..18#ROOT2024 build_fn
GROUP () 42:Root[0000, 0]@18..19#ROOT2024 42:Root[0000, 0]@50..51#ROOT2024 42:Root[0000, 0]@18..51#ROOT2024
IDENT 42:Root[0000, 0]@19..26#ROOT2024 private
PUNCT 42:Root[0000, 0]@26..27#ROOT2024 , [alone]
IDENT 42:Root[0000, 0]@28..32#ROOT2024 name
PUNCT 42:Root[0000, 0]@33..34#ROOT2024 = [alone]
LITER 42:Root[0000, 0]@35..50#ROOT2024 Str partial_build
IDENT 42:Root[0000, 0]@54..57#ROOT2024 pub
IDENT 42:Root[0000, 0]@58..64#ROOT2024 struct
IDENT 42:Root[0000, 0]@65..68#ROOT2024 Foo
GROUP {} 42:Root[0000, 0]@69..70#ROOT2024 42:Root[0000, 0]@190..191#ROOT2024 42:Root[0000, 0]@69..191#ROOT2024
PUNCT 42:Root[0000, 0]@0..0#ROOT2024 # [alone]
GROUP [] 42:Root[0000, 0]@0..0#ROOT2024 42:Root[0000, 0]@0..0#ROOT2024 42:Root[0000, 0]@0..0#ROOT2024
IDENT 42:Root[0000, 0]@0..0#ROOT2024 doc
PUNCT 42:Root[0000, 0]@0..0#ROOT2024 = [alone]
LITER 42:Root[0000, 0]@75..130#ROOT2024 Str / The domain where this federated instance is running
PUNCT 42:Root[0000, 0]@135..136#ROOT2024 # [joint]
GROUP [] 42:Root[0000, 0]@136..137#ROOT2024 42:Root[0000, 0]@157..158#ROOT2024 42:Root[0000, 0]@136..158#ROOT2024
IDENT 42:Root[0000, 0]@137..143#ROOT2024 helper
GROUP () 42:Root[0000, 0]@143..144#ROOT2024 42:Root[0000, 0]@156..157#ROOT2024 42:Root[0000, 0]@143..157#ROOT2024
IDENT 42:Root[0000, 0]@144..150#ROOT2024 setter
GROUP () 42:Root[0000, 0]@150..151#ROOT2024 42:Root[0000, 0]@155..156#ROOT2024 42:Root[0000, 0]@150..156#ROOT2024
IDENT 42:Root[0000, 0]@151..155#ROOT2024 into
IDENT 42:Root[0000, 0]@163..166#ROOT2024 pub
GROUP () 42:Root[0000, 0]@166..167#ROOT2024 42:Root[0000, 0]@172..173#ROOT2024 42:Root[0000, 0]@166..173#ROOT2024
IDENT 42:Root[0000, 0]@167..172#ROOT2024 crate
IDENT 42:Root[0000, 0]@174..180#ROOT2024 domain
PUNCT 42:Root[0000, 0]@180..181#ROOT2024 : [alone]
IDENT 42:Root[0000, 0]@182..188#ROOT2024 String
PUNCT 42:Root[0000, 0]@188..189#ROOT2024 , [alone]


PUNCT 42:Root[0000, 0]@1..2#ROOT2024 # [joint]
GROUP [] 42:Root[0000, 0]@2..3#ROOT2024 42:Root[0000, 0]@52..53#ROOT2024 42:Root[0000, 0]@2..53#ROOT2024
IDENT 42:Root[0000, 0]@3..9#ROOT2024 helper
GROUP () 42:Root[0000, 0]@9..10#ROOT2024 42:Root[0000, 0]@51..52#ROOT2024 42:Root[0000, 0]@9..52#ROOT2024
IDENT 42:Root[0000, 0]@10..18#ROOT2024 build_fn
GROUP () 42:Root[0000, 0]@18..19#ROOT2024 42:Root[0000, 0]@50..51#ROOT2024 42:Root[0000, 0]@18..51#ROOT2024
IDENT 42:Root[0000, 0]@19..26#ROOT2024 private
PUNCT 42:Root[0000, 0]@26..27#ROOT2024 , [alone]
IDENT 42:Root[0000, 0]@28..32#ROOT2024 name
PUNCT 42:Root[0000, 0]@33..34#ROOT2024 = [alone]
LITER 42:Root[0000, 0]@35..50#ROOT2024 Str partial_build
IDENT 42:Root[0000, 0]@54..57#ROOT2024 pub
IDENT 42:Root[0000, 0]@58..64#ROOT2024 struct
IDENT 42:Root[0000, 0]@65..68#ROOT2024 Foo
GROUP {} 42:Root[0000, 0]@69..70#ROOT2024 42:Root[0000, 0]@190..191#ROOT2024 42:Root[0000, 0]@69..191#ROOT2024
PUNCT 42:Root[0000, 0]@0..0#ROOT2024 # [alone]
GROUP [] 42:Root[0000, 0]@0..0#ROOT2024 42:Root[0000, 0]@0..0#ROOT2024 42:Root[0000, 0]@0..0#ROOT2024
IDENT 42:Root[0000, 0]@0..0#ROOT2024 doc
PUNCT 42:Root[0000, 0]@0..0#ROOT2024 = [alone]
LITER 42:Root[0000, 0]@75..130#ROOT2024 Str / The domain where this federated instance is running
PUNCT 42:Root[0000, 0]@135..136#ROOT2024 # [joint]
GROUP [] 42:Root[0000, 0]@136..137#ROOT2024 42:Root[0000, 0]@157..158#ROOT2024 42:Root[0000, 0]@136..158#ROOT2024
IDENT 42:Root[0000, 0]@137..143#ROOT2024 helper
GROUP () 42:Root[0000, 0]@143..144#ROOT2024 42:Root[0000, 0]@156..157#ROOT2024 42:Root[0000, 0]@143..157#ROOT2024
IDENT 42:Root[0000, 0]@144..150#ROOT2024 setter
GROUP () 42:Root[0000, 0]@150..151#ROOT2024 42:Root[0000, 0]@155..156#ROOT2024 42:Root[0000, 0]@150..156#ROOT2024
IDENT 42:Root[0000, 0]@151..155#ROOT2024 into
IDENT 42:Root[0000, 0]@163..166#ROOT2024 pub
GROUP () 42:Root[0000, 0]@166..167#ROOT2024 42:Root[0000, 0]@172..173#ROOT2024 42:Root[0000, 0]@166..173#ROOT2024
IDENT 42:Root[0000, 0]@167..172#ROOT2024 crate
IDENT 42:Root[0000, 0]@174..180#ROOT2024 domain
PUNCT 42:Root[0000, 0]@180..181#ROOT2024 : [alone]
IDENT 42:Root[0000, 0]@182..188#ROOT2024 String
PUNCT 42:Root[0000, 0]@188..189#ROOT2024 , [alone]
"#]],
);
}

#[test]
fn test_derive_error() {
assert_expand(
Expand All @@ -69,7 +228,7 @@ fn test_derive_error() {
IDENT 1 compile_error
PUNCT 1 ! [joint]
GROUP () 1 1 1
LITER 1 Str #[derive(DeriveError)] struct S {field 58 u32 }
LITER 1 Str #[derive(DeriveError)] struct S {field 58 u32}
PUNCT 1 ; [alone]
"#]],
expect![[r#"
Expand All @@ -83,9 +242,9 @@ fn test_derive_error() {

IDENT 42:Root[0000, 0]@0..13#ROOT2024 compile_error
PUNCT 42:Root[0000, 0]@13..14#ROOT2024 ! [joint]
GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@64..65#ROOT2024 42:Root[0000, 0]@14..65#ROOT2024
LITER 42:Root[0000, 0]@15..64#ROOT2024 Str #[derive(DeriveError)] struct S {field 58 u32 }
PUNCT 42:Root[0000, 0]@65..66#ROOT2024 ; [alone]
GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@63..64#ROOT2024 42:Root[0000, 0]@14..64#ROOT2024
LITER 42:Root[0000, 0]@15..63#ROOT2024 Str #[derive(DeriveError)] struct S {field 58 u32}
PUNCT 42:Root[0000, 0]@64..65#ROOT2024 ; [alone]
"#]],
);
}
Expand Down Expand Up @@ -472,7 +631,7 @@ fn test_attr_macro() {
IDENT 1 compile_error
PUNCT 1 ! [joint]
GROUP () 1 1 1
LITER 1 Str #[attr_error(some arguments )] mod m {}
LITER 1 Str #[attr_error(some arguments)] mod m {}
PUNCT 1 ; [alone]
"#]],
expect![[r#"
Expand All @@ -487,9 +646,9 @@ fn test_attr_macro() {

IDENT 42:Root[0000, 0]@0..13#ROOT2024 compile_error
PUNCT 42:Root[0000, 0]@13..14#ROOT2024 ! [joint]
GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@56..57#ROOT2024 42:Root[0000, 0]@14..57#ROOT2024
LITER 42:Root[0000, 0]@15..56#ROOT2024 Str #[attr_error(some arguments )] mod m {}
PUNCT 42:Root[0000, 0]@57..58#ROOT2024 ; [alone]
GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@55..56#ROOT2024 42:Root[0000, 0]@14..56#ROOT2024
LITER 42:Root[0000, 0]@15..55#ROOT2024 Str #[attr_error(some arguments)] mod m {}
PUNCT 42:Root[0000, 0]@56..57#ROOT2024 ; [alone]
"#]],
);
}
Expand Down Expand Up @@ -535,6 +694,7 @@ fn list_test_macros() {
attr_noop [Attr]
attr_panic [Attr]
attr_error [Attr]
DeriveReemit [CustomDerive]
DeriveEmpty [CustomDerive]
DerivePanic [CustomDerive]
DeriveError [CustomDerive]"#]]
Expand Down
Loading