Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve print_tts #114571

Merged
merged 3 commits into from
Dec 11, 2023
Merged

Improve print_tts #114571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
925f7fa
Improve `print_tts` by changing `tokenstream::Spacing`.
nnethercote Aug 8, 2023
4cfdbd3
Add spacing information to delimiters.
nnethercote Oct 12, 2023
940c885
Add a few cases with wonky formatting to `stringify.rs` test.
nnethercote Nov 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions compiler/rustc_ast/src/attr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ impl MetaItemKind {
tokens: &mut impl Iterator<Item = &'a TokenTree>,
) -> Option<MetaItemKind> {
match tokens.next() {
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
}
Some(TokenTree::Token(token, _)) => {
Expand All @@ -401,7 +401,7 @@ impl MetaItemKind {
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
) -> Option<MetaItemKind> {
match tokens.peek() {
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
let inner_tokens = inner_tokens.clone();
tokens.next();
MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
Expand Down Expand Up @@ -524,7 +524,7 @@ impl NestedMetaItem {
tokens.next();
return Some(NestedMetaItem::Lit(lit));
}
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
tokens.next();
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
}
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_ast/src/mut_visit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -682,7 +682,7 @@ pub fn visit_attr_tt<T: MutVisitor>(tt: &mut AttrTokenTree, vis: &mut T) {
AttrTokenTree::Token(token, _) => {
visit_token(token, vis);
}
AttrTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
AttrTokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
vis.visit_span(open);
vis.visit_span(close);
visit_attr_tts(tts, vis);
Expand All @@ -709,7 +709,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
TokenTree::Token(token, _) => {
visit_token(token, vis);
}
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
TokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
vis.visit_span(open);
vis.visit_span(close);
visit_tts(tts, vis);
Expand Down
137 changes: 99 additions & 38 deletions compiler/rustc_ast/src/tokenstream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ pub enum TokenTree {
/// delimiters are implicitly represented by `Delimited`.
Token(Token, Spacing),
/// A delimited sequence of token trees.
Delimited(DelimSpan, Delimiter, TokenStream),
Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream),
}

// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
Expand All @@ -62,11 +62,11 @@ where
}

impl TokenTree {
/// Checks if this `TokenTree` is equal to the other, regardless of span information.
/// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information.
pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
match (self, other) {
(TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
(TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
delim == delim2 && tts.eq_unspanned(tts2)
}
_ => false,
Expand Down Expand Up @@ -99,6 +99,11 @@ impl TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
}

/// Create a `TokenTree::Token` with joint-hidden spacing.
pub fn token_joint_hidden(kind: TokenKind, span: Span) -> TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::JointHidden)
}

pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
match self {
TokenTree::Token(token, spacing) => match token.uninterpolate() {
Expand Down Expand Up @@ -183,7 +188,7 @@ pub struct AttrTokenStream(pub Lrc<Vec<AttrTokenTree>>);
#[derive(Clone, Debug, Encodable, Decodable)]
pub enum AttrTokenTree {
Token(Token, Spacing),
Delimited(DelimSpan, Delimiter, AttrTokenStream),
Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream),
/// Stores the attributes for an attribute target,
/// along with the tokens for that attribute target.
/// See `AttributesData` for more information
Expand All @@ -208,9 +213,14 @@ impl AttrTokenStream {
AttrTokenTree::Token(inner, spacing) => {
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
}
AttrTokenTree::Delimited(span, delim, stream) => {
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
.into_iter()
AttrTokenTree::Delimited(span, spacing, delim, stream) => {
smallvec![TokenTree::Delimited(
*span,
*spacing,
*delim,
stream.to_tokenstream()
),]
.into_iter()
}
AttrTokenTree::Attributes(data) => {
let idx = data
Expand All @@ -230,7 +240,7 @@ impl AttrTokenStream {
let mut found = false;
// Check the last two trees (to account for a trailing semi)
for tree in target_tokens.iter_mut().rev().take(2) {
if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
// Inner attributes are only supported on extern blocks, functions,
// impls, and modules. All of these have their inner attributes
// placed at the beginning of the rightmost outermost braced group:
Expand All @@ -250,7 +260,7 @@ impl AttrTokenStream {
stream.push_stream(inner_attr.tokens());
}
stream.push_stream(delim_tokens.clone());
*tree = TokenTree::Delimited(*span, *delim, stream);
*tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
found = true;
break;
}
Expand Down Expand Up @@ -303,21 +313,64 @@ pub struct AttributesData {
#[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);

/// Similar to `proc_macro::Spacing`, but for tokens.
///
/// Note that all `ast::TokenTree::Token` instances have a `Spacing`, but when
/// we convert to `proc_macro::TokenTree` for proc macros only `Punct`
/// `TokenTree`s have a `proc_macro::Spacing`.
/// Indicates whether a token can join with the following token to form a
/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
/// guide pretty-printing, which is where the `JointHidden` value (which isn't
/// part of `proc_macro::Spacing`) comes in useful.
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
pub enum Spacing {
/// The token is not immediately followed by an operator token (as
/// determined by `Token::is_op`). E.g. a `+` token is `Alone` in `+ =`,
/// `+/*foo*/=`, `+ident`, and `+()`.
/// The token cannot join with the following token to form a compound
/// token.
///
/// In token streams parsed from source code, the compiler will use `Alone`
/// for any token immediately followed by whitespace, a non-doc comment, or
/// EOF.
///
/// When constructing token streams within the compiler, use this for each
/// token that (a) should be pretty-printed with a space after it, or (b)
/// is the last token in the stream. (In the latter case the choice of
/// spacing doesn't matter because it is never used for the last token. We
/// arbitrarily use `Alone`.)
///
/// Converts to `proc_macro::Spacing::Alone`, and
/// `proc_macro::Spacing::Alone` converts back to this.
Alone,

/// The token is immediately followed by an operator token. E.g. a `+`
/// token is `Joint` in `+=` and `++`.
/// The token can join with the following token to form a compound token.
///
/// In token streams parsed from source code, the compiler will use `Joint`
/// for any token immediately followed by punctuation (as determined by
/// `Token::is_punct`).
///
/// When constructing token streams within the compiler, use this for each
/// token that (a) should be pretty-printed without a space after it, and
/// (b) is followed by a punctuation token.
///
/// Converts to `proc_macro::Spacing::Joint`, and
/// `proc_macro::Spacing::Joint` converts back to this.
Joint,

/// The token can join with the following token to form a compound token,
/// but this will not be visible at the proc macro level. (This is what the
/// `Hidden` means; see below.)
///
/// In token streams parsed from source code, the compiler will use
/// `JointHidden` for any token immediately followed by anything not
/// covered by the `Alone` and `Joint` cases: an identifier, lifetime,
/// literal, delimiter, doc comment.
///
/// When constructing token streams, use this for each token that (a)
/// should be pretty-printed without a space after it, and (b) is followed
/// by a non-punctuation token.
///
/// Converts to `proc_macro::Spacing::Alone`, but
/// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`.
/// Because of that, pretty-printing of `TokenStream`s produced by proc
/// macros is unavoidably uglier (with more whitespace between tokens) than
/// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed
/// source code, internally constructed token streams, and token streams
/// produced by declarative macros).
JointHidden,
}

impl TokenStream {
Expand Down Expand Up @@ -421,21 +474,14 @@ impl TokenStream {
self
}

/// Create a token stream containing a single token with alone spacing.
/// Create a token stream containing a single token with alone spacing. The
/// spacing used for the final token in a constructed stream doesn't matter
/// because it's never used. In practice we arbitrarily use
/// `Spacing::Alone`.
pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
TokenStream::new(vec![TokenTree::token_alone(kind, span)])
}

/// Create a token stream containing a single token with joint spacing.
pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream {
TokenStream::new(vec![TokenTree::token_joint(kind, span)])
}

/// Create a token stream containing a single `Delimited`.
pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
}

pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
let Some(tokens) = node.tokens() else {
panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
Expand Down Expand Up @@ -482,6 +528,7 @@ impl TokenStream {
}
token::Interpolated(nt) => TokenTree::Delimited(
DelimSpan::from_single(token.span),
DelimSpacing::new(Spacing::JointHidden, spacing),
Delimiter::Invisible,
TokenStream::from_nonterminal_ast(&nt.0).flattened(),
),
Expand All @@ -492,8 +539,8 @@ impl TokenStream {
fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
match tree {
TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
TokenTree::Delimited(span, delim, tts) => {
TokenTree::Delimited(*span, *delim, tts.flattened())
TokenTree::Delimited(span, spacing, delim, tts) => {
TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
}
}
}
Expand All @@ -503,7 +550,7 @@ impl TokenStream {
fn can_skip(stream: &TokenStream) -> bool {
stream.trees().all(|tree| match tree {
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
TokenTree::Delimited(_, _, inner) => can_skip(inner),
TokenTree::Delimited(.., inner) => can_skip(inner),
})
}

Expand All @@ -517,7 +564,7 @@ impl TokenStream {
// If `vec` is not empty, try to glue `tt` onto its last token. The return
// value indicates if gluing took place.
fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last()
if let Some(TokenTree::Token(last_tok, Spacing::Joint | Spacing::JointHidden)) = vec.last()
&& let TokenTree::Token(tok, spacing) = tt
&& let Some(glued_tok) = last_tok.glue(tok)
{
Expand Down Expand Up @@ -592,9 +639,10 @@ impl TokenStream {

&TokenTree::Token(..) => i += 1,

&TokenTree::Delimited(sp, delim, ref delim_stream) => {
&TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => {
if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
let new_tt =
TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream);
Lrc::make_mut(&mut stream.0)[i] = new_tt;
modified = true;
}
Expand Down Expand Up @@ -622,10 +670,11 @@ impl TokenStream {
num_of_hashes = cmp::max(num_of_hashes, count);
}

// `/// foo` becomes `doc = r"foo"`.
// `/// foo` becomes `[doc = r"foo"]`.
let delim_span = DelimSpan::from_single(span);
let body = TokenTree::Delimited(
delim_span,
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
Delimiter::Bracket,
[
TokenTree::token_alone(token::Ident(sym::doc, false), span),
Expand All @@ -641,7 +690,7 @@ impl TokenStream {

if attr_style == AttrStyle::Inner {
vec![
TokenTree::token_alone(token::Pound, span),
TokenTree::token_joint(token::Pound, span),
TokenTree::token_alone(token::Not, span),
body,
]
Expand Down Expand Up @@ -738,6 +787,18 @@ impl DelimSpan {
}
}

#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
pub struct DelimSpacing {
pub open: Spacing,
pub close: Spacing,
}

impl DelimSpacing {
pub fn new(open: Spacing, close: Spacing) -> DelimSpacing {
DelimSpacing { open, close }
}
}

// Some types are used a lot. Make sure they don't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
mod size_asserts {
Expand Down
Loading
Loading