Skip to content

Commit

Permalink
[xtk-ui 1 failure] Don't skip invisible delimiters.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nnethercote
nnethercote committed Aug 22, 2023
1 parent 8fc2809 commit 848d2d2
Show file tree
Hide file tree
Showing 13 changed files with 261 additions and 137 deletions.
65 changes: 23 additions & 42 deletions compiler/rustc_ast/src/token.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,26 +62,6 @@ pub enum InvisibleSource {
// Converted from `proc_macro::Delimiter` in
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
ProcMacro,

// Converted from `TokenKind::Interpolated` in
// `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
FlattenToken,
}

impl Delimiter {
// Should the parser skip these delimiters? Only happens for certain kinds
// of invisible delimiters. Once all interpolated nonterminals are removed,
// the answer should become `false` for all kinds, whereupon this function
// can be removed.
pub fn skip(&self) -> bool {
match self {
Delimiter::Invisible(src) => match src {
InvisibleSource::FlattenToken | InvisibleSource::ProcMacro => true,
InvisibleSource::MetaVar(_) => false,
},
Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
}
}
}

// Note that the suffix is *not* considered when deciding the `LitKind` in this
Expand Down Expand Up @@ -136,21 +116,8 @@ impl Lit {
match token.uninterpolate().kind {
Ident(name, false) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
Literal(token_lit) => Some(token_lit),
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Literal))) => {
panic!("njn: FROM_TOKEN (1)");
// if let NtExpr(expr) | NtLiteral(expr) = &**nt
// && let ast::ExprKind::Lit(token_lit) = expr.kind =>
// {
// Some(token_lit)
// }
}
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Expr))) => {
panic!("njn: FROM_TOKEN (2)");
// if let NtExpr(expr) | NtLiteral(expr) = &**nt
// && let ast::ExprKind::Lit(token_lit) = expr.kind =>
// {
// Some(token_lit)
// }
OpenDelim(Delimiter::Invisible(source)) => {
panic!("njn: from_token {source:?}");
}
_ => None,
}
Expand Down Expand Up @@ -415,8 +382,8 @@ impl Token {
match self.kind {
InterpolatedIdent(_, _, uninterpolated_span)
| InterpolatedLifetime(_, uninterpolated_span) => uninterpolated_span,
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(kind))) => {
panic!("njn: uninterpolated_span {kind:?}");
OpenDelim(Delimiter::Invisible(source)) => {
panic!("njn: uninterpolated_span {source:?}");
}
_ => self.span,
}
Expand Down Expand Up @@ -473,8 +440,8 @@ impl Token {
NonterminalKind::Expr |
NonterminalKind::Literal |
NonterminalKind::Path
)))
=> true,
))) |
OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
_ => false,
}
}
Expand All @@ -501,7 +468,8 @@ impl Token {
NonterminalKind::PatWithOr |
NonterminalKind::Path |
NonterminalKind::Literal
))) => true,
))) |
OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
_ => false,
}
}
Expand All @@ -524,7 +492,8 @@ impl Token {
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
NonterminalKind::Ty |
NonterminalKind::Path
))) => true,
))) |
OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
_ => false,
}
}
Expand All @@ -536,6 +505,7 @@ impl Token {
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
NonterminalKind::Block | NonterminalKind::Expr | NonterminalKind::Literal,
))) => true,
OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
_ => self.can_begin_literal_maybe_minus(),
}
}
Expand Down Expand Up @@ -592,7 +562,8 @@ impl Token {
Ident(name, false) if name.is_bool_lit() => true,
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
NonterminalKind::Literal | NonterminalKind::Expr,
))) => true,
)))
| OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => true,
_ => false,
}
}
Expand Down Expand Up @@ -658,6 +629,7 @@ impl Token {
/// Would `maybe_reparse_metavar_expr` in `parser.rs` return `Ok(..)`?
/// That is, is this a pre-parsed expression dropped into the token stream
/// (which happens while parsing the result of macro expansion)?
// njn: proc macro?
pub fn is_metavar_expr(&self) -> bool {
matches!(
self.is_metavar_seq(),
Expand All @@ -672,6 +644,7 @@ impl Token {

/// Are we at a block from a metavar (`$b:block`)?
pub fn is_metavar_block(&self) -> bool {
// njn: handle proc-macro here too?
matches!(self.is_metavar_seq(), Some(NonterminalKind::Block))
}

Expand All @@ -687,6 +660,7 @@ impl Token {
pub fn is_path_start(&self) -> bool {
self == &ModSep
|| self.is_qpath_start()
// njn: proc macro?
|| matches!(self.is_metavar_seq(), Some(NonterminalKind::Path))
|| self.is_path_segment_keyword()
|| self.is_ident() && !self.is_reserved_ident()
Expand Down Expand Up @@ -760,6 +734,13 @@ impl Token {
}
}

/// Is this an invisible open delimiter at the start of a token sequence
/// from a proc macro?
// njn: need to use this more
pub fn is_proc_macro_seq(&self) -> bool {
matches!(self.kind, OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)))
}

pub fn glue(&self, joint: &Token) -> Option<Token> {
let kind = match self.kind {
Eq => match joint.kind {
Expand Down
6 changes: 4 additions & 2 deletions compiler/rustc_ast/src/tokenstream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

use crate::ast::AttrStyle;
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
use crate::token::{self, Delimiter, InvisibleSource, Token, TokenKind};
use crate::token::{self, Delimiter, InvisibleSource, NonterminalKind, Token, TokenKind};
use crate::AttrVec;

use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
Expand Down Expand Up @@ -456,9 +456,10 @@ impl TokenStream {
Token::new(token::Ident(name, is_raw), uninterpolated_span),
spacing,
),
// njn: not actually a metavar, but we know the kind
token::InterpolatedLifetime(name, uninterpolated_span) => TokenTree::Delimited(
DelimSpan::from_single(token.span),
Delimiter::Invisible(InvisibleSource::FlattenToken),
Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Lifetime)),
TokenStream::token_alone(token::Lifetime(name), uninterpolated_span),
),
_ => TokenTree::Token(*token, spacing),
Expand All @@ -474,6 +475,7 @@ impl TokenStream {
}
}

// njn: do we still need this?
#[must_use]
pub fn flattened(&self) -> TokenStream {
fn can_skip(stream: &TokenStream) -> bool {
Expand Down
3 changes: 1 addition & 2 deletions compiler/rustc_expand/src/mbe/macro_rules.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -721,8 +721,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
ident == sym::compile_error &&
let mbe::TokenTree::Token(bang) = bang &&
let TokenKind::Not = bang.kind &&
let mbe::TokenTree::Delimited(_, del) = args &&
!del.delim.skip()
let mbe::TokenTree::Delimited(..) = args
{
true
} else {
Expand Down
14 changes: 3 additions & 11 deletions compiler/rustc_expand/src/mbe/quoted.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,17 +147,9 @@ fn parse_tree<'a>(
match tree {
// `tree` is a `$` token. Look at the next token in `trees`
&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
// during parsing.
let mut next = outer_trees.next();
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
match next {
Some(tokenstream::TokenTree::Delimited(_, delim, tts)) if delim.skip() => {
trees = Box::new(tts.trees());
next = trees.next();
}
_ => trees = Box::new(outer_trees),
}
let next = outer_trees.next();
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>> =
Box::new(outer_trees);

match next {
// `tree` is followed by a delimited set of token trees.
Expand Down
57 changes: 53 additions & 4 deletions compiler/rustc_parse/src/parser/expr.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -709,6 +709,7 @@ impl<'a> Parser<'a> {
}

/// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
// njn: needs adjusting for ProcMacro?
fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
match self.prev_token.kind {
TokenKind::InterpolatedIdent(..) | TokenKind::InterpolatedLifetime(..) => {
Expand Down Expand Up @@ -1363,16 +1364,39 @@ impl<'a> Parser<'a> {
/// correctly if called from `parse_dot_or_call_expr()`.
fn parse_expr_bottom(&mut self) -> PResult<'a, P<Expr>> {
maybe_recover_from_interpolated_ty_qpath!(self, true);
//eprintln!("AAA {:?}", self.token);
//eprintln!("AAA {:?}", self.token.kind);
maybe_reparse_metavar_expr!(self);
//eprintln!("BBB {:?}", self.token);
//eprintln!("BBB {:?}", self.token.kind);

if let token::OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) = self.token.kind
{
//eprintln!("BUMP {:?}", self.token.kind);
self.bump();
// njn: parse_expr_force_collect?
match self.parse_expr() {
Ok(expr) => {
match self.expect(&token::CloseDelim(Delimiter::Invisible(
InvisibleSource::ProcMacro,
))) {
Ok(_) => {
return Ok(expr);
}
Err(_) => panic!("njn: no invisible close delim: {:?}", self.token),
}
}
Err(_) => {
panic!("njn: bad expr parse");
}
}
}
//eprintln!("CCC {:?}", self.token.kind);

// Outer attributes are already parsed and will be
// added to the return value after the fact.

// Note: when adding new syntax here, don't forget to adjust `TokenKind::can_begin_expr()`.
let lo = self.token.span;
if let token::Literal(_) = self.token.kind {
let res = if let token::Literal(_) = self.token.kind {
// This match arm is a special-case of the `_` match arm below and
// could be removed without changing functionality, but it's faster
// to have it here, especially for programs with large constants.
Expand Down Expand Up @@ -1473,7 +1497,9 @@ impl<'a> Parser<'a> {
}
} else {
self.parse_expr_lit()
}
};
//eprintln!("DDD {:?} -> {:#?}", self.token.kind, res);
res
}

fn parse_expr_lit(&mut self) -> PResult<'a, P<Expr>> {
Expand Down Expand Up @@ -1964,6 +1990,7 @@ impl<'a> Parser<'a> {
&mut self,
mk_lit_char: impl FnOnce(Symbol, Span) -> L,
) -> PResult<'a, L> {
// njn: proc macro too?
if let Some(NonterminalKind::Expr | NonterminalKind::Literal) = self.token.is_metavar_seq()
{
// njn: not checking for ExprKind::Err
Expand Down Expand Up @@ -2090,6 +2117,9 @@ impl<'a> Parser<'a> {
None
}
}
token::OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) => {
panic!("njn: maybe_parse_token_lit");
}
_ => None,
}
}
Expand Down Expand Up @@ -2157,6 +2187,25 @@ impl<'a> Parser<'a> {
pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
maybe_reparse_metavar_expr!(self);

// if let token::OpenDelim(Delimiter::Invisible) = self.token.kind {
// // njn: need parse_expr for negative case?
// self.bump();
// // njn: parse_expr_force_collect?
// match self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus()) {
// Ok(expr) => {
// match self.expect(&token::CloseDelim(Delimiter::Invisible)) {
// Ok(_) => {
// return Ok(expr);
// }
// Err(_) => panic!("njn: no invisible close delim: {:?}", self.token),
// }
// }
// Err(_) => {
// panic!("njn: bad expr parse");
// }
// }
// }

let lo = self.token.span;
let minus_present = self.eat(&token::BinOp(token::Minus));
let (token_lit, span) = self.parse_token_lit()?;
Expand Down
30 changes: 30 additions & 0 deletions compiler/rustc_parse/src/parser/item.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,36 @@ impl<'a> Parser<'a> {
return Ok(Some(item.into_inner()));
}

if let token::OpenDelim(Delimiter::Invisible(InvisibleSource::ProcMacro)) = self.token.kind
{
//eprintln!("ITEM BUMP {:?}", self.token.kind);
self.bump();
match self.parse_item(ForceCollect::Yes) {
Ok(Some(mut item)) => {
match self.expect(&token::CloseDelim(Delimiter::Invisible(
InvisibleSource::ProcMacro,
))) {
Ok(_) => {
attrs.prepend_to_nt_inner(&mut item.attrs);
return Ok(Some(item.into_inner()));
}
Err(_) => panic!("njn: no invisible close delim 1: {:?}", self.token),
}
}
Ok(None) => {
panic!("njn: missing item {:?}", self.token);
// match self.expect(&token::CloseDelim(Delimiter::Invisible)) {
// Ok(_) => return Ok(None),
// // njn: hitting on tests/ui/proc-macro/issue-75734-pp-paren.rs, hmm
// Err(_) => panic!("njn: no invisible close delim 2: {:?}", self.token),
// }
}
Err(_) => {
panic!("njn: bad item parse");
}
}
}

let item =
self.collect_tokens_trailing_token(attrs, force_collect, |this: &mut Self, attrs| {
let item =
Expand Down

0 comments on commit 848d2d2

Please sign in to comment.