diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs index e2022c7967d8..ecc8d4ae576e 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs @@ -582,8 +582,8 @@ macro_rules! arbitrary { } impl $crate::arbitrary::Arbitrary for Vec { - type Parameters = RangedParams1 ; - type Strategy = VecStrategy ; + type Parameters = RangedParams1; + type Strategy = VecStrategy; fn arbitrary_with(args: Self::Parameters) -> Self::Strategy { { let product_unpack![range, a] = args; vec(any_with::(a), range) @@ -862,8 +862,8 @@ ok!(b0 b1); #[test] fn test_issue_3861() { - // This is should (and does) produce a parse error. It used to infinite loop - // instead. + // Used to produce a parse error and infinite loop. + // Now parses correctly after fix for #20958 (type parameter followed by shift operator). check( r#" macro_rules! rgb_color { @@ -873,7 +873,7 @@ macro_rules! rgb_color { } }; } -// +tree +errors +// +tree rgb_color!(8 + 8, u32); "#, expect![[r#" @@ -884,14 +884,6 @@ macro_rules! rgb_color { } }; } -/* parse error: expected type */ -/* parse error: expected R_PAREN */ -/* parse error: expected R_ANGLE */ -/* parse error: expected `::` */ -/* parse error: expected COMMA */ -/* parse error: expected R_ANGLE */ -/* parse error: expected SEMICOLON */ -/* parse error: expected expression, item or let statement */ pub fn new() { let _ = 0 as u32<<(8+8); } @@ -908,42 +900,32 @@ pub fn new() { // BLOCK_EXPR@10..31 // STMT_LIST@10..31 // L_CURLY@10..11 "{" -// LET_STMT@11..28 +// LET_STMT@11..30 // LET_KW@11..14 "let" // WILDCARD_PAT@14..15 // UNDERSCORE@14..15 "_" // EQ@15..16 "=" -// CAST_EXPR@16..28 -// LITERAL@16..17 -// INT_NUMBER@16..17 "0" -// AS_KW@17..19 "as" -// PATH_TYPE@19..28 -// PATH@19..28 -// PATH_SEGMENT@19..28 -// NAME_REF@19..22 -// IDENT@19..22 "u32" -// GENERIC_ARG_LIST@22..28 -// L_ANGLE@22..23 "<" -// TYPE_ARG@23..27 -// DYN_TRAIT_TYPE@23..27 -// TYPE_BOUND_LIST@23..27 -// TYPE_BOUND@23..26 -// PATH_TYPE@23..26 -// PATH@23..26 -// PATH_SEGMENT@23..26 -// TYPE_ANCHOR@23..26 -// L_ANGLE@23..24 "<" -// PAREN_TYPE@24..26 -// L_PAREN@24..25 "(" -// ERROR@25..26 -// INT_NUMBER@25..26 "8" -// PLUS@26..27 "+" -// CONST_ARG@27..28 -// LITERAL@27..28 -// INT_NUMBER@27..28 "8" -// ERROR@28..29 -// R_PAREN@28..29 ")" -// SEMICOLON@29..30 ";" +// BIN_EXPR@16..29 +// CAST_EXPR@16..22 +// LITERAL@16..17 +// INT_NUMBER@16..17 "0" +// AS_KW@17..19 "as" +// PATH_TYPE@19..22 +// PATH@19..22 +// PATH_SEGMENT@19..22 +// NAME_REF@19..22 +// IDENT@19..22 "u32" +// SHL@22..24 "<<" +// PAREN_EXPR@24..29 +// L_PAREN@24..25 "(" +// BIN_EXPR@25..28 +// LITERAL@25..26 +// INT_NUMBER@25..26 "8" +// PLUS@26..27 "+" +// LITERAL@27..28 +// INT_NUMBER@27..28 "8" +// R_PAREN@28..29 ")" +// SEMICOLON@29..30 ";" // R_CURLY@30..31 "}" "#]], @@ -1169,3 +1151,63 @@ fn foo() { "#]], ); } + +#[test] +fn regression_20958() { + // Type parameter followed by shift operator should not trigger E0109 + check( + r#" +macro_rules! ub_prim_impl { + ($prim_type:ty, $n:literal) => { + fn mask() -> $prim_type { + (1 as $prim_type << (1 << $n)) - 1 + } + }; +} + +ub_prim_impl!(u8, 0); +"#, + expect![[r#" +macro_rules! ub_prim_impl { + ($prim_type:ty, $n:literal) => { + fn mask() -> $prim_type { + (1 as $prim_type << (1 << $n)) - 1 + } + }; +} + +fn mask() -> u8 { + (1 as u8<<(1<<0))-1 +} +"#]], + ); +} + +#[test] +fn test_issue_20958() { + // Issue #20958: False positive E0109 when type parameter followed by shift operator + // The parser should recognize that `$t` from `$t:ty` is a complete type, so `<<` + // after it is a shift operator, not the start of generic arguments. + check( + r#" +macro_rules! test { + ($t:ty) => { + fn f() -> u8 { 1 as $t << 2 } + }; +} + +test!(u8); +"#, + expect![[r#" +macro_rules! test { + ($t:ty) => { + fn f() -> u8 { 1 as $t << 2 } + }; +} + +fn f() -> u8 { + 1 as u8<<2 +} +"#]], + ); +} diff --git a/crates/ide-diagnostics/src/handlers/generic_args_prohibited.rs b/crates/ide-diagnostics/src/handlers/generic_args_prohibited.rs index 9ae6f013c70d..5c1255b0e87e 100644 --- a/crates/ide-diagnostics/src/handlers/generic_args_prohibited.rs +++ b/crates/ide-diagnostics/src/handlers/generic_args_prohibited.rs @@ -638,4 +638,22 @@ fn foo() { "#, ); } + + #[test] + fn no_false_positive_with_shift_after_cast() { + // Issue #20958: Type parameter followed by shift operator should not trigger E0109 + check_diagnostics( + r#" +macro_rules! make_shift { + ($t:ty) => { + fn foo() { + let _ = 1 as $t << 2; + } + }; +} + +make_shift!(u8); + "#, + ); + } } diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index f910f9f9d753..a23d8243017b 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -140,6 +140,10 @@ enum Fragment<'a> { /// would cause a syntax error. We need to fix it up just before transcribing; /// see `transcriber::fix_up_and_push_path_tt()`. Path(tt::TokenTreesView<'a, Span>), + /// Type fragments need special handling to preserve operator precedence when followed by + /// shift operators like `<<` or `>>`. Unlike other fragments, we don't use `extend_with_tt_alone` + /// which would force spacing rules that break operator recognition. + Ty(tt::TokenTreesView<'a, Span>), TokensOwned(tt::TopSubtree), } @@ -150,6 +154,7 @@ impl Fragment<'_> { Fragment::Tokens(it) => it.len() == 0, Fragment::Expr(it) => it.len() == 0, Fragment::Path(it) => it.len() == 0, + Fragment::Ty(it) => it.len() == 0, Fragment::TokensOwned(it) => it.0.is_empty(), } } diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 189efcd15c2f..8e964d0659b6 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -844,7 +844,10 @@ fn match_meta_var<'t>( let tt_result = input.from_savepoint(savepoint); return ValueResult { value: Fragment::Tokens(tt_result), err }; } - MetaVarKind::Ty => parser::PrefixEntryPoint::Ty, + MetaVarKind::Ty => { + return expect_fragment(input, parser::PrefixEntryPoint::Ty, edition, delim_span) + .map(Fragment::Ty); + } MetaVarKind::Pat => parser::PrefixEntryPoint::PatTop, MetaVarKind::PatParam => parser::PrefixEntryPoint::Pat, MetaVarKind::Stmt => parser::PrefixEntryPoint::Stmt, diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 3e4ab8bdc1d8..3719aae34488 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -397,6 +397,14 @@ fn expand_var( Fragment::TokensOwned(tt) => { builder.extend_with_tt_alone(tt.view().strip_invisible()) } + // Type fragments - wrap in InvisibleTy delimiter to mark tokens for parser + Fragment::Ty(tt) => { + let mut span = id; + marker(&mut span); + builder.open(tt::DelimiterKind::InvisibleTy, span); + builder.extend_with_tt(tt.strip_invisible()); + builder.close(span); + } Fragment::Expr(sub) => { let sub = sub.strip_invisible(); let mut span = id; diff --git a/crates/parser/src/grammar/paths.rs b/crates/parser/src/grammar/paths.rs index dfe7cb57d24b..2d99de772b89 100644 --- a/crates/parser/src/grammar/paths.rs +++ b/crates/parser/src/grammar/paths.rs @@ -149,6 +149,12 @@ pub(crate) fn opt_path_type_args(p: &mut Parser<'_>) { m = p.start(); p.bump(T![::]); } else if (p.current() == T![<] && p.nth(1) != T![=]) || p.current() == T!['('] { + // If the previous token was from a type fragment (e.g., `$t:ty` in a macro) + // and we're seeing `<<`, treat it as a shift operator, not generic args. + // This fixes cases like `1 as $t << 2` where $t expands to `u8`. + if p.prev_from_type_fragment() && p.at(T![<<]) { + return; + } m = p.start(); } else { return; diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index 331bc58dd052..7cc1da6d1ed8 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -16,6 +16,9 @@ pub struct Input { kind: Vec, joint: Vec, contextual_kind: Vec, + /// Tracks tokens that came from type fragments in macro expansion. + /// This helps disambiguate shift operators from generic arguments. + from_type_fragment: Vec, } /// `pub` impl used by callers to create `Tokens`. @@ -26,6 +29,7 @@ impl Input { kind: Vec::with_capacity(capacity), joint: Vec::with_capacity(capacity / size_of::()), contextual_kind: Vec::with_capacity(capacity), + from_type_fragment: Vec::with_capacity(capacity / size_of::()), } } #[inline] @@ -58,11 +62,20 @@ impl Input { let (idx, b_idx) = self.bit_index(n); self.joint[idx] |= 1 << b_idx; } + /// Marks the last token as coming from a type fragment in macro expansion. + /// This helps the parser disambiguate shift operators from generic arguments. + #[inline] + pub fn was_from_type_fragment(&mut self) { + let n = self.len() - 1; + let (idx, b_idx) = self.bit_index(n); + self.from_type_fragment[idx] |= 1 << b_idx; + } #[inline] fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) { let idx = self.len(); if idx.is_multiple_of(bits::BITS as usize) { self.joint.push(0); + self.from_type_fragment.push(0); } self.kind.push(kind); self.contextual_kind.push(contextual_kind); @@ -81,6 +94,10 @@ impl Input { let (idx, b_idx) = self.bit_index(n); self.joint[idx] & (1 << b_idx) != 0 } + pub(crate) fn is_from_type_fragment(&self, n: usize) -> bool { + let (idx, b_idx) = self.bit_index(n); + self.from_type_fragment.get(idx).map_or(false, |bits| bits & (1 << b_idx) != 0) + } } impl Input { diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index ca02d9fdfde1..07da83e70244 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -167,6 +167,15 @@ impl<'t> Parser<'t> { self.inp.contextual_kind(self.pos + n) == kw } + /// Checks if the previous token came from a type fragment in macro expansion. + /// This helps disambiguate shift operators from generic arguments. + pub(crate) fn prev_from_type_fragment(&self) -> bool { + if self.pos == 0 { + return false; + } + self.inp.is_from_type_fragment(self.pos - 1) + } + /// Starts a new node in the syntax tree. All nodes and tokens /// consumed between the `start` and the corresponding `Marker::complete` /// belong to the same node. diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs index fb3542d24f46..a7f57dd71bbe 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs @@ -281,6 +281,7 @@ impl SubtreeRepr { tt::DelimiterKind::Parenthesis => 1, tt::DelimiterKind::Brace => 2, tt::DelimiterKind::Bracket => 3, + tt::DelimiterKind::InvisibleTy => 4, }; [self.open.0, kind, self.tt[0], self.tt[1]] } @@ -290,6 +291,7 @@ impl SubtreeRepr { 1 => tt::DelimiterKind::Parenthesis, 2 => tt::DelimiterKind::Brace, 3 => tt::DelimiterKind::Bracket, + 4 => tt::DelimiterKind::InvisibleTy, other => panic!("bad kind {other}"), }; SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] } @@ -300,6 +302,7 @@ impl SubtreeRepr { tt::DelimiterKind::Parenthesis => 1, tt::DelimiterKind::Brace => 2, tt::DelimiterKind::Bracket => 3, + tt::DelimiterKind::InvisibleTy => 4, }; [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] } @@ -309,6 +312,7 @@ impl SubtreeRepr { 1 => tt::DelimiterKind::Parenthesis, 2 => tt::DelimiterKind::Brace, 3 => tt::DelimiterKind::Bracket, + 4 => tt::DelimiterKind::InvisibleTy, other => panic!("bad kind {other}"), }; SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] } diff --git a/crates/syntax-bridge/src/lib.rs b/crates/syntax-bridge/src/lib.rs index 4e525be3fe3c..6b6f209b46c9 100644 --- a/crates/syntax-bridge/src/lib.rs +++ b/crates/syntax-bridge/src/lib.rs @@ -231,7 +231,7 @@ where tt::DelimiterKind::Parenthesis => char == ')', tt::DelimiterKind::Brace => char == '}', tt::DelimiterKind::Bracket => char == ']', - tt::DelimiterKind::Invisible => false, + tt::DelimiterKind::Invisible | tt::DelimiterKind::InvisibleTy => false, }); if let Some((idx, _)) = found_expected_delimiter { for _ in 0..=idx { @@ -266,7 +266,9 @@ where tt::DelimiterKind::Parenthesis => kind == T![')'], tt::DelimiterKind::Brace => kind == T!['}'], tt::DelimiterKind::Bracket => kind == T![']'], - tt::DelimiterKind::Invisible => false, + tt::DelimiterKind::Invisible | tt::DelimiterKind::InvisibleTy => { + false + } } }); @@ -873,7 +875,7 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> { tt::DelimiterKind::Parenthesis => "()", tt::DelimiterKind::Brace => "{}", tt::DelimiterKind::Bracket => "[]", - tt::DelimiterKind::Invisible => return None, + tt::DelimiterKind::Invisible | tt::DelimiterKind::InvisibleTy => return None, }; let idx = closing as usize; diff --git a/crates/syntax-bridge/src/to_parser_input.rs b/crates/syntax-bridge/src/to_parser_input.rs index c0ff8e1db2c2..6589901aa350 100644 --- a/crates/syntax-bridge/src/to_parser_input.rs +++ b/crates/syntax-bridge/src/to_parser_input.rs @@ -16,6 +16,8 @@ pub fn to_parser_input( let mut current = buffer.cursor(); let mut syntax_context_to_edition_cache = FxHashMap::default(); + // Track nesting level of InvisibleTy delimiters + let mut invisible_ty_depth = 0usize; while !current.eof() { let tt = current.token_tree(); @@ -88,28 +90,36 @@ pub fn to_parser_input( } } } + // Mark token as from type fragment if we're inside an InvisibleTy delimiter + if invisible_ty_depth > 0 { + res.was_from_type_fragment(); + } current.bump(); } Some(tt::TokenTree::Subtree(subtree)) => { - if let Some(kind) = match subtree.delimiter.kind { - tt::DelimiterKind::Parenthesis => Some(T!['(']), - tt::DelimiterKind::Brace => Some(T!['{']), - tt::DelimiterKind::Bracket => Some(T!['[']), - tt::DelimiterKind::Invisible => None, - } { - res.push(kind); + match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => res.push(T!['(']), + tt::DelimiterKind::Brace => res.push(T!['{']), + tt::DelimiterKind::Bracket => res.push(T!['[']), + tt::DelimiterKind::Invisible => (), + tt::DelimiterKind::InvisibleTy => { + // Entering an InvisibleTy delimiter - increment depth + invisible_ty_depth += 1; + } } current.bump(); } None => { let subtree = current.end(); - if let Some(kind) = match subtree.delimiter.kind { - tt::DelimiterKind::Parenthesis => Some(T![')']), - tt::DelimiterKind::Brace => Some(T!['}']), - tt::DelimiterKind::Bracket => Some(T![']']), - tt::DelimiterKind::Invisible => None, - } { - res.push(kind); + match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => res.push(T![')']), + tt::DelimiterKind::Brace => res.push(T!['}']), + tt::DelimiterKind::Bracket => res.push(T![']']), + tt::DelimiterKind::Invisible => (), + tt::DelimiterKind::InvisibleTy => { + // Exiting an InvisibleTy delimiter - decrement depth + invisible_ty_depth = invisible_ty_depth.saturating_sub(1); + } } } }; diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index f9a547f61138..7ce6969ae648 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -429,7 +429,7 @@ impl fmt::Display for TokenTreesView<'_, S> { DelimiterKind::Parenthesis => ("(", ")"), DelimiterKind::Brace => ("{", "}"), DelimiterKind::Bracket => ("[", "]"), - DelimiterKind::Invisible => ("", ""), + DelimiterKind::Invisible | DelimiterKind::InvisibleTy => ("", ""), }; f.write_str(l)?; token_trees_display(f, iter)?; @@ -562,6 +562,11 @@ pub enum DelimiterKind { Brace, Bracket, Invisible, + /// Invisible delimiter wrapping a type fragment from macro expansion. + /// This helps the parser disambiguate cases like `$t << 2` where `$t:ty` + /// expands to a complete type and `<<` should be parsed as a shift operator, + /// not the start of generic arguments. + InvisibleTy, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -731,6 +736,7 @@ fn print_debug_subtree( let Delimiter { kind, open, close } = &subtree.delimiter; let delim = match kind { DelimiterKind::Invisible => "$$", + DelimiterKind::InvisibleTy => "$$ty", DelimiterKind::Parenthesis => "()", DelimiterKind::Brace => "{}", DelimiterKind::Bracket => "[]", @@ -948,6 +954,7 @@ impl TopSubtree { DelimiterKind::Bracket => ("[", "]"), DelimiterKind::Parenthesis => ("(", ")"), DelimiterKind::Invisible => ("$", "$"), + DelimiterKind::InvisibleTy => ("$ty", "$ty"), }; output.push_str(delim.0); @@ -1023,7 +1030,7 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String { DelimiterKind::Brace => ("{", "}"), DelimiterKind::Bracket => ("[", "]"), DelimiterKind::Parenthesis => ("(", ")"), - DelimiterKind::Invisible => ("", ""), + DelimiterKind::Invisible | DelimiterKind::InvisibleTy => ("", ""), }; format!("{open}{content}{close}") }