-
Notifications
You must be signed in to change notification settings - Fork 12.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Auto merge of #78712 - petrochenkov:visitok, r=Aaron1011
rustc_ast: Visit tokens stored in AST nodes in mutable visitor After #77271 token visiting is enabled only for one visitor in `rustc_expand\src\mbe\transcribe.rs` which applies hygiene marks to tokens produced by declarative macros (`macro_rules` or `macro`), so this change doesn't affect anything else. When a macro has some interpolated token from an outer macro in its output ```rust macro inner() { $interpolated } ``` we can use the usual interpretation of interpolated tokens in token-based model - a None-delimited group - to write this macro in an equivalent form ```rust macro inner() { ⟪ a b c d ⟫ } ``` When we are expanding the macro `inner` we need to apply hygiene marks to all tokens produced by it, including the tokens inside the group. Before this PR we did this by visiting the AST piece inside the interpolated token and applying marks to all spans in it. I'm not sure this is 100% correct (ideally we should apply the marks to tokens and then re-parse the AST from tokens), but it's a very good approximation at least. We didn't however apply the marks to actual tokens stored in the nonterminal, so if we used the nonterminal as a token rather than as an AST piece (e.g. passed it to a proc macro), then we got hygiene bugs. This PR applies the marks to tokens in addition to the AST pieces thus fixing the issue. r? `@Aaron1011`
- Loading branch information
Showing
3 changed files
with
156 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| // Make sure that marks from declarative macros are applied to tokens in nonterminal. | ||
|
|
||
| // check-pass | ||
| // compile-flags: -Z span-debug -Z macro-backtrace -Z unpretty=expanded,hygiene | ||
| // compile-flags: -Z trim-diagnostic-paths=no | ||
| // normalize-stdout-test "\d+#" -> "0#" | ||
| // aux-build:test-macros.rs | ||
|
|
||
| #![feature(decl_macro)] | ||
|
|
||
| #![no_std] // Don't load unnecessary hygiene information from std | ||
| extern crate std; | ||
|
|
||
| #[macro_use] | ||
| extern crate test_macros; | ||
|
|
||
| macro_rules! outer { | ||
| ($item:item) => { | ||
| macro inner() { | ||
| print_bang! { $item } | ||
| } | ||
|
|
||
| inner!(); | ||
| }; | ||
| } | ||
|
|
||
| struct S; | ||
|
|
||
| outer! { | ||
| struct S; // OK, not a duplicate definition of `S` | ||
| } | ||
|
|
||
| fn main() {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| PRINT-BANG INPUT (DISPLAY): struct S; | ||
| PRINT-BANG RE-COLLECTED (DISPLAY): struct S ; | ||
| PRINT-BANG INPUT (DEBUG): TokenStream [ | ||
| Group { | ||
| delimiter: None, | ||
| stream: TokenStream [ | ||
| Ident { | ||
| ident: "struct", | ||
| span: $DIR/nonterminal-token-hygiene.rs:30:5: 30:11 (#5), | ||
| }, | ||
| Ident { | ||
| ident: "S", | ||
| span: $DIR/nonterminal-token-hygiene.rs:30:12: 30:13 (#5), | ||
| }, | ||
| Punct { | ||
| ch: ';', | ||
| spacing: Alone, | ||
| span: $DIR/nonterminal-token-hygiene.rs:30:13: 30:14 (#5), | ||
| }, | ||
| ], | ||
| span: $DIR/nonterminal-token-hygiene.rs:20:27: 20:32 (#6), | ||
| }, | ||
| ] | ||
| #![feature /* 0#0 */(prelude_import)] | ||
| #![no_std /* 0#0 */] | ||
| // Make sure that marks from declarative macros are applied to tokens in nonterminal. | ||
|
|
||
| // check-pass | ||
| // compile-flags: -Z span-debug -Z macro-backtrace -Z unpretty=expanded,hygiene | ||
| // compile-flags: -Z trim-diagnostic-paths=no | ||
| // normalize-stdout-test "\d+#" -> "0#" | ||
| // aux-build:test-macros.rs | ||
|
|
||
| #![feature /* 0#0 */(decl_macro)] | ||
|
|
||
| #![no_std /* 0#0 */] | ||
| #[prelude_import /* 0#1 */] | ||
| use ::core /* 0#1 */::prelude /* 0#1 */::v1 /* 0#1 */::*; | ||
| #[macro_use /* 0#1 */] | ||
| extern crate core /* 0#2 */; | ||
| #[macro_use /* 0#1 */] | ||
| extern crate compiler_builtins /* 0#2 */; | ||
| // Don't load unnecessary hygiene information from std | ||
| extern crate std /* 0#0 */; | ||
|
|
||
| #[macro_use /* 0#0 */] | ||
| extern crate test_macros /* 0#0 */; | ||
|
|
||
| macro_rules! outer | ||
| /* | ||
| 0#0 | ||
| */ { | ||
| ($ item : item) => | ||
| { | ||
| macro inner() { print_bang ! { $ item } } inner ! () ; | ||
|
|
||
| } ; | ||
| } | ||
|
|
||
| struct S /* 0#0 */; | ||
| macro inner /* 0#4 */ { () => { print_bang ! { struct S; } } } | ||
|
|
||
| struct S /* 0#5 */; | ||
| // OK, not a duplicate definition of `S` | ||
|
|
||
| fn main /* 0#0 */() { } | ||
|
|
||
| /* | ||
| Expansions: | ||
| 0: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: Root | ||
| 1: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) | ||
| 2: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "outer") | ||
| 3: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) | ||
| 4: parent: ExpnId(2), call_site_ctxt: #4, def_site_ctxt: #4, kind: Macro(Bang, "inner") | ||
| 5: parent: ExpnId(4), call_site_ctxt: #6, def_site_ctxt: #0, kind: Macro(Bang, "print_bang") | ||
|
|
||
| SyntaxContexts: | ||
| #0: parent: #0, outer_mark: (ExpnId(0), Opaque) | ||
| #1: parent: #0, outer_mark: (ExpnId(1), Opaque) | ||
| #2: parent: #0, outer_mark: (ExpnId(1), Transparent) | ||
| #3: parent: #0, outer_mark: (ExpnId(3), Opaque) | ||
| #4: parent: #0, outer_mark: (ExpnId(2), SemiTransparent) | ||
| #5: parent: #0, outer_mark: (ExpnId(4), Opaque) | ||
| #6: parent: #4, outer_mark: (ExpnId(4), Opaque) | ||
| #7: parent: #0, outer_mark: (ExpnId(5), Opaque) | ||
| #8: parent: #6, outer_mark: (ExpnId(5), Transparent) | ||
| #9: parent: #5, outer_mark: (ExpnId(5), SemiTransparent) | ||
| */ |