Auto merge of #17153 - Veykril:doc-comment-desugaring, r=Veykril

fix: Fix doc comment desugaring for proc-macros Fixes #16259
rust-lang · Apr 27, 2024 · f216be4 · f216be4
2 parents e6f3399 + 36c1c77
commit f216be4
Show file tree

Hide file tree

Showing 14 changed files with 228 additions and 52 deletions.
diff --git a/crates/cfg/src/tests.rs b/crates/cfg/src/tests.rs
@@ -1,22 +1,32 @@
 use arbitrary::{Arbitrary, Unstructured};
 use expect_test::{expect, Expect};
-use mbe::{syntax_node_to_token_tree, DummyTestSpanMap, DUMMY};
+use mbe::{syntax_node_to_token_tree, DocCommentDesugarMode, DummyTestSpanMap, DUMMY};
 use syntax::{ast, AstNode, Edition};
 
 use crate::{CfgAtom, CfgExpr, CfgOptions, DnfExpr};
 
 fn assert_parse_result(input: &str, expected: CfgExpr) {
     let source_file = ast::SourceFile::parse(input, Edition::CURRENT).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
+    let tt = syntax_node_to_token_tree(
+        tt.syntax(),
+        DummyTestSpanMap,
+        DUMMY,
+        DocCommentDesugarMode::ProcMacro,
+    );
     let cfg = CfgExpr::parse(&tt);
     assert_eq!(cfg, expected);
 }
 
 fn check_dnf(input: &str, expect: Expect) {
     let source_file = ast::SourceFile::parse(input, Edition::CURRENT).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
+    let tt = syntax_node_to_token_tree(
+        tt.syntax(),
+        DummyTestSpanMap,
+        DUMMY,
+        DocCommentDesugarMode::ProcMacro,
+    );
     let cfg = CfgExpr::parse(&tt);
     let actual = format!("#![cfg({})]", DnfExpr::new(cfg));
     expect.assert_eq(&actual);
@@ -25,7 +35,12 @@ fn check_dnf(input: &str, expect: Expect) {
 fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) {
     let source_file = ast::SourceFile::parse(input, Edition::CURRENT).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
+    let tt = syntax_node_to_token_tree(
+        tt.syntax(),
+        DummyTestSpanMap,
+        DUMMY,
+        DocCommentDesugarMode::ProcMacro,
+    );
     let cfg = CfgExpr::parse(&tt);
     let dnf = DnfExpr::new(cfg);
     let why_inactive = dnf.why_inactive(opts).unwrap().to_string();
@@ -36,7 +51,12 @@ fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) {
 fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) {
     let source_file = ast::SourceFile::parse(input, Edition::CURRENT).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
+    let tt = syntax_node_to_token_tree(
+        tt.syntax(),
+        DummyTestSpanMap,
+        DUMMY,
+        DocCommentDesugarMode::ProcMacro,
+    );
     let cfg = CfgExpr::parse(&tt);
     let dnf = DnfExpr::new(cfg);
     let hints = dnf.compute_enable_hints(opts).map(|diff| diff.to_string()).collect::<Vec<_>>();

diff --git a/crates/hir-def/src/attr/tests.rs b/crates/hir-def/src/attr/tests.rs
@@ -5,7 +5,7 @@ use triomphe::Arc;
 
 use base_db::FileId;
 use hir_expand::span_map::{RealSpanMap, SpanMap};
-use mbe::syntax_node_to_token_tree;
+use mbe::{syntax_node_to_token_tree, DocCommentDesugarMode};
 use syntax::{ast, AstNode, TextRange};
 
 use crate::attr::{DocAtom, DocExpr};
@@ -18,6 +18,7 @@ fn assert_parse_result(input: &str, expected: DocExpr) {
         tt.syntax(),
         map.as_ref(),
         map.span_for_range(TextRange::empty(0.into())),
+        DocCommentDesugarMode::ProcMacro,
     );
     let cfg = DocExpr::parse(&tt);
     assert_eq!(cfg, expected);

diff --git a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs
@@ -186,3 +186,33 @@ fn#0:1@45..47#0# foo#0:1@48..51#0#(#0:1@51..52#0#&#0:1@52..53#0#self#0:1@53..57#
 }#0:1@76..77#0#"#]],
     );
 }
+
+#[test]
+fn attribute_macro_doc_desugaring() {
+    check(
+        r#"
+//- proc_macros: identity
+#[proc_macros::identity]
+/// doc string \n with newline
+/**
+     MultiLines Doc
+     MultiLines Doc
+*/
+#[doc = "doc attr"]
+struct S;
+"#,
+        expect![[r##"
+#[proc_macros::identity]
+/// doc string \n with newline
+/**
+     MultiLines Doc
+     MultiLines Doc
+*/
+#[doc = "doc attr"]
+struct S;
+
+#[doc = " doc string \\n with newline"]
+#[doc = "\n     MultiLines Doc\n     MultiLines Doc\n"]
+#[doc = "doc attr"] struct S;"##]],
+    );
+}
diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs
@@ -5,7 +5,7 @@ use base_db::CrateId;
 use cfg::CfgExpr;
 use either::Either;
 use intern::Interned;
-use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct};
+use mbe::{syntax_node_to_token_tree, DelimiterKind, DocCommentDesugarMode, Punct};
 use smallvec::{smallvec, SmallVec};
 use span::{Span, SyntaxContextId};
 use syntax::unescape;
@@ -239,7 +239,12 @@ impl Attr {
                 span,
             })))
         } else if let Some(tt) = ast.token_tree() {
-            let tree = syntax_node_to_token_tree(tt.syntax(), span_map, span);
+            let tree = syntax_node_to_token_tree(
+                tt.syntax(),
+                span_map,
+                span,
+                DocCommentDesugarMode::ProcMacro,
+            );
             Some(Interned::new(AttrInput::TokenTree(Box::new(tree))))
         } else {
             None

diff --git a/crates/hir-expand/src/builtin_derive_macro.rs b/crates/hir-expand/src/builtin_derive_macro.rs
@@ -1,6 +1,7 @@
 //! Builtin derives.
 
 use itertools::izip;
+use mbe::DocCommentDesugarMode;
 use rustc_hash::FxHashSet;
 use span::{MacroCallId, Span};
 use stdx::never;
@@ -262,23 +263,40 @@ fn parse_adt(tt: &tt::Subtree, call_site: Span) -> Result<BasicAdtInfo, ExpandEr
                 match this {
                     Some(it) => {
                         param_type_set.insert(it.as_name());
-                        mbe::syntax_node_to_token_tree(it.syntax(), tm, call_site)
+                        mbe::syntax_node_to_token_tree(
+                            it.syntax(),
+                            tm,
+                            call_site,
+                            DocCommentDesugarMode::ProcMacro,
+                        )
                     }
                     None => {
                         tt::Subtree::empty(::tt::DelimSpan { open: call_site, close: call_site })
                     }
                 }
             };
             let bounds = match &param {
-                ast::TypeOrConstParam::Type(it) => it
-                    .type_bound_list()
-                    .map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm, call_site)),
+                ast::TypeOrConstParam::Type(it) => it.type_bound_list().map(|it| {
+                    mbe::syntax_node_to_token_tree(
+                        it.syntax(),
+                        tm,
+                        call_site,
+                        DocCommentDesugarMode::ProcMacro,
+                    )
+                }),
                 ast::TypeOrConstParam::Const(_) => None,
             };
             let ty = if let ast::TypeOrConstParam::Const(param) = param {
                 let ty = param
                     .ty()
-                    .map(|ty| mbe::syntax_node_to_token_tree(ty.syntax(), tm, call_site))
+                    .map(|ty| {
+                        mbe::syntax_node_to_token_tree(
+                            ty.syntax(),
+                            tm,
+                            call_site,
+                            DocCommentDesugarMode::ProcMacro,
+                        )
+                    })
                     .unwrap_or_else(|| {
                         tt::Subtree::empty(::tt::DelimSpan { open: call_site, close: call_site })
                     });
@@ -292,7 +310,14 @@ fn parse_adt(tt: &tt::Subtree, call_site: Span) -> Result<BasicAdtInfo, ExpandEr
 
     let where_clause = if let Some(w) = where_clause {
         w.predicates()
-            .map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm, call_site))
+            .map(|it| {
+                mbe::syntax_node_to_token_tree(
+                    it.syntax(),
+                    tm,
+                    call_site,
+                    DocCommentDesugarMode::ProcMacro,
+                )
+            })
             .collect()
     } else {
         vec![]
@@ -322,7 +347,14 @@ fn parse_adt(tt: &tt::Subtree, call_site: Span) -> Result<BasicAdtInfo, ExpandEr
             let name = p.path()?.qualifier()?.as_single_name_ref()?.as_name();
             param_type_set.contains(&name).then_some(p)
         })
-        .map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm, call_site))
+        .map(|it| {
+            mbe::syntax_node_to_token_tree(
+                it.syntax(),
+                tm,
+                call_site,
+                DocCommentDesugarMode::ProcMacro,
+            )
+        })
         .collect();
     let name_token = name_to_token(tm, name)?;
     Ok(BasicAdtInfo { name: name_token, shape, param_types, where_clause, associated_types })

diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs
@@ -3,7 +3,7 @@
 use base_db::{salsa, CrateId, FileId, SourceDatabase};
 use either::Either;
 use limit::Limit;
-use mbe::{syntax_node_to_token_tree, MatchedArmIndex};
+use mbe::{syntax_node_to_token_tree, DocCommentDesugarMode, MatchedArmIndex};
 use rustc_hash::FxHashSet;
 use span::{AstIdMap, Span, SyntaxContextData, SyntaxContextId};
 use syntax::{ast, AstNode, Parse, SyntaxElement, SyntaxError, SyntaxNode, SyntaxToken, T};
@@ -156,11 +156,25 @@ pub fn expand_speculative(
     // Build the subtree and token mapping for the speculative args
     let (mut tt, undo_info) = match loc.kind {
         MacroCallKind::FnLike { .. } => (
-            mbe::syntax_node_to_token_tree(speculative_args, span_map, span),
+            mbe::syntax_node_to_token_tree(
+                speculative_args,
+                span_map,
+                span,
+                if loc.def.is_proc_macro() {
+                    DocCommentDesugarMode::ProcMacro
+                } else {
+                    DocCommentDesugarMode::Mbe
+                },
+            ),
             SyntaxFixupUndoInfo::NONE,
         ),
         MacroCallKind::Attr { .. } if loc.def.is_attribute_derive() => (
-            mbe::syntax_node_to_token_tree(speculative_args, span_map, span),
+            mbe::syntax_node_to_token_tree(
+                speculative_args,
+                span_map,
+                span,
+                DocCommentDesugarMode::ProcMacro,
+            ),
             SyntaxFixupUndoInfo::NONE,
         ),
         MacroCallKind::Derive { derive_attr_index: index, .. }
@@ -176,7 +190,12 @@ pub fn expand_speculative(
 
             let censor_cfg =
                 cfg_process::process_cfg_attrs(db, speculative_args, &loc).unwrap_or_default();
-            let mut fixups = fixup::fixup_syntax(span_map, speculative_args, span);
+            let mut fixups = fixup::fixup_syntax(
+                span_map,
+                speculative_args,
+                span,
+                DocCommentDesugarMode::ProcMacro,
+            );
             fixups.append.retain(|it, _| match it {
                 syntax::NodeOrToken::Token(_) => true,
                 it => !censor.contains(it) && !censor_cfg.contains(it),
@@ -191,6 +210,7 @@ pub fn expand_speculative(
                     fixups.append,
                     fixups.remove,
                     span,
+                    DocCommentDesugarMode::ProcMacro,
                 ),
                 fixups.undo_info,
             )
@@ -212,7 +232,12 @@ pub fn expand_speculative(
             }?;
             match attr.token_tree() {
                 Some(token_tree) => {
-                    let mut tree = syntax_node_to_token_tree(token_tree.syntax(), span_map, span);
+                    let mut tree = syntax_node_to_token_tree(
+                        token_tree.syntax(),
+                        span_map,
+                        span,
+                        DocCommentDesugarMode::ProcMacro,
+                    );
                     tree.delimiter = tt::Delimiter::invisible_spanned(span);
 
                     Some(tree)
@@ -432,7 +457,16 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
                 return dummy_tt(kind);
             }
 
-            let mut tt = mbe::syntax_node_to_token_tree(tt.syntax(), map.as_ref(), span);
+            let mut tt = mbe::syntax_node_to_token_tree(
+                tt.syntax(),
+                map.as_ref(),
+                span,
+                if loc.def.is_proc_macro() {
+                    DocCommentDesugarMode::ProcMacro
+                } else {
+                    DocCommentDesugarMode::Mbe
+                },
+            );
             if loc.def.is_proc_macro() {
                 // proc macros expect their inputs without parentheses, MBEs expect it with them included
                 tt.delimiter.kind = tt::DelimiterKind::Invisible;
@@ -469,7 +503,8 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
     let (mut tt, undo_info) = {
         let syntax = item_node.syntax();
         let censor_cfg = cfg_process::process_cfg_attrs(db, syntax, &loc).unwrap_or_default();
-        let mut fixups = fixup::fixup_syntax(map.as_ref(), syntax, span);
+        let mut fixups =
+            fixup::fixup_syntax(map.as_ref(), syntax, span, DocCommentDesugarMode::ProcMacro);
         fixups.append.retain(|it, _| match it {
             syntax::NodeOrToken::Token(_) => true,
             it => !censor.contains(it) && !censor_cfg.contains(it),
@@ -484,6 +519,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
                 fixups.append,
                 fixups.remove,
                 span,
+                DocCommentDesugarMode::ProcMacro,
             ),
             fixups.undo_info,
         )

diff --git a/crates/hir-expand/src/declarative.rs b/crates/hir-expand/src/declarative.rs
@@ -2,6 +2,7 @@
 use std::sync::OnceLock;
 
 use base_db::{CrateId, VersionReq};
+use mbe::DocCommentDesugarMode;
 use span::{Edition, MacroCallId, Span, SyntaxContextId};
 use stdx::TupleExt;
 use syntax::{ast, AstNode};
@@ -158,6 +159,7 @@ impl DeclarativeMacroExpander {
                             map.span_for_range(
                                 macro_rules.macro_rules_token().unwrap().text_range(),
                             ),
+                            DocCommentDesugarMode::Mbe,
                         );
 
                         mbe::DeclarativeMacro::parse_macro_rules(&tt, edition, new_meta_vars)
@@ -175,6 +177,7 @@ impl DeclarativeMacroExpander {
                             arg.syntax(),
                             map.as_ref(),
                             map.span_for_range(macro_def.macro_token().unwrap().text_range()),
+                            DocCommentDesugarMode::Mbe,
                         );
 
                         mbe::DeclarativeMacro::parse_macro2(&tt, edition, new_meta_vars)

diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs
@@ -19,6 +19,7 @@
 //!
 //! See the full discussion : <https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/Eager.20expansion.20of.20built-in.20macros>
 use base_db::CrateId;
+use mbe::DocCommentDesugarMode;
 use span::SyntaxContextId;
 use syntax::{ted, Parse, SyntaxElement, SyntaxNode, TextSize, WalkEvent};
 use triomphe::Arc;
@@ -80,7 +81,12 @@ pub fn expand_eager_macro_input(
         return ExpandResult { value: None, err };
     };
 
-    let mut subtree = mbe::syntax_node_to_token_tree(&expanded_eager_input, arg_map, span);
+    let mut subtree = mbe::syntax_node_to_token_tree(
+        &expanded_eager_input,
+        arg_map,
+        span,
+        DocCommentDesugarMode::Mbe,
+    );
 
     subtree.delimiter.kind = crate::tt::DelimiterKind::Invisible;