Add support for marking doctest items as distinct from normal code

crates/ra_ide/src/snapshots/highlight_doctest.html

@@ -5,6 +5,8 @@
 
 .lifetime           { color: #DFAF8F; font-style: italic; }
 .comment            { color: #7F9F7F; }
+.documentation      { color: #629755; }
+.injected           { opacity: 0.65 ; }
 .struct, .enum      { color: #7CB8BB; }
 .enum_variant       { color: #BDE0F3; }
 .string_literal     { color: #CC9393; }
@@ -33,7 +35,7 @@
 .control            { font-style: italic; }
 </style>
 <pre><code><span class="comment documentation">/// ```</span>
-<span class="comment documentation">/// </span><span class="keyword">let</span> _ = <span class="string_literal">"early doctests should not go boom"</span>;
+<span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="generic injected"> _ = </span><span class="string_literal injected">"early doctests should not go boom"</span><span class="generic injected">;</span>
 <span class="comment documentation">/// ```</span>
 <span class="keyword">struct</span> <span class="struct declaration">Foo</span> {
     <span class="field declaration">bar</span>: <span class="builtin_type">bool</span>,
@@ -47,8 +49,8 @@
     <span class="comment documentation">/// # Examples</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```</span>
-    <span class="comment documentation">/// #</span> <span class="attribute">#![</span><span class="function attribute">allow</span><span class="attribute">(unused_mut)]</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="keyword">mut</span> <span class="variable declaration mutable">foo</span>: <span class="struct">Foo</span> = <span class="struct">Foo</span>::<span class="function">new</span>();
+    <span class="comment documentation">/// #</span><span class="generic injected"> </span><span class="attribute injected">#![</span><span class="function attribute injected">allow</span><span class="attribute injected">(unused_mut)]</span>
+    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="generic injected"> </span><span class="keyword injected">mut</span><span class="generic injected"> </span><span class="variable declaration injected mutable">foo</span><span class="generic injected">: </span><span class="struct injected">Foo</span><span class="generic injected"> = </span><span class="struct injected">Foo</span><span class="generic injected">::</span><span class="function injected">new</span><span class="generic injected">();</span>
     <span class="comment documentation">/// ```</span>
     <span class="keyword">pub</span> <span class="keyword">const</span> <span class="keyword">fn</span> <span class="function declaration">new</span>() -&gt; <span class="struct">Foo</span> {
         <span class="struct">Foo</span> { <span class="field">bar</span>: <span class="bool_literal">true</span> }
@@ -59,26 +61,26 @@
     <span class="comment documentation">/// # Examples</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```</span>
-    <span class="comment documentation">/// </span><span class="keyword">use</span> <span class="module">x</span>::<span class="module">y</span>;
+    <span class="comment documentation">/// </span><span class="keyword injected">use</span><span class="generic injected"> </span><span class="module injected">x</span><span class="generic injected">::</span><span class="module injected">y</span><span class="generic injected">;</span>
     <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="variable declaration">foo</span> = <span class="struct">Foo</span>::<span class="function">new</span>();
+    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="generic injected"> </span><span class="variable declaration injected">foo</span><span class="generic injected"> = </span><span class="struct injected">Foo</span><span class="generic injected">::</span><span class="function injected">new</span><span class="generic injected">();</span>
     <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="comment">// calls bar on foo</span>
-    <span class="comment documentation">/// </span><span class="macro">assert!</span>(foo.bar());
+    <span class="comment documentation">/// </span><span class="comment injected">// calls bar on foo</span>
+    <span class="comment documentation">/// </span><span class="macro injected">assert!</span><span class="generic injected">(foo.bar());</span>
     <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="variable declaration">bar</span> = <span class="variable">foo</span>.<span class="field">bar</span> || <span class="struct">Foo</span>::<span class="constant">bar</span>;
+    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="generic injected"> </span><span class="variable declaration injected">bar</span><span class="generic injected"> = </span><span class="variable injected">foo</span><span class="generic injected">.</span><span class="field injected">bar</span><span class="generic injected"> || </span><span class="struct injected">Foo</span><span class="generic injected">::</span><span class="constant injected">bar</span><span class="generic injected">;</span>
     <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="comment">/* multi-line
-    </span><span class="comment documentation">/// </span><span class="comment">       comment */</span>
+    <span class="comment documentation">/// </span><span class="comment injected">/* multi-line
+    </span><span class="comment documentation">/// </span><span class="comment injected">       comment */</span>
     <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="variable declaration">multi_line_string</span> = <span class="string_literal">"Foo
-    </span><span class="comment documentation">/// </span><span class="string_literal">  bar
-    </span><span class="comment documentation">/// </span><span class="string_literal">         "</span>;
+    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="generic injected"> </span><span class="variable declaration injected">multi_line_string</span><span class="generic injected"> = </span><span class="string_literal injected">"Foo
+    </span><span class="comment documentation">/// </span><span class="string_literal injected">  bar
+    </span><span class="comment documentation">/// </span><span class="string_literal injected">         "</span><span class="generic injected">;</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```rust,no_run</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="variable declaration">foobar</span> = <span class="struct">Foo</span>::<span class="function">new</span>().<span class="function">bar</span>();
+    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="generic injected"> </span><span class="variable declaration injected">foobar</span><span class="generic injected"> = </span><span class="struct injected">Foo</span><span class="generic injected">::</span><span class="function injected">new</span><span class="generic injected">().</span><span class="function injected">bar</span><span class="generic injected">();</span>
     <span class="comment documentation">/// ```</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```sh</span>
@@ -90,7 +92,7 @@
 }
 
 <span class="comment documentation">/// ```</span>
-<span class="comment documentation">/// </span><span class="macro">noop!</span>(<span class="numeric_literal">1</span>);
+<span class="comment documentation">/// </span><span class="macro injected">noop!</span><span class="generic injected">(</span><span class="numeric_literal injected">1</span><span class="generic injected">);</span>
 <span class="comment documentation">/// ```</span>
 <span class="macro">macro_rules!</span> <span class="macro declaration">noop</span> {
     ($expr:expr) =&gt; {

crates/ra_ide/src/snapshots/highlight_injection.html

@@ -5,6 +5,8 @@
 
 .lifetime           { color: #DFAF8F; font-style: italic; }
 .comment            { color: #7F9F7F; }
+.documentation      { color: #629755; }
+.injected           { opacity: 0.65 ; }
 .struct, .enum      { color: #7CB8BB; }
 .enum_variant       { color: #BDE0F3; }
 .string_literal     { color: #CC9393; }

crates/ra_ide/src/snapshots/highlight_strings.html

@@ -5,6 +5,8 @@
 
 .lifetime           { color: #DFAF8F; font-style: italic; }
 .comment            { color: #7F9F7F; }
+.documentation      { color: #629755; }
+.injected           { opacity: 0.65 ; }
 .struct, .enum      { color: #7CB8BB; }
 .enum_variant       { color: #BDE0F3; }
 .string_literal     { color: #CC9393; }

crates/ra_ide/src/snapshots/highlight_unsafe.html

@@ -5,6 +5,8 @@
 
 .lifetime           { color: #DFAF8F; font-style: italic; }
 .comment            { color: #7F9F7F; }
+.documentation      { color: #629755; }
+.injected           { opacity: 0.65 ; }
 .struct, .enum      { color: #7CB8BB; }
 .enum_variant       { color: #BDE0F3; }
 .string_literal     { color: #CC9393; }

crates/ra_ide/src/snapshots/highlighting.html

@@ -5,6 +5,8 @@
 
 .lifetime           { color: #DFAF8F; font-style: italic; }
 .comment            { color: #7F9F7F; }
+.documentation      { color: #629755; }
+.injected           { opacity: 0.65 ; }
 .struct, .enum      { color: #7CB8BB; }
 .enum_variant       { color: #BDE0F3; }
 .string_literal     { color: #CC9393; }

crates/ra_ide/src/snapshots/rainbow_highlighting.html

@@ -5,6 +5,8 @@
 
 .lifetime           { color: #DFAF8F; font-style: italic; }
 .comment            { color: #7F9F7F; }
+.documentation      { color: #629755; }
+.injected           { opacity: 0.65 ; }
 .struct, .enum      { color: #7CB8BB; }
 .enum_variant       { color: #BDE0F3; }
 .string_literal     { color: #CC9393; }

crates/ra_ide/src/syntax_highlighting.rs

@@ -236,7 +236,7 @@ pub(crate) fn highlight(
                             });
                         }
                     }
-                    stack.pop_and_inject(false);
+                    stack.pop_and_inject(None);
                 }
             } else if let Some(string) =
                 element_to_highlight.as_token().cloned().and_then(ast::RawString::cast)
@@ -324,16 +324,27 @@ impl HighlightedRangeStack {
         cloned
     }
 
+    /// Remove the `HighlightRange` of `parent` that's currently covered by `child`.
+    fn intersect_partial(parent: &mut HighlightedRange, child: &HighlightedRange) {
+        assert!(
+            parent.range.start() <= child.range.start()
+                && parent.range.end() >= child.range.start()
+                && child.range.end() > parent.range.end()
+        );
+
+        parent.range = TextRange::new(parent.range.start(), child.range.start());
+    }
+
     /// Similar to `pop`, but can modify arbitrary prior ranges (where `pop`)
     /// can only modify the last range currently on the stack.
     /// Can be used to do injections that span multiple ranges, like the
     /// doctest injection below.
-    /// If `delete` is set to true, the parent range is deleted instead of
-    /// intersected.
+    /// If `overwrite_parent` is non-optional, the highlighting of the parent range
+    /// is overwritten with the argument.
     ///
     /// Note that `pop` can be simulated by `pop_and_inject(false)` but the
     /// latter is computationally more expensive.
-    fn pop_and_inject(&mut self, delete: bool) {
+    fn pop_and_inject(&mut self, overwrite_parent: Option<Highlight>) {
         let mut children = self.stack.pop().unwrap();
         let prev = self.stack.last_mut().unwrap();
         children.sort_by_key(|range| range.range.start());
@@ -343,26 +354,45 @@ impl HighlightedRangeStack {
             if let Some(idx) =
                 prev.iter().position(|parent| parent.range.contains_range(child.range))
             {
+                if let Some(tag) = overwrite_parent {
+                    prev[idx].highlight = tag;
+                }
+
                 let cloned = Self::intersect(&mut prev[idx], &child);
-                let insert_idx = if delete || prev[idx].range.is_empty() {
+                let insert_idx = if prev[idx].range.is_empty() {
                     prev.remove(idx);
                     idx
                 } else {
                     idx + 1
                 };
                 prev.insert(insert_idx, child);
-                if !delete && !cloned.range.is_empty() {
+                if !cloned.range.is_empty() {
                     prev.insert(insert_idx + 1, cloned);
                 }
-            } else if let Some(_idx) =
-                prev.iter().position(|parent| parent.range.contains(child.range.start()))
-            {
-                unreachable!("child range should be completely contained in parent range");
             } else {
-                let idx = prev
-                    .binary_search_by_key(&child.range.start(), |range| range.range.start())
-                    .unwrap_or_else(|x| x);
-                prev.insert(idx, child);
+                let maybe_idx =
+                    prev.iter().position(|parent| parent.range.contains(child.range.start()));
+                match (overwrite_parent, maybe_idx) {
+                    (Some(_), Some(idx)) => {
+                        Self::intersect_partial(&mut prev[idx], &child);
+                        let insert_idx = if prev[idx].range.is_empty() {
+                            prev.remove(idx);
+                            idx
+                        } else {
+                            idx + 1
+                        };
+                        prev.insert(insert_idx, child);
+                    }
+                    (_, None) => {
+                        let idx = prev
+                            .binary_search_by_key(&child.range.start(), |range| range.range.start())
+                            .unwrap_or_else(|x| x);
+                        prev.insert(idx, child);
+                    }
+                    _ => {
+                        unreachable!("child range should be completely contained in parent range");
+                    }
+                }
             }
         }
     }
@@ -516,11 +546,9 @@ fn highlight_element(
             let ty = sema.type_of_expr(&expr)?;
             if !ty.is_raw_ptr() {
                 return None;
+            } else {
+                HighlightTag::Operator | HighlightModifier::Unsafe
             }
-
-            let mut h = Highlight::new(HighlightTag::Operator);
-            h |= HighlightModifier::Unsafe;
-            h
         }
         T![!] if element.parent().and_then(ast::MacroCall::cast).is_some() => {
             Highlight::new(HighlightTag::Macro)

crates/ra_ide/src/syntax_highlighting/html.rs

@@ -64,6 +64,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 
 .lifetime           { color: #DFAF8F; font-style: italic; }
 .comment            { color: #7F9F7F; }
+.documentation      { color: #629755; }
+.injected           { opacity: 0.65 ; }
 .struct, .enum      { color: #7CB8BB; }
 .enum_variant       { color: #BDE0F3; }
 .string_literal     { color: #CC9393; }

crates/ra_ide/src/syntax_highlighting/injection.rs

@@ -8,8 +8,8 @@ use ra_syntax::{ast, AstToken, SyntaxNode, SyntaxToken, TextRange, TextSize};
 use stdx::SepBy;
 
 use crate::{
-    call_info::ActiveParameter, Analysis, HighlightModifier, HighlightTag, HighlightedRange,
-    RootDatabase,
+    call_info::ActiveParameter, Analysis, Highlight, HighlightModifier, HighlightTag,
+    HighlightedRange, RootDatabase,
 };
 
 use super::HighlightedRangeStack;
@@ -172,6 +172,7 @@ pub(super) fn highlight_doc_comment(
                 h.range.end() + end_offset.unwrap_or(start_offset) - h.range.start(),
             );
 
+            h.highlight |= HighlightModifier::Injected;
             stack.add(h);
         }
     }
@@ -181,6 +182,7 @@ pub(super) fn highlight_doc_comment(
     for comment in new_comments {
         stack.add(comment);
     }
-    stack.pop_and_inject(false);
-    stack.pop_and_inject(true);
+    stack.pop_and_inject(None);
+    stack
+        .pop_and_inject(Some(Highlight::from(HighlightTag::Generic) | HighlightModifier::Injected));
 }

crates/ra_ide/src/syntax_highlighting/tags.rs

@@ -27,6 +27,7 @@ pub enum HighlightTag {
     Field,
     FormatSpecifier,
     Function,
+    Generic,
     Keyword,
     Lifetime,
     Macro,
@@ -57,6 +58,7 @@ pub enum HighlightModifier {
     /// not.
     Definition,
     Documentation,
+    Injected,
     Mutable,
     Unsafe,
 }
@@ -77,6 +79,7 @@ impl HighlightTag {
             HighlightTag::Field => "field",
             HighlightTag::FormatSpecifier => "format_specifier",
             HighlightTag::Function => "function",
+            HighlightTag::Generic => "generic",
             HighlightTag::Keyword => "keyword",
             HighlightTag::Lifetime => "lifetime",
             HighlightTag::Macro => "macro",
@@ -110,6 +113,7 @@ impl HighlightModifier {
         HighlightModifier::ControlFlow,
         HighlightModifier::Definition,
         HighlightModifier::Documentation,
+        HighlightModifier::Injected,
         HighlightModifier::Mutable,
         HighlightModifier::Unsafe,
     ];
@@ -120,6 +124,7 @@ impl HighlightModifier {
             HighlightModifier::ControlFlow => "control",
             HighlightModifier::Definition => "declaration",
             HighlightModifier::Documentation => "documentation",
+            HighlightModifier::Injected => "injected",
             HighlightModifier::Mutable => "mutable",
             HighlightModifier::Unsafe => "unsafe",
         }

crates/rust-analyzer/src/semantic_tokens.rs

@@ -39,13 +39,14 @@ define_semantic_token_types![
     (BOOLEAN, "boolean"),
     (BUILTIN_TYPE, "builtinType"),
     (ENUM_MEMBER, "enumMember"),
+    (ESCAPE_SEQUENCE, "escapeSequence"),
+    (FORMAT_SPECIFIER, "formatSpecifier"),
+    (GENERIC, "generic"),
     (LIFETIME, "lifetime"),
     (SELF_KEYWORD, "selfKeyword"),
     (TYPE_ALIAS, "typeAlias"),
     (UNION, "union"),
     (UNRESOLVED_REFERENCE, "unresolvedReference"),
-    (FORMAT_SPECIFIER, "formatSpecifier"),
-    (ESCAPE_SEQUENCE, "escapeSequence"),
 ];
 
 macro_rules! define_semantic_token_modifiers {
@@ -68,6 +69,7 @@ macro_rules! define_semantic_token_modifiers {
 define_semantic_token_modifiers![
     (CONSTANT, "constant"),
     (CONTROL_FLOW, "controlFlow"),
+    (INJECTED, "injected"),
     (MUTABLE, "mutable"),
     (UNSAFE, "unsafe"),
     (ATTRIBUTE_MODIFIER, "attribute"),

crates/rust-analyzer/src/to_proto.rs

@@ -295,6 +295,7 @@ fn semantic_token_type_and_modifiers(
         HighlightTag::SelfType => lsp_types::SemanticTokenType::TYPE,
         HighlightTag::Field => lsp_types::SemanticTokenType::PROPERTY,
         HighlightTag::Function => lsp_types::SemanticTokenType::FUNCTION,
+        HighlightTag::Generic => semantic_tokens::GENERIC,
         HighlightTag::Module => lsp_types::SemanticTokenType::NAMESPACE,
         HighlightTag::Constant => {
             mods |= semantic_tokens::CONSTANT;
@@ -331,6 +332,7 @@ fn semantic_token_type_and_modifiers(
             HighlightModifier::Attribute => semantic_tokens::ATTRIBUTE_MODIFIER,
             HighlightModifier::Definition => lsp_types::SemanticTokenModifier::DECLARATION,
             HighlightModifier::Documentation => lsp_types::SemanticTokenModifier::DOCUMENTATION,
+            HighlightModifier::Injected => semantic_tokens::INJECTED,
             HighlightModifier::ControlFlow => semantic_tokens::CONTROL_FLOW,
             HighlightModifier::Mutable => semantic_tokens::MUTABLE,
             HighlightModifier::Unsafe => semantic_tokens::UNSAFE,