support inline styled text

include/mrdocs/Support/String.hpp

@@ -45,8 +45,10 @@ rtrim(std::string_view s) noexcept
 {
     auto it = s.end() - 1;
     while(it > s.begin() && std::isspace(*it))
+    {
         --it;
-    return s.substr(0, it - s.begin());
+    }
+    return s.substr(0, it - s.begin() + 1);
 }
 
 /** Return the substring without leading and trailing horizontal whitespace.

src/lib/AST/ASTVisitor.cpp

@@ -1754,13 +1754,6 @@ generateJavadoc(
     {
         return false;
     }
-    // KRYSTIAN FIXME: clang ignores documentation comments
-    // when there is a preprocessor directive between the end
-    // of the comment and the declaration location. there are two
-    // ways to fix this: either set the declaration begin location
-    // to be before and preprocessor directives, or submit a patch
-    // which disables this behavior (it's not entirely clear why
-    // this check occurs anyways, so some investigation is needed)
     parseJavadoc(javadoc, FC, D, config_, diags_);
     return true;
 }

src/lib/AST/ParseJavadoc.cpp

@@ -16,6 +16,7 @@
 #include <mrdocs/Support/Error.hpp>
 #include <mrdocs/Support/Path.hpp>
 #include <mrdocs/Support/String.hpp>
+#include <mrdocs/Support/ScopeExit.hpp>
 #include <clang/AST/CommentCommandTraits.h>
 #include <clang/AST/ASTContext.h>
 #include <clang/AST/RawCommentList.h>
@@ -397,22 +398,183 @@ ensureUTF8(
     return s;
 }
 
+/*  Parse the inline content of a text
+
+    This function takes a string from a comment
+    and parses it into a sequence of styled text
+    nodes.
+
+    The string may contain inline commands that
+    change the style of the text:
+
+    Regular text is stored as a doc::Text.
+    Styled text is stored as a doc::Styled.
+
+    The styles can be one of: mono, bold, or italic.
+
+    The tags "`", "*", and "_" are used to indicate
+    the start and end of styled text. They can be
+    escaped by prefixing them with a backslash.
+
+ */
+doc::List<doc::Text>
+parseStyled(StringRef s)
+{
+    doc::List<doc::Text> result;
+    std::string currentText;
+    doc::Style currentStyle = doc::Style::none;
+    bool escapeNext = false;
+
+    auto isStyleMarker = [](char c) {
+        return c == '`' || c == '*' || c == '_';
+    };
+
+    auto flushCurrentText = [&]() {
+        if (!currentText.empty()) {
+            if (currentStyle == doc::Style::none) {
+                bool const lastIsSame =
+                    !result.empty() &&
+                    result.back()->kind == doc::Kind::text;
+                if (lastIsSame)
+                {
+                    auto& lastText = static_cast<doc::Text&>(*result.back());
+                    lastText.string.append(currentText);
+                }
+                else
+                {
+                    result.emplace_back(std::make_unique<doc::Text>(std::move(currentText)));
+                }
+            } else {
+                bool const lastIsSame =
+                    !result.empty() &&
+                    result.back()->kind == doc::Kind::styled &&
+                    static_cast<doc::Styled&>(*result.back()).style == currentStyle;
+                if (lastIsSame)
+                {
+                    auto& lastStyled = static_cast<doc::Styled&>(*result.back());
+                    lastStyled.string.append(currentText);
+                }
+                else
+                {
+                    result.emplace_back(std::make_unique<doc::Styled>(std::move(currentText), currentStyle));
+                }
+            }
+            currentText.clear();
+        }
+    };
+
+    auto isPunctuationOrSpace = [](char c) {
+        return std::isspace(c) || std::ispunct(c);
+    };
+
+    for (std::size_t i = 0; i < s.size(); ++i) {
+        char c = s[i];
+        if (escapeNext) {
+            currentText.push_back(c);
+            escapeNext = false;
+        } else if (c == '\\') {
+            escapeNext = true;
+        } else if (isStyleMarker(c)) {
+            bool const atWordBoundary =
+                (currentStyle == doc::Style::none && ((i == 0) || isPunctuationOrSpace(s[i - 1]))) ||
+                (currentStyle != doc::Style::none && ((i == s.size() - 1) || isPunctuationOrSpace(s[i + 1])));
+            if (atWordBoundary) {
+                flushCurrentText();
+                if (c == '`') {
+                    currentStyle = (currentStyle == doc::Style::mono) ? doc::Style::none : doc::Style::mono;
+                } else if (c == '*') {
+                    currentStyle = (currentStyle == doc::Style::bold) ? doc::Style::none : doc::Style::bold;
+                } else if (c == '_') {
+                    currentStyle = (currentStyle == doc::Style::italic) ? doc::Style::none : doc::Style::italic;
+                }
+            } else {
+                currentText.push_back(c);
+            }
+        } else {
+            currentText.push_back(c);
+        }
+    }
+
+    // Whatever style we started, we should end it because
+    // we reached the end of the string without a closing
+    // marker.
+    currentStyle = doc::Style::none;
+    flushCurrentText();
+
+    return result;
+}
+
 void
 JavadocVisitor::
 visitChildren(
     Comment const* C)
 {
-    auto const it0 = it_;
-    auto const end0 = end_;
-    it_ = C->child_begin();
-    end_ = C->child_end();
+    ScopeExitRestore s1(it_, C->child_begin());
+    ScopeExitRestore s2(end_, C->child_end());
     while(it_ != end_)
     {
         visit(*it_);
         ++it_; // must happen after
     }
-    it_ = it0;
-    end_ = end0;
+
+    if (!block_)
+    {
+        return;
+    }
+
+    bool const isVerbatim = block_->kind == doc::Kind::code;
+    if (isVerbatim)
+    {
+        return;
+    }
+
+    // Merge consecutive plain text nodes in the current block
+    auto it = block_->children.begin();
+    while(it != block_->children.end())
+    {
+        auto& child = *it;
+        if (child.get()->kind == doc::Kind::text)
+        {
+            auto* text = dynamic_cast<doc::Text*>(child.get());
+            MRDOCS_ASSERT(text);
+            auto next = std::next(it);
+            if(next != block_->children.end())
+            {
+                if(next->get()->kind == doc::Kind::text)
+                {
+                    auto* next_text = dynamic_cast<doc::Text*>(next->get());
+                    MRDOCS_ASSERT(next_text);
+                    text->string.append(next_text->string);
+                    it = block_->children.erase(next);
+                    continue;
+                }
+            }
+        }
+        ++it;
+    }
+
+    // Parse any Text nodes for styled text
+    for (auto it = block_->children.begin(); it != block_->children.end();)
+    {
+        MRDOCS_ASSERT(it->get());
+        if (it->get()->kind == doc::Kind::text)
+        {
+            auto* text = dynamic_cast<doc::Text*>(it->get());
+            auto styledText = parseStyled(text->string);
+            std::size_t const offset = std::distance(block_->children.begin(), it);
+            std::size_t const n = styledText.size();
+            block_->children.erase(it);
+            block_->children.insert(
+                block_->children.begin() + offset,
+                std::make_move_iterator(styledText.begin()),
+                std::make_move_iterator(styledText.end()));
+            it = block_->children.begin() + offset + n;
+        }
+        else
+        {
+            ++it;
+        }
+    }
 }
 
 //------------------------------------------------
@@ -462,16 +624,18 @@ visitTextComment(
     // If this is the first text comment in the
     // paragraph then remove all the leading space.
     // Otherwise, just remove the trailing space.
-    if(block_->children.empty())
+    if (block_->children.empty())
+    {
         s = s.ltrim();
-    else
-        s = s.rtrim();
+    }
 
     // Only insert non-empty text nodes
     if(! s.empty())
+    {
         emplaceText<doc::Text>(
             C->hasTrailingNewline(),
             ensureUTF8(s.str()));
+    }
 }
 
 Expected<JavadocVisitor::TagComponents>

src/lib/Gen/adoc/DocVisitor.cpp

@@ -18,6 +18,7 @@
 #include <llvm/Support/raw_ostream.h>
 #include <mrdocs/Support/RangeFor.hpp>
 #include <mrdocs/Support/String.hpp>
+#include <ranges>
 
 namespace clang::mrdocs::adoc {
 
@@ -65,7 +66,7 @@ operator()(
         doc::visit(*it.value,
             [&]<class T>(T const& text)
             {
-                if constexpr(std::is_same_v<T, doc::Text>)
+                if constexpr(std::derived_from<T, doc::Text>)
                 {
                     if(! text.string.empty())
                     {
@@ -89,7 +90,7 @@ DocVisitor::
 operator()(
     doc::Heading const& I) const
 {
-    fmt::format_to(ins_, "\n=== {}\n", AdocEscape(I.string));
+    fmt::format_to(ins_, "\n=== {}\n\n", AdocEscape(I.string));
 }
 
 // Also handles doc::Brief
@@ -103,15 +104,23 @@ operator()(
     {
         return;
     }
-    bool non_empty = write(*children.front(), *this);
-    for(auto const& child : children.subspan(1))
+
+    std::size_t i = 0;
+    for (auto it = children.begin(); it != children.end(); ++it)
     {
-        if (non_empty)
+        auto& child = *it;
+        if (i == 0)
         {
-            dest_.push_back('\n');
+            child->string = ltrim(child->string);
         }
-        non_empty = write(*child, *this);
+        if (i == children.size() - 1)
+        {
+            child->string = rtrim(child->string);
+        }
+        write(*child, *this);
+        i = i + 1;
     }
+
     dest_.push_back('\n');
     dest_.push_back('\n');
 }
@@ -188,37 +197,32 @@ void
 DocVisitor::
 operator()(doc::Text const& I) const
 {
-    // Asciidoc text must not have leading
-    // else they can be rendered up as code.
-    std::string_view s = trim(I.string);
-    // Render empty lines as paragraph delimiters.
-    if (s.empty())
+    if (I.string.empty())
+    {
+        dest_.append("\n\n");
+    } else
     {
-        s = "\n";
+        dest_.append(AdocEscape(I.string));
     }
-    dest_.append(AdocEscape(s));
 }
 
 void
 DocVisitor::
 operator()(doc::Styled const& I) const
 {
-    // VFALCO We need to apply Asciidoc escaping
-    // depending on the contents of the string.
-    std::string_view s = trim(I.string);
     switch(I.style)
     {
     case doc::Style::none:
-        dest_.append(s);
+        dest_.append(AdocEscape(I.string));
         break;
     case doc::Style::bold:
-        fmt::format_to(std::back_inserter(dest_), "*{}*", s);
+        fmt::format_to(std::back_inserter(dest_), "*{}*", AdocEscape(I.string));
         break;
     case doc::Style::mono:
-        fmt::format_to(std::back_inserter(dest_), "`{}`", s);
+        fmt::format_to(std::back_inserter(dest_), "`{}`", AdocEscape(I.string));
         break;
     case doc::Style::italic:
-        fmt::format_to(std::back_inserter(dest_), "_{}_", s);
+        fmt::format_to(std::back_inserter(dest_), "_{}_", AdocEscape(I.string));
         break;
     default:
         MRDOCS_UNREACHABLE();