RakuAST: make =for foo and =foo grab trailing whitespace

Sometimes extra trailing whitespace can be significant for rendering
rakudoc.  But =for foo  and  =foo  were only taking the first newline.
This is now fixed, with necessary changes to legacy podification and
the POC text rendering.

lib/RakuDoc/To/Text.rakumod

@@ -70,13 +70,7 @@ my multi sub rakudoc2text(RakuAST::Node:D $ast --> Str:D) {
 
 # the general handler, with specific sub-actions
 my multi sub rakudoc2text(RakuAST::Doc::Block:D $ast --> Str:D) {
-    state $needs-leading-nl;
-
-    my str $type   = $ast.type;
-    my str $prefix = $needs-leading-nl && $type ne 'item' ?? "\n" !! '';
-    $needs-leading-nl = $type eq 'item';
-
-    $prefix ~ do given $type {
+    given $ast.type {
         when 'code'          { code2text($ast)    }
         when 'comment'       { ''                 }
         when 'config'        { ''                 }
@@ -200,8 +194,8 @@ my sub code2text(RakuAST::Doc::Block:D $ast --> Str:D) {
 
 # handle =head
 my sub heading2text(RakuAST::Doc::Block:D $ast --> Str:D) {
-    my str $text  = textify($ast);
-    $text ~= ('-' x $text.chars - 1) ~ "\n";
+    my str $text = textify($ast).trim-trailing;
+    $text = $text ~ "\n" ~ ('-' x $text.chars) ~ "\n";
 
     my int $level = $ast.level.Int;
     $text.indent($level > 2 ?? 4 !! ($level - 1) * 2)

src/Raku/Grammar.nqp

@@ -3694,7 +3694,7 @@ if $*COMPILING_CORE_SETTING {
         <.doc-newline>
 
         # and any following lines as well
-        $<lines>=[^^ \h* [ <-[=\n]> | '=' ** 2..* ] \N* \n? ]*
+        $<lines>=[[^^ \h* [ <-[=\n]> | '=' ** 2..* ] \N* \n? ]* \n*]
     }
 
     token doc-block:sym<config> {
@@ -3727,7 +3727,7 @@ if $*COMPILING_CORE_SETTING {
         [ [ \h+ $<header>=[\N+ \n?]? ] | <doc-newline> ]
 
         # and any following lines as well
-        $<lines>=[^^ \h* [ <-[=\n]> | '=' ** 2..* ] \N* \n? ]*
+        $<lines>=[[^^ \h* [ <-[=\n]> | '=' ** 2..* ] \N* \n? ]* \n*]
     }
 
     token doc-block:sym<lines> {

src/core.c/RakuAST/LegacyPodify.pm6

@@ -15,15 +15,16 @@ class RakuAST::LegacyPodify {
     my sub sanitize(str $string, :$add-space --> Str:D) {
         return ' ' if $string eq "\n";
 
+        # work with integers instead of characters for speed
         nqp::strtocodes($string,nqp::const::NORMALIZE_NFC,my int32 @input);
-        my int $end = nqp::elems(@input);
-        return '' unless $end;
 
         # remove any trailing newlines
+        my int $elems = nqp::elems(@input);
         nqp::while(
-          $end && nqp::iseq_i(nqp::atpos_i(@input,--$end),$nl),
+          $elems && nqp::iseq_i(nqp::atpos_i(@input,--$elems),$nl),
           nqp::pop_i(@input)
         );
+        return '' unless $elems = nqp::elems(@input);
 
         my int32 @output;
         my int32 $curr;
@@ -35,15 +36,15 @@ class RakuAST::LegacyPodify {
         # normal characters, and collapse all other consecutive whitespace
         # into a single space character
         nqp::while(
-          nqp::isle_i(++$i,$end),
+          nqp::islt_i(++$i,$elems),
           nqp::if(                                    # for all codes
             nqp::iseq_i(($curr = nqp::atpos_i(@input,$i)),$nbsp)
               || nqp::iseq_i($curr,$nnbsp)
               || nqp::iseq_i($curr,$wj)
               || nqp::iseq_i($curr,$zwnbsp),
             nqp::push_i(@output,$prev = $curr),       # non-breaking whitespace
             nqp::if(                                  # not nb whitespace
-              nqp::iseq_s(($prop=nqp::getuniprop_str($curr,$gcprop)),'Zs')
+              nqp::iseq_s(($prop = nqp::getuniprop_str($curr,$gcprop)),'Zs')
                 || nqp::iseq_s($prop,'Cf')
                 || nqp::iseq_s($prop,'Cc'),
               nqp::if(                                # all other whitespace
@@ -57,7 +58,6 @@ class RakuAST::LegacyPodify {
 
         # add a space if there is something and were asked to add one
         @output.push($space) if $add-space && nqp::elems(@output);
-
         nqp::strfromcodes(@output)
     }
 
@@ -238,7 +238,9 @@ class RakuAST::LegacyPodify {
               # from here on without level
               !! $type eq 'comment'
                 ?? Pod::Block::Comment.new(
-                     :$config, :contents([$ast.paragraphs.head])
+                     :$config, :contents([
+                       $ast.paragraphs.head.trim-trailing ~ "\n"
+                     ])
                    )
                 !! $type eq 'config' && $ast.abbreviated
                   ?? Pod::Config.new(
@@ -312,6 +314,10 @@ class RakuAST::LegacyPodify {
             ).Slip
         });
 
+        # only keep one "\n" at end
+        @contents.pop while @contents.tail eq "\n";
+        @contents.push("\n");
+
         ::("Pod::Block::$type.tc()").new: :@contents, :config($ast.config)
     }