RakuAST: refactor Markup.meta handling

It was decided that the meta of a markup *could* contain Markup
objects as well.  This commit ensures this is now handled that way.
However, this has a number of ramifications:

- The interpretation of Markup.meta is now entirely up to the renderer
  So there is no splitting on ; or , anymore: any rakudoc (or other)
  semantics need to be applied by the renderer.
- Since there is no post-processing of the meta anymore, it doesn't
  make sense to keep the $!separator attribute around, so that is
  removed.
- Tests / deparsing / rakufication / stringification were adapted
  to reflect the absence of this attribute.

src/Raku/ast/doc-block.rakumod

@@ -192,11 +192,10 @@ class RakuAST::Doc::Markup
     has str  $.letter;     # the letter of Markup: A..Z
     has str  $.opener;     # opening sequence: < << «
     has str  $.closer;     # closing sequence: > >> »
-    has str  $.separator;  # separator inside meta: for deparsing mainly
     has List $!atoms;      # any contents of this markup (str | nested markup)
     has List $!meta;       # any meta info (e.g. url | lemma | original value)
 
-    method new(:$letter!, :$opener, :$closer, :$atoms, :$meta, :$separator) {
+    method new(:$letter!, :$opener, :$closer, :$atoms, :$meta) {
         my $obj := nqp::create(self);
         nqp::bindattr_s($obj, RakuAST::Doc::Markup, '$!letter',
           $letter // nqp::die("Must specify a letter"));
@@ -207,7 +206,6 @@ class RakuAST::Doc::Markup
 
         $obj.set-atoms($atoms);
         $obj.set-meta($meta);
-        $obj.set-separator($separator);
         $obj
     }
 
@@ -239,9 +237,4 @@ class RakuAST::Doc::Markup
     }
     method add-meta($meta) { nqp::push($!meta, $meta) }
     method meta() { self.IMPL-WRAP-LIST($!meta) }
-
-    method set-separator($separator) {
-        nqp::bindattr_s(self, RakuAST::Doc::Markup, '$!separator',
-          $separator // "");
-    }
 }

src/core.c/RakuAST/Fixups.pm6

@@ -329,40 +329,58 @@ augment class RakuAST::Doc::Markup {
         $string
     }
 
-    # extract any meta information from the last atom
-    method !extract-meta() {
-        my @atoms       = self.atoms;
-        my $last       := @atoms.tail;
-        if nqp::istype($last,Str) {
-            my ($str,$meta) = @atoms.tail.split('|', 2);
-            if $meta {
-                $str ?? ($last = $str) !! @atoms.pop;
-                self.set-atoms(@atoms.List);
-                $meta
+    # Extract any meta information from the atoms, perform the expected
+    # flattening of 'C', 'V' and letterless markup, and set that in the
+    # meta information of the given markup
+    method !extract-meta(--> Nil) {
+        my @atoms;
+        my @meta;
+        for self.atoms {
+            if nqp::istype($_,RakuAST::Doc::Markup) {
+                if @meta {
+                    my str $letter = .letter;
+                    # flatten verbatim markup here
+                    if $letter eq "" {
+                        my $atom := .opener ~ .atoms ~ .closer;
+                        @meta.push: nqp::istype(@meta.tail,Str)
+                          ?? @meta.pop ~ $atom
+                          !! $atom;
+                    }
+                    else {
+                        .set-atoms(.atoms.join) if $letter eq 'C' | 'V';
+                        @meta.push($_);
+                    }
+                }
+                else {
+                    @atoms.push($_);
+                }
+            }
+
+            # it's a string
+            elsif @meta {
+                @meta.push: nqp::istype(@meta.tail,RakuAST::Doc::Markup)
+                  ?? $_
+                  !! @meta.pop ~ $_;
             }
             else {
-                Nil
+                my ($before, $after) = nqp::hllize($_).split("|", 2);
+                @atoms.push($before) if $before;
+                @meta.push($_)     with $after;
             }
         }
 
-        # missing | while there *are* atoms
-        else {
-            Nil
+        if @meta {
+            self.set-atoms(@atoms);
+            @meta.shift if @meta > 1 && !@meta.head;  # empty leading string
+            self.set-meta(@meta);
         }
     }
 
     # set up meta info from the last atom as appropriate
     method check-meta(RakuAST::Doc::Markup:D:) {
-        my str $letter = self.letter;
-        if $letter eq 'L' {
-            self.set-meta($_) with self!extract-meta;
-        }
-        elsif $letter eq 'D' | 'M' | 'X' {
-            if self!extract-meta -> $meta {
-                self.add-meta(.trim.split(',')>>.trim.List)
-                  for $meta.split(';');
-                self.set-separator(';');
-            }
+        my str $letter = $!letter;
+        if $letter eq 'L' | 'D' | 'M' | 'X' {
+            self!extract-meta;
         }
         elsif $letter eq 'E' {
             my @atoms = self.atoms;
@@ -404,7 +422,7 @@ augment class RakuAST::Doc::Markup {
             $!letter eq 'E'
               ?? @parts.pop # stringification so far is incorrect
               !! @parts.push('|');
-            @parts.push: @meta.join($!separator)
+            @parts.push: @meta.join
         }
 
         @parts.push: $!closer if $container;
@@ -468,7 +486,7 @@ augment class RakuAST::Doc::Markup {
                 if self.meta -> @meta {
                     @parts.push: '|';
                     @parts.push:
-                      @meta.map({ $_.join(", ") }).join(self.separator ~ ' ');
+                      @meta.map({ $_.join(", ") }).join(' ');
                 }
             }
         }

src/core.c/RakuAST/Raku.pm6

@@ -404,7 +404,7 @@ augment class RakuAST::Node {
     }
 
     multi method raku(RakuAST::Doc::Markup:D: --> Str:D) {
-        self!nameds: <letter opener closer atoms meta separator>
+        self!nameds: <letter opener closer atoms meta>
     }
 
     multi method raku(RakuAST::Doc::Paragraph:D: --> Str:D) {

t/12-rakuast/doc-markup.rakutest

@@ -70,10 +70,8 @@ subtest 'index entry with multiple lemmas' => {
           letter => $letter,
           atoms  => ("hash",),
           meta  => (
-            "hashes, definition of",
-            "associative arrays",
+            "hashes, definition of; associative arrays",
           ),
-          separator => ';',
         );
         for 'Str', $deparsed, 'Raku', $raku.EVAL.DEPARSE -> $type, $it {
             is-deeply $it,