Merge branch 'master' into post-release

src/QAST/Block.nqp

@@ -93,7 +93,9 @@ class QAST::Block is QAST::Node does QAST::Children {
     }
 
     method dump_extra_node_info() {
-        ":cuid($!cuid)"
-          ~ (nqp::chars(self.blocktype) ?? " :blocktype($!blocktype)" !! "");
+        my @extra;
+        @extra.push(":cuid($!cuid)") unless nqp::isnull_s($!cuid);
+        @extra.push(":blocktype($!blocktype)") unless nqp::chars(self.blocktype);
+        nqp::join(' ', @extra);
     }
 }

src/vm/js/RegexCompiler.nqp

@@ -123,9 +123,9 @@ class RegexCompiler {
     }
 
     method ignore_suffix(str $subtype) {
-        return 'i' if $subtype eq 'ignorecase';
-        return 'm' if $subtype eq 'ignoremark';
-        return 'im' if $subtype eq 'ignorecase+ignoremark';
+        return '_i' if $subtype eq 'ignorecase';
+        return '_m' if $subtype eq 'ignoremark';
+        return '_im' if $subtype eq 'ignorecase+ignoremark';
         return '';
     }
 
@@ -135,7 +135,7 @@ class RegexCompiler {
 
         if self.ignore_suffix($node.subtype) -> str $suffix {
             my str $offset := $*BLOCK.add_tmp;
-            return "$offset = nqp.literal_{$suffix}($!target, $!pos, $qconst);\n"
+            return "$offset = nqp.literal{$suffix}($!target, $!pos, $qconst);\n"
                 ~ "if ($offset === -1)  \{{self.fail}\} else \{{$!pos}+=$offset\}\n";
         }
 
@@ -169,19 +169,13 @@ class RegexCompiler {
     method enumcharlist($node) {
         my str $charlist := quote_string($node[0]);
 
-        if self.ignore_suffix($node.subtype) -> str $suffix {
-            my str $offset := $*BLOCK.add_tmp;
-            my str $negate := $node.negate ?? 'true' !! 'false';
-            return "$offset = nqp.enumcharlist_{$suffix}($negate, $!target, $!pos, $charlist);\n"
-                ~ "if ($offset === -1)  \{{self.fail}\} else \{{$!pos}+=$offset\}\n";
-        }
+        my str $suffix := self.ignore_suffix($node.subtype);
 
-        my str $testop := $node.negate ?? '!=' !! '==';
-
-        my str $end_of_string := ($node.negate && $node.subtype eq 'zerowidth') ?? "$!pos < $!target.length &&" !! "$!pos >= $!target.length ||";
-
-        "if ($end_of_string $charlist.indexOf($!target.substr($!pos,1)) $testop -1) \{{self.fail()}\}"
-        ~ ($node.subtype eq 'zerowidth' ?? '' !! "$!pos++;\n")
+        my str $offset := $*BLOCK.add_tmp;
+        my str $negate := $node.negate ?? 'true' !! 'false';
+        return "$offset = nqp.enumcharlist{$suffix}($negate, $!target, $!pos, $charlist, {$node.subtype eq 'zerowidth' ?? "true" !! "false"});\n"
+            ~ "if ($offset === -1)  \{{self.fail}\}"
+            ~ ($node.subtype eq 'zerowidth' ?? '' !! "else \{{$!pos}+=$offset\}\n");
     }
 
     method charrange($node) {
@@ -331,11 +325,8 @@ class RegexCompiler {
 
         if $node.name ne '.' {
             $code := $code ~ "if ({$node.negate ?? '' !! '!'}nqp.op.iscclass($cclass,$!target,$!pos)) \{{self.fail}\}\n";
-            if $node.name eq 'n' && $node.subtype ne 'zerowidth' {
-                $code := $code ~ "if ($!target.substr($!pos,2) == \"\\r\\n\") \{$!pos++\}\n";
-            }
         }
-        $code := $code ~ "$!pos++;\n" unless $node.subtype eq 'zerowidth';
+        $code := $code ~ "$!pos += nqp.nextGrapheme($!target, $!pos);\n" unless $node.subtype eq 'zerowidth';
         $code;
     }
 

src/vm/js/nqp-runtime/runtime.js

@@ -24,6 +24,7 @@ const NativeStrArg = exports.NativeStrArg = nativeArgs.NativeStrArg;
 
 const stripMarks = require('./strip-marks.js');
 const foldCase = require('fold-case');
+const graphemeBreaker = require('grapheme-breaker');
 
 const fs = require('fs');
 
@@ -662,6 +663,20 @@ exports.enumcharlist_m = function(negate, target, pos, charlist) {
   }
 }
 
+exports.enumcharlist = function(negate, target, pos, charlist, zerowidth) {
+  if (pos >= target.length) return (zerowidth && negate ? 0 : -1);
+  const found = charlist.indexOf(target.substr(pos,1)) != -1;
+  if (negate ? !found : found) {
+    return (graphemeBreaker.nextBreak(target, pos) - pos);
+  } else {
+    return -1;
+  }
+};
+
+exports.nextGrapheme = function(target, pos) {
+  return graphemeBreaker.nextBreak(target, pos) - pos;
+};
+
 exports.noNamed = function(_NAMED) {
   if (Object.keys(_NAMED) != 0) {
     throw new NQPException(`Unexpected named argument ${Object.keys(_NAMED)[0]} passed`);

src/vm/js/nqp-runtime/unicode-props.js

@@ -3,6 +3,9 @@ const xregexp = require('xregexp');
 const names = require('./unicode-data/names.js');
 const core = require('./core.js');
 
+//TODO - the regexes should be tweaked to match full graphemes instead
+const graphemeBreaker = require('grapheme-breaker');
+
 function mangled(name) {
   return name.toLowerCase(name).replace(/_/g, '');
 }
@@ -37,7 +40,7 @@ function matchClass(shouldMatch, category, negated) {
   return function(target, pos) {
     regexp.lastIndex = pos;
     if (regexp.test(target)) {
-      return regexp.lastIndex - pos;
+      return graphemeBreaker.nextBreak(target, pos) - pos;
     } else {
       return -1;
     }
@@ -88,7 +91,7 @@ function matchDerived(shouldMatch, match, avoid) {
   return function(target, pos) {
     regexp.lastIndex = pos;
     if (regexp.test(target)) {
-      return regexp.lastIndex - pos;
+      return graphemeBreaker.nextBreak(target, pos) - pos;
     } else {
       return -1;
     }
@@ -175,7 +178,7 @@ function matchRegex(shouldMatch, regexString) {
   return function(target, pos) {
     regexp.lastIndex = pos;
     if (regexp.test(target)) {
-      return regexp.lastIndex - pos;
+      return graphemeBreaker.nextBreak(target, pos) - pos;
     } else {
       return -1;
     }

t/nqp/005-comments.t

@@ -2,7 +2,7 @@
 
 # check comments
 
-say('1..8');
+say('1..16');
 
 #Comment preceding
 say("ok 1");
@@ -54,4 +54,44 @@ say("ok 7");
 
     say("ok 8");
 
+    =begin comment
+    this is indented pod with an unaligned =end comment
+      =end comment
+say("ok 9");
 
+
+    =begin comment
+    this is another indented pod with an unaligned =end comment
+  =end comment
+say("ok 10");
+
+# Parsing breaks down here: no errors are found
+# with the second =end and how it's interpreted.
+# seems to be a pod object of unknown type.
+# the second =begin seems to be ignored or
+# considered part of the comment.
+# may need a :nested config key
+    =begin comment
+    this is indented pod with an unaligned =end comment
+    =begin comment
+    this is a nested comment
+  =end comment
+    say("ok 11");
+
+    say("ok 12");
+  # the following doesn't trigger a panic:
+  =end comment
+
+say("ok 13");
+
+
+=begin comment
+this is indented pod with an unaligned =end comment
+ =begin comment
+ this is a nested comment
+ =end comment
+say("ok 14");
+say("ok 15");
+# the following triggers a panic if uncommented:
+#=end comment
+say("ok 16");

t/nqp/031-grammar.t

@@ -2,7 +2,7 @@
 
 # Test grammars and regexes
 
-plan(15);
+plan(18);
 
 grammar ABC {
     token TOP { ok ' ' <integer> }
@@ -13,6 +13,10 @@ grammar ABC {
     token a-or-b { <[ab]> }
 
     token not_a_or_b { ^ <- a-or-b>+ $ }
+
+    token only_integer { ^ \d $ }
+    token only_L { ^ <:L> $ }
+    token only_not_backslash { ^ <-[\]\\]> $ }
 }
 
 my $match := ABC.parse('not ok');
@@ -33,6 +37,16 @@ ok( $match<int-num> == 123, 'captured $<int-num>');
 ok(?ABC.parse('ccc', :rule<not_a_or_b> ), "<- name-with-hyphen> matches");
 ok(!ABC.parse('cac', :rule<not_a_or_b> ), "<- name-with-hyphen> doesn't match");
 
+if nqp::getcomp('nqp').backend.name eq 'jvm' {
+    skip('not yet fixed on the JVM', 3);
+} else {
+    ok( ?ABC.parse("7\x[308]", :rule<only_integer>), '\d matches a combining character');
+
+    ok( ?ABC.parse("a\c[COMBINING DIAERESIS]", :rule<only_L>), '<:L> matches a combining character');
+    ok( ?ABC.parse("a\c[COMBINING DIAERESIS]", :rule<only_not_backslash>), 'a charclass matches a combining character');
+}
+
+
 my %args;
 %args<arg1> := 123;
 %args<arg2> := 456;
@@ -44,8 +58,8 @@ grammar G {
         foo
     }
     method foo(*%args) {
-        ok(%args<arg1> == 678);
-        ok(%args<arg2> == 456);
+        ok(%args<arg1> == 678, 'correct named args passed to subrule 1/2');
+        ok(%args<arg2> == 456, 'correct named args passed to subrule 2/2');
         self.literal;
     }
 }

tools/build/MOAR_REVISION

@@ -1 +1 @@
-2017.12.1-34-g4a0a912
+2017.12.1-62-ga04d1099b