Add more tests for streaming decoder

Raku · Oct 10, 2020 · ff21c24 · ff21c24
1 parent 965dbc6
commit ff21c24
Showing 1 changed file with 98 additions and 1 deletion.
diff --git a/t/nqp/116-streaming-decoder.t b/t/nqp/116-streaming-decoder.t
@@ -1,4 +1,4 @@
-plan(55);
+plan(70);
 
 sub dies-ok(&code, $message) {
     my int $died := 0;
@@ -190,3 +190,100 @@ nqp::composetype($buf_type, nqp::hash('array', nqp::hash('type', uint8)));
     nqp::decoderaddbytes($dec, $emptybuf);
     ok(nqp::decoderempty($dec), 'Decoder is still empty after adding an empty buffer');
 }
+
+{
+    ## 'д' is 'CYRILLIC SMALL LETTER DE', (U+0434), UFT-8(hex) 0xD0 0xB4
+    my $testbuf1 := nqp::create($buf_type);
+    my $testbuf2 := nqp::create($buf_type);
+    my $dec := nqp::create(VMDecoder);
+    nqp::decoderconfigure($dec, 'utf8', nqp::hash());
+
+    ## decode complete code point
+    nqp::bindpos_i($testbuf1, 0, 0xD0);
+    nqp::bindpos_i($testbuf1, 1, 0xB4);
+    nqp::decoderaddbytes($dec, $testbuf1);
+    ok(nqp::decodertakeallchars($dec) eq 'д', 'Got valid code point back (bytes pushed together)');
+    nqp::bindpos_i($testbuf2, 0, 0xD0);
+    nqp::decoderaddbytes($dec, $testbuf2);
+    nqp::bindpos_i($testbuf2, 0, 0xB4);
+    nqp::decoderaddbytes($dec, $testbuf2);
+    ok(nqp::decodertakeallchars($dec) eq 'д', 'Got valid code point back (bytes pushed separately)');
+
+    ## push invalid code point (second byte of multibyte character must by larger than 0x7F)
+    nqp::bindpos_i($testbuf1, 0, 0xD0);
+    nqp::bindpos_i($testbuf1, 1, 0x7F);
+    nqp::decoderaddbytes($dec, $testbuf1);
+    dies-ok({ nqp::decodertakeallchars($dec) }, 'error with invalid code point (bytes pushed together)');
+    $dec := nqp::create(VMDecoder);
+    nqp::decoderconfigure($dec, 'utf8', nqp::hash());
+    nqp::bindpos_i($testbuf2, 0, 0xD0);
+    nqp::decoderaddbytes($dec, $testbuf2);
+    nqp::bindpos_i($testbuf2, 0, 0x7F);
+    nqp::decoderaddbytes($dec, $testbuf2);
+    dies-ok({ nqp::decodertakeallchars($dec) }, 'error with invalid code point (bytes pushed separately)');
+}
+
+{
+    my $testbuf := nqp::encode("два\n", 'utf8', nqp::create($buf_type));
+    my $dec := nqp::create(VMDecoder);
+    nqp::decoderconfigure($dec, 'utf8', nqp::hash());
+    nqp::decoderaddbytes($dec, nqp::slice($testbuf, 0, 0));
+    ok(nqp::decodertakeavailablechars($dec) eq '',
+        'No valid char after 1st byte (first byte of "д" not valid)');
+    nqp::decoderaddbytes($dec, nqp::slice($testbuf, 1, 1));
+    ok(nqp::decodertakeavailablechars($dec) eq '',
+        'No valid char after 2nd byte ("д" held by normalization)');
+    nqp::decoderaddbytes($dec, nqp::slice($testbuf, 2, 2));
+    ok(nqp::decodertakeavailablechars($dec) eq '',
+        'No valid char after 3rd byte ("д" held by normalization, first byte of "в" not valid');
+    nqp::decoderaddbytes($dec, nqp::slice($testbuf, 3, 3));
+    ok(nqp::decodertakeavailablechars($dec) eq 'д',
+        'Got "д" after 4th byte ("в" held by normalization)');
+    nqp::decoderaddbytes($dec, nqp::slice($testbuf, 4, 4));
+    ok(nqp::decodertakeavailablechars($dec) eq '',
+        'No valid char after 5th byte ("в" held by normalization, first byte of "а" not valid');
+    nqp::decoderaddbytes($dec, nqp::slice($testbuf, 5, 5));
+    ok(nqp::decodertakeavailablechars($dec) eq 'в',
+        'Got "в" after 6th byte ("а" held by normalization)');
+    nqp::decoderaddbytes($dec, nqp::slice($testbuf, 6, 6));
+    ok(nqp::decodertakeavailablechars($dec) eq "а\n",
+        'Got "а\n" after 7th byte');
+}
+
+{
+    my $testbuf := nqp::encode("подводка\n", 'utf8', nqp::create($buf_type));
+    my $dec := nqp::create(VMDecoder);
+    nqp::decoderconfigure($dec, 'utf8', nqp::hash());
+    my $bufpart1 := nqp::slice($testbuf, 0, 8);
+    nqp::decoderaddbytes($dec, $bufpart1);
+    ok(nqp::decodertakeavailablechars($dec) eq 'под',
+        'Got first three chars ("в" held by normalization, first byte of "о" not valid)');
+    my $bufpart2 := nqp::slice($testbuf, 9, 16);
+    nqp::decoderaddbytes($dec, $bufpart2);
+    ok(nqp::decodertakeavailablechars($dec) eq "водка\n",
+        'Got remaining chars, no complaint about invalid UTF-8 after more bytes are added');
+}
+
+{
+    my $testbuf := nqp::encode("подводка\n", 'utf8', nqp::create($buf_type));
+    my $dec := nqp::create(VMDecoder);
+    nqp::decoderconfigure($dec, 'utf8', nqp::hash());
+    my $bufpart1 := nqp::slice($testbuf, 0, 8);
+    nqp::decoderaddbytes($dec, $bufpart1);
+    my $bufpart2 := nqp::slice($testbuf, 9, 16);
+    nqp::decoderaddbytes($dec, $bufpart2);
+    ok(nqp::decodertakeavailablechars($dec) eq "подводка\n",
+        'Got complete string from decodertakeavailablechars (string pushed in two parts, split on multibyte char)');
+}
+
+{
+    my $testbuf := nqp::encode("подводка\n", 'utf8', nqp::create($buf_type));
+    my $dec := nqp::create(VMDecoder);
+    nqp::decoderconfigure($dec, 'utf8', nqp::hash());
+    my $bufpart1 := nqp::slice($testbuf, 0, 8);
+    nqp::decoderaddbytes($dec, $bufpart1);
+    my $bufpart2 := nqp::slice($testbuf, 9, 16);
+    nqp::decoderaddbytes($dec, $bufpart2);
+    ok(nqp::decodertakeallchars($dec) eq "подводка\n",
+        'Got complete string from decodertakeallchars (string pushed in two parts, split on multibyte char)');
+}