Merge pull request #1611 from samcv/decodestream

Add support to Encoding::Decoder and Buf.decode to use replacements
rakudo · Mar 12, 2018 · 015d5a1 · 015d5a1
2 parents 71012be + ea92f55
commit 015d5a1
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 3 deletions.
diff --git a/src/core/Buf.pm6 b/src/core/Buf.pm6
@@ -119,10 +119,30 @@ my role Blob[::T = uint8] does Positional[T] does Stringy is repr('VMArray') is
     multi method decode(Blob:D:) {
         nqp::p6box_s(nqp::decode(self, 'utf8'))
     }
-    multi method decode(Blob:D: $encoding) {
+#?if moar
+    multi method decode(Blob:D: $encoding, Str :$replacement!, Bool:D :$strict = False) {
+        nqp::p6box_s(
+          nqp::decoderepconf(self,
+            Rakudo::Internals.NORMALIZE_ENCODING($encoding),
+            $replacement.defined ?? $replacement !! nqp::null_s(),
+            $strict ?? 0 !! 1))
+    }
+    multi method decode(Blob:D: $encoding, Bool:D :$strict = False) {
+        nqp::p6box_s(
+          nqp::decodeconf(self,
+            Rakudo::Internals.NORMALIZE_ENCODING($encoding),
+            $strict ?? 0 !! 1))
+    }
+#?endif
+#?if !moar
+    multi method decode(Blob:D: $encoding, Bool:D :$strict = False) {
         nqp::p6box_s(
           nqp::decode(self, Rakudo::Internals.NORMALIZE_ENCODING($encoding)))
     }
+    multi method decode(Blob:D: $encoding, Str:D :$replacement!, Bool:D :$strict = False) {
+        X::NYI.new(:feature<decode-with-replacement>).throw
+    }
+#?endif
 
     multi method list(Blob:D:) {
         Seq.new(class :: does Rakudo::Iterator::Blobby {

diff --git a/src/core/Encoding/Decoder/Builtin.pm6 b/src/core/Encoding/Decoder/Builtin.pm6
@@ -1,7 +1,18 @@
 my class Encoding::Decoder::Builtin is repr('Decoder') does Encoding::Decoder {
-    method new(str $encoding, :$translate-nl) {
+    method new(str $encoding, :$translate-nl, Str :$replacement, Bool:D :$strict = False) {
         nqp::decoderconfigure(nqp::create(self), $encoding,
-            $translate-nl ?? nqp::hash('translate_newlines', 1) !! nqp::null())
+            nqp::hash(
+                'translate_newlines', $translate-nl ?? 1 !! 0,
+                'replacement', $replacement.defined ?? nqp::unbox_s($replacement) !! nqp::null(),
+                'config', $strict ?? 0 !! 1
+                # Config set to 0 uses the decoder's new default, which is strict
+                # decoding. Setting to 1 uses the 6.c specced functionality where
+                # unmapped codepoints will still decode, e.g. codepoint 129, which
+                # in windows-1252 does not exist.
+
+                # In 6.d, 'config' will default to 0
+            )
+        )
     }
 
     method add-bytes(Blob:D $bytes --> Nil) {