[js] Basic implementation of new encoding ops

Author: pmurias

src/vm/js/Operations.nqp

@@ -541,7 +541,13 @@ class QAST::OperationsJS {
 
     add_simple_op('encode', $T_OBJ, [$T_STR, $T_STR, $T_OBJ], :side_effects);
     add_simple_op('encoderep', $T_OBJ, [$T_STR, $T_STR, $T_STR, $T_OBJ], :side_effects);
+
+    add_simple_op('encodeconf', $T_OBJ, [$T_STR, $T_STR, $T_OBJ, $T_INT], :side_effects);
+    add_simple_op('encoderepconf', $T_OBJ, [$T_STR, $T_STR, $T_STR, $T_OBJ, $T_INT], :side_effects);
+
     add_simple_op('decode', $T_STR, [$T_OBJ, $T_STR]);
+    add_simple_op('decodeconf', $T_STR, [$T_OBJ, $T_STR, $T_INT]);
+    add_simple_op('decoderepconf', $T_STR, [$T_OBJ, $T_STR, $T_STR, $T_INT]);
 
     add_simple_op('gethostname', $T_STR, [$T_STR]);
 

src/vm/js/nqp-runtime/codecs.js

@@ -22,23 +22,35 @@ class SingleByteCodec {
     this.encodeBuf = new Buffer(65536);
     this.encodeBuf.fill(0);
 
+    this.encodeBufPermissive = new Buffer(65536);
+    this.encodeBufPermissive.fill(0);
+
     // stored separately so that we can have a unmapped flag in encodeBuf
     this.zero = codes.charCodeAt(0);
     for (let i = 1; i < codes.length; i++) {
-      this.encodeBuf[codes.charCodeAt(i)] = i;
+      if (codes[i] === '�') {
+        this.encodeBufPermissive[i] = i;
+      } else {
+        this.encodeBufPermissive[codes.charCodeAt(i)] = i;
+        this.encodeBuf[codes.charCodeAt(i)] = i;
+      }
     }
 
     this.decodeBuf = Buffer.from(codes, 'ucs2');
+    this.decodeBufPermissive = Buffer.from(codes.replace(/�/g, function(match, offset, string) {
+      return String.fromCharCode(offset);
+    }), 'ucs2');
   }
 
-  encode(str) {
+  encode(str, permissive) {
     const buf = new Buffer(str.length);
+    const encodeBuf = permissive ? this.encodeBufPermissive : this.encodeBuf;
     for (let i = 0; i < str.length; i++) {
       const unit = str.charCodeAt(i);
       if (unit === this.zero) {
         buf[i] = 0;
       } else {
-        const encoded = this.encodeBuf[unit];
+        const encoded = encodeBuf[unit];
         if (encoded === 0) {
           throw new NQPException('Error encoding ' + this.name + ' string: could not encode codepoint ' + unit);
         } else {
@@ -50,7 +62,9 @@ class SingleByteCodec {
     return buf;
   }
 
-  encodeWithReplacement(str, replacement) {
+  encodeWithReplacement(str, replacement, permissive) {
+    const encodeBuf = permissive ? this.encodeBufPermissive : this.encodeBuf;
+
     const replacementBuffer = this.encode(replacement);
 
     let replacementCount = 0;
@@ -60,7 +74,7 @@ class SingleByteCodec {
         i++;
         replacementCount++;
       } else {
-        if (code !== this.zero && this.encodeBuf[code] === 0) {
+        if (code !== this.zero && encodeBuf[code] === 0) {
           replacementCount++;
         }
       }
@@ -75,7 +89,7 @@ class SingleByteCodec {
       if (unit === this.zero) {
         buf[i] = 0;
       } else {
-        const encoded = this.encodeBuf[unit];
+        const encoded = encodeBuf[unit];
         if (encoded === 0) {
           offset += replacementBuffer.copy(buf, offset);
         } else {
@@ -91,8 +105,8 @@ class SingleByteCodec {
     return buf;
   }
 
-  decode(buf) {
-    const decodeBuf = this.decodeBuf;
+  decode(buf, permissive) {
+    const decodeBuf = permissive ? this.decodeBufPermissive : this.decodeBuf;
     const newBuf = new Buffer(buf.length*2);
     let idx1 = 0;
     let idx2 = 0;
@@ -103,6 +117,10 @@ class SingleByteCodec {
     }
     return newBuf.toString('ucs2');
   }
+
+  decodeWithReplacement(buf, replacement, permissive) {
+    return this.decode(buf, permissive).replace(/�/g, replacement);
+  }
 };
 
 function withASCII(codes) {
@@ -114,7 +132,8 @@ function withASCII(codes) {
   return codes;
 }
 
-const windows1252 = new SingleByteCodec('Windows-1252', withASCII('€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ'));
+
+const windows1252 = new SingleByteCodec('Windows-1252', withASCII('€�‚ƒ„…†‡ˆ‰Š‹Œ�Ž��‘’“”•–—˜™š›œ�žŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ'));
 
 const latin1 = new SingleByteCodec('Latin-1', withASCII('€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ'));
 

src/vm/js/nqp-runtime/core.js

@@ -971,7 +971,7 @@ function writeBuffer(highLevel, lowLevel) {
   }
 }
 
-op.encode = function(str, encoding_, output) {
+op.encodeconf = function(str, encoding_, output, permissive) {
   if (output.array.length) {
     throw new NQPException('encode requires an empty array');
   }
@@ -981,7 +981,7 @@ op.encode = function(str, encoding_, output) {
   let buffer;
 
   if (encoding in codecs) {
-    buffer = codecs[encoding].encode(str);
+    buffer = codecs[encoding].encode(str, permissive);
   } else {
     buffer = new Buffer(str, encoding);
   }
@@ -992,13 +992,17 @@ op.encode = function(str, encoding_, output) {
   return output;
 };
 
-op.encoderep = function(str, encoding_, replacement, output) {
+op.encode = function(str, encoding, output) {
+  return op.encodeconf(str, encoding, output, 1);
+};
+
+op.encoderepconf = function(str, encoding_, replacement, output, permissive) {
   const encoding = renameEncoding(encoding_);
 
   let buffer;
 
   if (encoding in codecs) {
-    buffer = codecs[encoding].encodeWithReplacement(str, replacement);
+    buffer = codecs[encoding].encodeWithReplacement(str, replacement, permissive);
   } else {
     throw new NQPException('encoding unsupported in encoderep');
   }
@@ -1008,6 +1012,10 @@ op.encoderep = function(str, encoding_, replacement, output) {
   return output;
 };
 
+op.encoderep = function(str, encoding, replacement, output) {
+  return op.encoderepconf(str, encoding, replacement, output, 1);
+};
+
 function toRawBuffer(buf) {
   const elementSize = byteSize(buf);
   const isUnsigned = buf._STable.REPR.type._STable.REPR.isUnsigned;
@@ -1040,10 +1048,10 @@ function bufferDifference(a, b) {
   return a.length;
 }
 
-op.decode = function(buf, encoding) {
+op.decodeconf = function(buf, encoding, permissive) {
   let rawBuffer = toRawBuffer(buf);
   if (encoding === 'windows-1252' || encoding === 'utf8-c8') {
-    return codecs[encoding].decode(rawBuffer);
+    return codecs[encoding].decode(rawBuffer, permissive);
   } else if (encoding === 'utf8') {
     const decoded = rawBuffer.toString(renameEncoding(encoding));
     const reencoded = Buffer.from(decoded, renameEncoding(encoding));
@@ -1081,6 +1089,19 @@ op.decode = function(buf, encoding) {
   }
 };
 
+op.decode = function(buf, encoding) {
+  return op.decodeconf(buf, encoding, 1);
+};
+
+op.decoderepconf = function(buf, encoding, replacement, permissive) {
+  if (encoding === 'windows-1252') {
+    const rawBuffer = toRawBuffer(buf);
+    return codecs[encoding].decodeWithReplacement(rawBuffer, replacement, permissive);
+  } else {
+    return op.decodeconf(buf, encoding, permissive);
+  }
+};
+
 op.objprimspec = function(obj) {
   if (obj === Null) return 0;
   if (typeof obj === 'object') {