Switch to CESU-8 for encoding

dukpy/evaljs.py

@@ -3,6 +3,10 @@
 import os
 import logging
 
+# Duktape uses CESU-8 encoding, so we need support for it.
+# see https://github.com/svaarala/duktape/issues/504#issuecomment-168336246
+from mutf8 import encode_modified_utf8, decode_modified_utf8
+
 from dukpy.module_loader import JSModuleLoader
 from . import _dukpy
 
@@ -45,20 +49,20 @@ def evaljs(self, code, **kwargs):
 
         Returns the last object on javascript stack.
         """
-        jsvars = json.dumps(kwargs, ensure_ascii=False)
+        jsvars = json.dumps(kwargs)
         jscode = self._adapt_code(code)
 
         if not isinstance(jscode, bytes):
-            jscode = jscode.encode('utf-8')
+            jscode = encode_modified_utf8(jscode)
 
         if not isinstance(jsvars, bytes):
-            jsvars = jsvars.encode('utf-8')
+            jsvars = encode_modified_utf8(jsvars)
 
         res = _dukpy.eval_string(self, jscode, jsvars)
         if res is None:
             return None
 
-        return json.loads(res.decode('utf-8'))
+        return json.loads(decode_modified_utf8(res))
 
     def export_function(self, name, func):
         """Exports a python function to the javascript layer with the given name.
@@ -70,18 +74,18 @@ def export_function(self, name, func):
         self._funcs[name] = func
 
     def _check_exported_function_exists(self, func):
-        func = func.decode('utf-8')
+        func = func.decode('ascii')
         return func in self._funcs
 
     def _call_python(self, func, json_args):
         # Arguments came in reverse order from JS
-        func = func.decode('utf-8')
-        json_args = json_args.decode('utf-8')
+        func = func.decode('ascii')
+        json_args = decode_modified_utf8(json_args)
 
         args = list(reversed(json.loads(json_args)))
         ret = self._funcs[func](*args)
         if ret is not None:
-            return json.dumps(ret).encode('utf-8')
+            return encode_modified_utf8(json.dumps(ret))
 
     def _init_process(self):
         self.evaljs("process = {}; process.env = dukpy.environ", environ=dict(os.environ))

setup.py

@@ -12,7 +12,7 @@
 except IOError:
     README = ''
 
-INSTALL_REQUIRES = []
+INSTALL_REQUIRES = ["mutf8"]
 if py_version == (2, 6):
     INSTALL_REQUIRES.append('argparse')
 

src/pyduktape.c

@@ -1,11 +1,13 @@
 #include <stdlib.h>
 #include <string.h>
+#define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include "duktape.h"
 #include "duk_v1_compat.h"
 #include "duk_module_duktape.h"
 #include "_support.h"
 
+#include <stdio.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -32,13 +34,16 @@ static PyObject *DukPy_eval_string(PyObject *self, PyObject *args) {
     PyObject *pyctx;
     duk_context *ctx;
     const char *command;
+    size_t command_len;
     const char *vars;
+    size_t vars_len;
     int res;
     duk_int_t rc;
     const char *output;
     PyObject *result;
 
-    if (!PyArg_ParseTuple(args, CONDITIONAL_PY3("Oyy", "Oss"), &interpreter, &command, &vars))
+    if (!PyArg_ParseTuple(args, CONDITIONAL_PY3("Oy#y#", "Os#s#"), &interpreter,
+                          &command, &command_len, &vars, &vars_len))
         return NULL;
 
     pyctx = PyObject_GetAttrString(interpreter, "_ctx");
@@ -64,7 +69,7 @@ static PyObject *DukPy_eval_string(PyObject *self, PyObject *args) {
     duk_pop(ctx);
 
     /* Make passed arguments available as the dukpy global object */
-    duk_push_string(ctx, vars);
+    duk_push_lstring(ctx, vars, vars_len);
     duk_json_decode(ctx, -1);
     duk_put_global_string(ctx, "dukpy");
 
@@ -76,7 +81,7 @@ static PyObject *DukPy_eval_string(PyObject *self, PyObject *args) {
     duk_push_c_function(ctx, require_set_module_id, 2);
     duk_put_global_string(ctx, "_require_set_module_id");
 
-    res = duk_peval_string(ctx, command);
+    res = duk_peval_lstring(ctx, command, command_len);
     if (res != 0) {
         duk_get_prop_string(ctx, -1, "stack");
         PyErr_SetString(DukPyError, duk_safe_to_string(ctx, -1));
@@ -93,7 +98,22 @@ static PyObject *DukPy_eval_string(PyObject *self, PyObject *args) {
         return NULL;
     }
 
+    /* In some cases the JSON encoding will emit null because 
+       the object was valid but had no properties that could be
+       represented as JSON, in this case let's return an empty object */
+    if (duk_is_null(ctx, -1)) {
+        duk_pop(ctx);
+        duk_push_string(ctx, "{}");
+    }
+
     output = duk_get_string(ctx, -1);
+    if (output == NULL) {
+        PyErr_SetString(DukPyError, "Invalid Result Value");
+        duk_pop(ctx);
+        Py_XDECREF(pyctx);
+        return NULL;
+    }
+
     result = Py_BuildValue(CONDITIONAL_PY3("y", "s"), output);
     duk_pop(ctx);
     Py_XDECREF(pyctx);

tests/test_evaljs.py

@@ -39,6 +39,12 @@ def test_unicode_emoji(self):
         s3 = dukpy.evaljs("dukpy.c + '華'", c="🏠")
         assert s3 == '🏠華'
 
+    def test_unicode_emoji_code(self):
+        dukpy.evaljs("call_python('dukpy.log.info', dukpy.c, '🏠')", c="🏠")
+
+        s3 = dukpy.evaljs("dukpy.c + '🏠'", c="🏠")
+        assert s3 == '🏠🏠'
+
     def test_eval_files(self):
         testfile = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'test.js')
         with open(testfile) as f: