Make a pyproxy of an awaitable py object an awaitable js object (#1170)

docs/project/changelog.md

@@ -57,14 +57,18 @@
   raise a `KeyboardInterrupt` by writing to the interrupt buffer.
   [#1148](https://github.com/iodide-project/pyodide/pull/1148) and
   [#1173](https://github.com/iodide-project/pyodide/pull/1173)
-- A `JsProxy` of a Javascript `Promise` or other awaitable object is now a 
-  Python awaitable.
-  [#880](https://github.com/iodide-project/pyodide/pull/880)
 - Added a Python event loop to support asyncio by scheduling coroutines to run 
   as jobs on the browser event loop. This event loop is available by default and 
   automatically enabled by any relevant asyncio API, so for instance 
   `asyncio.ensure_future` works without any configuration.
   [#1158](https://github.com/iodide-project/pyodide/pull/1158)
+- A `PyProxy` of a Python coroutine or awaitable is now an awaitable javascript
+  object. Awaiting a coroutine will schedule it to run on the Python event loop
+  using `asyncio.ensure_future`.
+  [#1170](https://github.com/iodide-project/pyodide/pull/1170)
+- A `JsProxy` of a Javascript `Promise` or other awaitable object is now a 
+  Python awaitable.
+  [#880](https://github.com/iodide-project/pyodide/pull/880)
 - Made PyProxy of an iterable Python object an iterable Js object: defined the
   `[Symbol.iterator]` method, can be used like `for(let x of proxy)`.
   Made a PyProxy of a Python iterator an iterator: `proxy.next()` is
@@ -87,10 +91,6 @@
 - JsBoundMethod is now a subclass of JsProxy, which fixes nested attribute
   access and various other strange bugs.
   [#1124](https://github.com/iodide-project/pyodide/pull/1124)
-- In console.html: sync behavior, full stdout/stderr support, clean namespace,
-  bigger font, correct result representation, clean traceback
-  [#1125](https://github.com/iodide-project/pyodide/pull/1125) and
-  [#1141](https://github.com/iodide-project/pyodide/pull/1141)
 - Javascript functions imported like `from js import fetch` no longer trigger
   "invalid invocation" errors (issue
   [#461](https://github.com/iodide-project/pyodide/issues/461)) and
@@ -99,11 +99,15 @@
   [#1126](https://github.com/iodide-project/pyodide/pull/1126)
 - Javascript bound method calls now work correctly with keyword arguments.
   [#1138](https://github.com/iodide-project/pyodide/pull/1138)
+- In console.html: sync behavior, full stdout/stderr support, clean namespace,
+  bigger font, correct result representation, clean traceback
+  [#1125](https://github.com/iodide-project/pyodide/pull/1125) and
+  [#1141](https://github.com/iodide-project/pyodide/pull/1141)
 - Switched from ̀Jedi to rlcompleter for completion in
   `pyodide.console.InteractiveConsole` and so in `console.html`. This fixes
   some completion issues (see
   [#821](https://github.com/iodide-project/pyodide/issues/821) and
-  [#1160](https://github.com/iodide-project/pyodide/issues/821)
+  [#1160](https://github.com/iodide-project/pyodide/issues/1160)
 
 ## Version 0.16.1
 *December 25, 2020*

src/core/hiwire.c

@@ -322,6 +322,12 @@ EM_JS_REF(JsRef, hiwire_call, (JsRef idfunc, JsRef idargs), {
   return Module.hiwire.new_value(jsfunc(... jsargs));
 });
 
+EM_JS_REF(JsRef, hiwire_call_OneArg, (JsRef idfunc, JsRef idarg), {
+  let jsfunc = Module.hiwire.get_value(idfunc);
+  let jsarg = Module.hiwire.get_value(idarg);
+  return Module.hiwire.new_value(jsfunc(jsarg));
+});
+
 EM_JS_REF(JsRef,
           hiwire_call_bound,
           (JsRef idfunc, JsRef idthis, JsRef idargs),

src/core/hiwire.h

@@ -371,6 +371,9 @@ hiwire_dir(JsRef idobj);
 JsRef
 hiwire_call(JsRef idobj, JsRef idargs);
 
+JsRef
+hiwire_call_OneArg(JsRef idfunc, JsRef idarg);
+
 JsRef
 hiwire_call_bound(JsRef idfunc, JsRef idthis, JsRef idargs);
 

src/core/pyproxy.c

@@ -7,6 +7,12 @@
 #include "js2python.h"
 #include "python2js.h"
 
+_Py_IDENTIFIER(result);
+_Py_IDENTIFIER(ensure_future);
+_Py_IDENTIFIER(add_done_callback);
+
+static PyObject* asyncio;
+
 JsRef
 _pyproxy_repr(PyObject* pyobj)
 {
@@ -223,6 +229,176 @@ _pyproxy_destroy(PyObject* ptrobj)
   EM_ASM({ delete Module.PyProxies[$0]; }, ptrobj);
 }
 
+/**
+ * Test if a PyObject is awaitable.
+ * Uses _PyCoro_GetAwaitableIter like in the implementation of the GET_AWAITABLE
+ * opcode (see ceval.c). Unfortunately this is not a public API (see issue
+ * https://bugs.python.org/issue24510) so it could be a source of instability.
+ *
+ * :param pyobject: The Python object.
+ * :return: 1 if the python code "await obj" would succeed, 0 otherwise. Never
+ * fails.
+ */
+bool
+_pyproxy_is_awaitable(PyObject* pyobject)
+{
+  PyObject* awaitable = _PyCoro_GetAwaitableIter(pyobject);
+  PyErr_Clear();
+  bool result = awaitable != NULL;
+  Py_CLEAR(awaitable);
+  return result;
+}
+
+// clang-format off
+/**
+ * A simple Callable python object. Intended to be called with a single argument
+ * which is the future that was resolved.
+ */
+typedef struct {
+    PyObject_HEAD
+    /** Will call this function with the result if the future succeeded */
+    JsRef resolve_handle;
+    /** Will call this function with the error if the future succeeded */
+    JsRef reject_handle;
+} FutureDoneCallback;
+// clang-format on
+
+static void
+FutureDoneCallback_dealloc(FutureDoneCallback* self)
+{
+  hiwire_CLEAR(self->resolve_handle);
+  hiwire_CLEAR(self->reject_handle);
+  Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+/**
+ * Helper method: if the future resolved successfully, call resolve_handle on
+ * the result.
+ */
+int
+FutureDoneCallback_call_resolve(FutureDoneCallback* self, PyObject* result)
+{
+  bool success = false;
+  JsRef result_js = NULL;
+  JsRef output = NULL;
+  result_js = python2js(result);
+  output = hiwire_call_OneArg(self->resolve_handle, result_js);
+
+  success = true;
+finally:
+  hiwire_CLEAR(result_js);
+  hiwire_CLEAR(output);
+  return success ? 0 : -1;
+}
+
+/**
+ * Helper method: if the future threw an error, call reject_handle on a
+ * converted exception. The caller leaves the python error indicator set.
+ */
+int
+FutureDoneCallback_call_reject(FutureDoneCallback* self)
+{
+  bool success = false;
+  JsRef excval = NULL;
+  JsRef result = NULL;
+  // wrap_exception looks up the current exception and wraps it in a Js error.
+  excval = wrap_exception();
+  FAIL_IF_NULL(excval);
+  result = hiwire_call_OneArg(self->reject_handle, excval);
+
+  success = true;
+finally:
+  hiwire_CLEAR(excval);
+  hiwire_CLEAR(result);
+  return success ? 0 : -1;
+}
+
+/**
+ * Intended to be called with a single argument which is the future that was
+ * resolved. Resolves the promise as appropriate based on the result of the
+ * future.
+ */
+PyObject*
+FutureDoneCallback_call(FutureDoneCallback* self,
+                        PyObject* args,
+                        PyObject* kwargs)
+{
+  PyObject* fut;
+  if (!PyArg_UnpackTuple(args, "future_done_callback", 1, 1, &fut)) {
+    return NULL;
+  }
+  PyObject* result = _PyObject_CallMethodIdObjArgs(fut, &PyId_result, NULL);
+  int errcode;
+  if (result != NULL) {
+    errcode = FutureDoneCallback_call_resolve(self, result);
+  } else {
+    errcode = FutureDoneCallback_call_reject(self);
+  }
+  if (errcode == 0) {
+    Py_RETURN_NONE;
+  } else {
+    return NULL;
+  }
+}
+
+// clang-format off
+static PyTypeObject FutureDoneCallbackType = {
+    .tp_name = "FutureDoneCallback",
+    .tp_doc = "Callback for internal use to allow awaiting a future from javascript",
+    .tp_basicsize = sizeof(FutureDoneCallback),
+    .tp_itemsize = 0,
+    .tp_flags = Py_TPFLAGS_DEFAULT,
+    .tp_dealloc = (destructor) FutureDoneCallback_dealloc,
+    .tp_call = (ternaryfunc) FutureDoneCallback_call,
+};
+// clang-format on
+
+static PyObject*
+FutureDoneCallback_cnew(JsRef resolve_handle, JsRef reject_handle)
+{
+  FutureDoneCallback* self =
+    (FutureDoneCallback*)FutureDoneCallbackType.tp_alloc(
+      &FutureDoneCallbackType, 0);
+  self->resolve_handle = hiwire_incref(resolve_handle);
+  self->reject_handle = hiwire_incref(reject_handle);
+  return (PyObject*)self;
+}
+
+/**
+ * Intended to be called with a single argument which is the future that was
+ * resolved. Resolves the promise as appropriate based on the result of the
+ * future.
+ *
+ * :param pyobject: An awaitable python object
+ * :param resolve_handle: The resolve javascript method for a promise
+ * :param reject_handle: The reject javascript method for a promise
+ * :return: 0 on success, -1 on failure
+ */
+int
+_pyproxy_ensure_future(PyObject* pyobject,
+                       JsRef resolve_handle,
+                       JsRef reject_handle)
+{
+  bool success = false;
+  PyObject* future = NULL;
+  PyObject* callback = NULL;
+  PyObject* retval = NULL;
+  future =
+    _PyObject_CallMethodIdObjArgs(asyncio, &PyId_ensure_future, pyobject, NULL);
+  FAIL_IF_NULL(future);
+  callback = FutureDoneCallback_cnew(resolve_handle, reject_handle);
+  retval = _PyObject_CallMethodIdObjArgs(
+    future, &PyId_add_done_callback, callback, NULL);
+  FAIL_IF_NULL(retval);
+
+  success = true;
+finally:
+  Py_CLEAR(future);
+  Py_CLEAR(callback);
+  Py_CLEAR(retval);
+  return success ? 0 : -1;
+}
+
 EM_JS_REF(JsRef, pyproxy_new, (PyObject * ptrobj), {
   // Technically, this leaks memory, since we're holding on to a reference
   // to the proxy forever.  But we have that problem anyway since we don't
@@ -249,11 +425,15 @@ EM_JS_REF(JsRef, pyproxy_new, (PyObject * ptrobj), {
   }
   // clang-format on
   Module.PyProxies[ptrobj] = proxy;
+  let is_awaitable = __pyproxy_is_awaitable(ptrobj);
+  if (is_awaitable) {
+    Object.assign(target, Module.PyProxyAwaitableMethods);
+  }
 
   return Module.hiwire.new_value(proxy);
 });
 
-EM_JS(int, pyproxy_init, (), {
+EM_JS_NUM(int, pyproxy_init_js, (), {
   // clang-format off
   Module.PyProxies = {};
   function _getPtr(jsobj) {
@@ -314,7 +494,7 @@ EM_JS(int, pyproxy_init, (), {
     },
   };
 
-  // See: 
+  // See:
   // https://docs.python.org/3/c-api/iter.html
   // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols
   // This avoids allocating a PyProxy wrapper for the temporary iterator.
@@ -338,7 +518,7 @@ EM_JS(int, pyproxy_init, (), {
   Module.PyProxyIteratorMethods = {
     [Symbol.iterator] : function() {
       return this;
-    },    
+    },
     next : function(arg) {
       let idresult;
       // Note: arg is optional, if arg is not supplied, it will be undefined
@@ -350,7 +530,7 @@ EM_JS(int, pyproxy_init, (), {
         Module.fatal_error(e);
       } finally {
         Module.hiwire.decref(idarg);
-      }      
+      }
 
       let done = false;
       if(idresult === 0){
@@ -473,6 +653,61 @@ EM_JS(int, pyproxy_init, (), {
     },
   };
 
+  Module.PyProxyAwaitableMethods = {
+    _ensure_future : function(){
+      let resolve_handle_id = 0;
+      let reject_handle_id = 0;
+      let resolveHandle;
+      let rejectHandle;
+      let promise;
+      try {
+        promise = new Promise((resolve, reject) => {
+          resolveHandle = resolve;
+          rejectHandle = reject;
+        });
+        resolve_handle_id = Module.hiwire.new_value(resolveHandle);
+        reject_handle_id = Module.hiwire.new_value(rejectHandle);
+        let ptrobj = _getPtr(this);
+        let errcode = __pyproxy_ensure_future(ptrobj, resolve_handle_id, reject_handle_id);
+        if(errcode === -1){
+          _pythonexc2js();
+        }
+      } finally {
+        Module.hiwire.decref(resolve_handle_id);
+        Module.hiwire.decref(reject_handle_id);
+      }
+      return promise;
+    },
+    then : function(onFulfilled, onRejected){
+      let promise = this._ensure_future();
+      return promise.then(onFulfilled, onRejected);
+    },
+    catch : function(onRejected){
+      let promise = this._ensure_future();
+      return promise.catch(onRejected);
+    },
+    finally : function(onFinally){
+      let promise = this._ensure_future();
+      return promise.finally(onFinally);
+    }
+  };
+
   return 0;
 // clang-format on
 });
+
+int
+pyproxy_init()
+{
+  asyncio = PyImport_ImportModule("asyncio");
+  if (asyncio == NULL) {
+    return -1;
+  }
+  if (PyType_Ready(&FutureDoneCallbackType)) {
+    return -1;
+  }
+  if (pyproxy_init_js()) {
+    return -1;
+  }
+  return 0;
+}