From ffc4b1cccda9e17a5db231674e35af330425ebdf Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 29 Mar 2023 18:25:01 +0000 Subject: [PATCH 01/73] feat(promise): add the `PromiseType` class inheriting from the PyType struct --- include/PromiseType.hh | 41 +++++++++++++++++++++++++++++++++++++++++ include/TypeEnum.hh | 1 + src/PromiseType.cc | 27 +++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 include/PromiseType.hh create mode 100644 src/PromiseType.cc diff --git a/include/PromiseType.hh b/include/PromiseType.hh new file mode 100644 index 00000000..d6d63df6 --- /dev/null +++ b/include/PromiseType.hh @@ -0,0 +1,41 @@ +/** + * @file PromiseType.hh + * @author Tom Tang (xmader@distributive.network) + * @brief Struct for representing Promises + * @version 0.1 + * @date 2023-03-29 + * + * @copyright Copyright (c) 2023 + * + */ + +#ifndef PythonMonkey_PromiseType_ +#define PythonMonkey_PromiseType_ + +#include "PyType.hh" +#include "TypeEnum.hh" + +#include +#include + +#include + +/** + * @brief This struct represents the JS Promise type in Python using our custom pythonmonkey.promise type. It inherits from the PyType struct + */ +struct PromiseType : public PyType { +public: + PromiseType(PyObject *object); + + /** + * @brief Construct a new PromiseType object from a JS::PromiseObject. + * + * @param cx - javascript context pointer + * @param promise - JS::PromiseObject pointer + */ + PromiseType(JSContext *cx, JS::Handle *promise); + + const TYPE returnType = TYPE::PYTHONMONKEY_PROMISE; +}; + +#endif \ No newline at end of file diff --git a/include/TypeEnum.hh b/include/TypeEnum.hh index 3fb38d11..899db23e 100644 --- a/include/TypeEnum.hh +++ b/include/TypeEnum.hh @@ -23,6 +23,7 @@ enum class TYPE { LIST, TUPLE, DATE, + PYTHONMONKEY_PROMISE, NONE, PYTHONMONKEY_NULL, }; diff --git a/src/PromiseType.cc b/src/PromiseType.cc new file mode 100644 index 00000000..03bd5d2a --- /dev/null +++ b/src/PromiseType.cc @@ -0,0 +1,27 @@ +/** + * @file PromiseType.cc + * @author Tom Tang (xmader@distributive.network) + * @brief Struct for representing Promises + * @version 0.1 + * @date 2023-03-29 + * + * @copyright Copyright (c) 2023 + * + */ + +#include "include/modules/pythonmonkey/pythonmonkey.hh" + +#include "include/PromiseType.hh" + +#include "include/PyType.hh" +#include "include/TypeEnum.hh" + +#include +#include + +#include + +PromiseType::PromiseType(PyObject *object) : PyType(object) {} + +PromiseType::PromiseType(JSContext *cx, JS::Handle *promise) {} + From ecaee4a25723eca0e47d56148b9bd52434ee4140 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 3 Apr 2023 14:32:55 +0000 Subject: [PATCH 02/73] feat(event-loop): use internal job queues --- src/modules/pythonmonkey/pythonmonkey.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index f05fa9de..66cd20d9 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -22,6 +22,7 @@ #include "include/StrType.hh" #include +#include #include #include #include @@ -194,6 +195,11 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) return NULL; } + if (!js::UseInternalJobQueues(GLOBAL_CX)) { + PyErr_SetString(SpiderMonkeyError, "Spidermonkey could not create the event-loop."); + return NULL; + } + if (!JS::InitSelfHostedCode(GLOBAL_CX)) { PyErr_SetString(SpiderMonkeyError, "Spidermonkey could not initialize self-hosted code."); return NULL; From c2ce56e242628a6274e256b88bf45aa2b12dd242 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 3 Apr 2023 17:58:26 +0000 Subject: [PATCH 03/73] feat(event-loop): provide our own job queue (not implemented yet) --- include/JobQueue.hh | 54 ++++++++++++++++++++ include/modules/pythonmonkey/pythonmonkey.hh | 2 + src/JobQueue.cc | 39 ++++++++++++++ src/modules/pythonmonkey/pythonmonkey.cc | 6 ++- 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 include/JobQueue.hh create mode 100644 src/JobQueue.cc diff --git a/include/JobQueue.hh b/include/JobQueue.hh new file mode 100644 index 00000000..ce530a40 --- /dev/null +++ b/include/JobQueue.hh @@ -0,0 +1,54 @@ +/** + * @file JobQueue.hh + * @author Tom Tang (xmader@distributive.network) + * @brief + * @version 0.1 + * @date 2023-04-03 + * + * @copyright Copyright (c) 2023 + * + */ + +#ifndef PythonMonkey_JobQueue_ +#define PythonMonkey_JobQueue_ + +#include +#include + +class JobQueue : public JS::JobQueue { +// +// JS::JobQueue methods. +// see also: js::InternalJobQueue https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/JSContext.h#l88 +// +public: +~JobQueue() = default; + +JSObject *getIncumbentGlobal(JSContext *cx) override; + +bool enqueuePromiseJob(JSContext *cx, JS::HandleObject promise, + JS::HandleObject job, JS::HandleObject allocationSite, + JS::HandleObject incumbentGlobal) override; + +void runJobs(JSContext *cx) override; + +/** + * @return true if the job queue is empty, false otherwise. + */ +bool empty() const override; + +private: +js::UniquePtr saveJobQueue(JSContext *) override; + +// +// Custom methods +// +public: +/** + * @brief Initialize PythonMonkey's event-loop job queue + * @param cx - javascript context pointer + * @return success + */ +bool init(JSContext *cx); +}; + +#endif \ No newline at end of file diff --git a/include/modules/pythonmonkey/pythonmonkey.hh b/include/modules/pythonmonkey/pythonmonkey.hh index d6c06a59..08247429 100644 --- a/include/modules/pythonmonkey/pythonmonkey.hh +++ b/include/modules/pythonmonkey/pythonmonkey.hh @@ -12,6 +12,7 @@ #define PythonMonkey_Module_PythonMonkey #include "include/PyType.hh" +#include "include/JobQueue.hh" #include #include @@ -25,6 +26,7 @@ static JSContext *GLOBAL_CX; /**< pointer to PythonMonkey's JSContext */ static JS::Rooted *global; /**< pointer to the global object of PythonMonkey's JSContext */ static JSAutoRealm *autoRealm; /**< pointer to PythonMonkey's AutoRealm */ +static JobQueue *JOB_QUEUE; /**< pointer to PythonMonkey's event-loop job queue */ /** * @brief Destroys the JSContext and deletes associated memory. Called when python quits or faces a fatal exception. diff --git a/src/JobQueue.cc b/src/JobQueue.cc new file mode 100644 index 00000000..9df06f5a --- /dev/null +++ b/src/JobQueue.cc @@ -0,0 +1,39 @@ +#include "include/JobQueue.hh" + +#include + +JSObject *JobQueue::getIncumbentGlobal(JSContext *cx) { + return JS::CurrentGlobalOrNull(cx); +} + +bool JobQueue::enqueuePromiseJob(JSContext *cx, + [[maybe_unused]] JS::HandleObject promise, + JS::HandleObject job, + [[maybe_unused]] JS::HandleObject allocationSite, + [[maybe_unused]] JS::HandleObject incumbentGlobal) { + printf("JobQueue::enqueuePromiseJob is unimplemented\n"); + return true; +} + +void JobQueue::runJobs(JSContext *cx) { + printf("JobQueue::runJobs is unimplemented\n"); + return; +} + +// is empty +bool JobQueue::empty() const { + printf("JobQueue::empty is unimplemented\n"); + return false; +} + +js::UniquePtr JobQueue::saveJobQueue(JSContext *cx) { + // TODO (Tom Tang): implement this method + printf("JobQueue::saveJobQueue is unimplemented\n"); + return nullptr; +} + +bool JobQueue::init(JSContext *cx) { + JS::SetJobQueue(cx, this); + // return js::UseInternalJobQueues(cx); + return true; +} diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 66cd20d9..d19ee268 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -60,9 +60,12 @@ static PyTypeObject BigIntType = { }; static void cleanup() { + JS::ShutdownAsyncTasks(GLOBAL_CX); if (GLOBAL_CX) JS_DestroyContext(GLOBAL_CX); JS_ShutDown(); delete global; + delete autoRealm; + delete JOB_QUEUE; } void memoizePyTypeAndGCThing(PyType *pyType, JS::Handle GCThing) { @@ -195,7 +198,8 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) return NULL; } - if (!js::UseInternalJobQueues(GLOBAL_CX)) { + JOB_QUEUE = new JobQueue(); + if (!JOB_QUEUE->init(GLOBAL_CX)) { PyErr_SetString(SpiderMonkeyError, "Spidermonkey could not create the event-loop."); return NULL; } From 031b29a76b9bd133ec4e8ec2318b38edb35bf881 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 3 Apr 2023 18:10:47 +0000 Subject: [PATCH 04/73] fix(event-loop): `JS_DestroyContext` also calls `JS::ShutdownAsyncTasks` (offThreadPromiseState shutdown) --- src/modules/pythonmonkey/pythonmonkey.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index d19ee268..9352c764 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -60,7 +60,6 @@ static PyTypeObject BigIntType = { }; static void cleanup() { - JS::ShutdownAsyncTasks(GLOBAL_CX); if (GLOBAL_CX) JS_DestroyContext(GLOBAL_CX); JS_ShutDown(); delete global; From 391677a68878f7fdbbbc0fe08fe106579bd18b4d Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 4 Apr 2023 01:12:13 +0000 Subject: [PATCH 05/73] feat(event-loop): queue promise jobs to the Python event-loop --- src/JobQueue.cc | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/src/JobQueue.cc b/src/JobQueue.cc index 9df06f5a..ee560105 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -1,4 +1,7 @@ #include "include/JobQueue.hh" +#include "include/pyTypeFactory.hh" + +#include #include @@ -11,29 +14,60 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, JS::HandleObject job, [[maybe_unused]] JS::HandleObject allocationSite, [[maybe_unused]] JS::HandleObject incumbentGlobal) { - printf("JobQueue::enqueuePromiseJob is unimplemented\n"); + + // Convert the `job` JS function to a Python function for event-loop callback + // TODO (Tom Tang): assert `job` is JS::Handle by JS::GetBuiltinClass(...) == js::ESClass::Function (17) + // FIXME (Tom Tang): objects not free-ed + auto global = new JS::RootedObject(cx, getIncumbentGlobal(cx)); + auto jobv = new JS::RootedValue(cx, JS::ObjectValue(*job.get())); + auto callback = pyTypeFactory(cx, global, jobv)->getPyObject(); + + // Get the running Python event-loop + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_event_loop + auto asyncio = PyImport_ImportModule("asyncio"); + auto loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); + // FIXME (Tom Tang): assert `loop` is not None + + // Enqueue job to the Python event-loop + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon + auto methodName = PyUnicode_DecodeFSDefault("call_soon"); + auto asyncHandle = PyObject_CallMethodOneArg(loop, methodName, callback); + // TODO (Tom Tang): refactor python calls into its own method + + // Inform the JS runtime that the job queue is no longer empty + JS::JobQueueMayNotBeEmpty(cx); + + // Clean up + Py_DECREF(asyncio); + Py_DECREF(loop); + Py_DECREF(methodName); + Py_DECREF(asyncHandle); + return true; } void JobQueue::runJobs(JSContext *cx) { + // TODO (Tom Tang): printf("JobQueue::runJobs is unimplemented\n"); return; } // is empty bool JobQueue::empty() const { + // TODO (Tom Tang): implement using `get_running_loop` and getting job count on loop??? printf("JobQueue::empty is unimplemented\n"); return false; } js::UniquePtr JobQueue::saveJobQueue(JSContext *cx) { - // TODO (Tom Tang): implement this method + // TODO (Tom Tang): implement this method way later printf("JobQueue::saveJobQueue is unimplemented\n"); return nullptr; } bool JobQueue::init(JSContext *cx) { JS::SetJobQueue(cx, this); - // return js::UseInternalJobQueues(cx); + // TODO (Tom Tang): JS::InitDispatchToEventLoop(...) + // see inside js::UseInternalJobQueues(cx); (initInternalDispatchQueue) return true; } From 17facfd77b1720b5e156fbebf9357b7177b67abf Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 4 Apr 2023 14:26:09 +0000 Subject: [PATCH 06/73] feat(exception-propagation): check if a Python exception has already been set when SpiderMonkey fails --- src/setSpiderMonkeyException.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/setSpiderMonkeyException.cc b/src/setSpiderMonkeyException.cc index 1f57395c..80ee6f30 100644 --- a/src/setSpiderMonkeyException.cc +++ b/src/setSpiderMonkeyException.cc @@ -21,6 +21,9 @@ #include void setSpiderMonkeyException(JSContext *cx) { + if (PyErr_Occurred()) { // Check if a Python exception has already been set, otherwise `PyErr_SetString` would overwrite the exception set + return; + } if (!JS_IsExceptionPending(cx)) { PyErr_SetString(SpiderMonkeyError, "Spidermonkey failed, but spidermonkey did not set an exception."); return; From 38878bcacbe4e4f4dbef86dd7cf2a57e22f7ae03 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 4 Apr 2023 14:46:26 +0000 Subject: [PATCH 07/73] feat(event-loop): assert Python event-loop running --- src/JobQueue.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/JobQueue.cc b/src/JobQueue.cc index ee560105..79d92cc2 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -23,10 +23,14 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, auto callback = pyTypeFactory(cx, global, jobv)->getPyObject(); // Get the running Python event-loop - // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_event_loop - auto asyncio = PyImport_ImportModule("asyncio"); - auto loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); - // FIXME (Tom Tang): assert `loop` is not None + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_running_loop + PyObject *asyncio = PyImport_ImportModule("asyncio"); + PyObject *loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); + if (loop == nullptr) { // `get_running_loop` would raise a RuntimeError if there is no running event loop + // PyErr_SetString(PyExc_RuntimeError, "No running Python event-loop."); // override the error raised by `get_running_loop` + Py_DECREF(asyncio); + return false; + } // Enqueue job to the Python event-loop // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon From 0198ddf8ab14d3e211b0dff5af773b251184d76c Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 4 Apr 2023 14:56:40 +0000 Subject: [PATCH 08/73] fix(event-loop): more specific error when no Python event-loop running --- src/JobQueue.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/JobQueue.cc b/src/JobQueue.cc index 79d92cc2..9a9f7cca 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -27,7 +27,9 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, PyObject *asyncio = PyImport_ImportModule("asyncio"); PyObject *loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); if (loop == nullptr) { // `get_running_loop` would raise a RuntimeError if there is no running event loop - // PyErr_SetString(PyExc_RuntimeError, "No running Python event-loop."); // override the error raised by `get_running_loop` + // Overwrite the error raised by `get_running_loop` + PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop to make asynchronous calls."); + // Clean up references Py_DECREF(asyncio); return false; } From 4935aaec0469aca31808a23c036974fbcb37a426 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 4 Apr 2023 15:04:44 +0000 Subject: [PATCH 09/73] refactor(event-loop): use `PyObject_CallMethod` with building values --- src/JobQueue.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/JobQueue.cc b/src/JobQueue.cc index 9a9f7cca..0944b0b1 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -36,8 +36,7 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, // Enqueue job to the Python event-loop // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon - auto methodName = PyUnicode_DecodeFSDefault("call_soon"); - auto asyncHandle = PyObject_CallMethodOneArg(loop, methodName, callback); + auto asyncHandle = PyObject_CallMethod(loop, "call_soon", "O", callback); // https://docs.python.org/3/c-api/arg.html#other-objects // TODO (Tom Tang): refactor python calls into its own method // Inform the JS runtime that the job queue is no longer empty @@ -46,7 +45,6 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, // Clean up Py_DECREF(asyncio); Py_DECREF(loop); - Py_DECREF(methodName); Py_DECREF(asyncHandle); return true; From d9895f3bf549dc194c795147201f2313397bf5fa Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 4 Apr 2023 21:14:34 +0000 Subject: [PATCH 10/73] feat(event-loop): support off-thread promises for async WebAssembly APIs (not implemented yet) --- include/IntType.hh | 2 +- include/JobQueue.hh | 9 +++++++++ src/JobQueue.cc | 13 +++++++++---- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/include/IntType.hh b/include/IntType.hh index 36cf5c60..71f13cb8 100644 --- a/include/IntType.hh +++ b/include/IntType.hh @@ -33,7 +33,7 @@ public: * @brief Construct a new IntType object from a JS::BigInt. * * @param cx - javascript context pointer - * @param str - JS::BigInt pointer + * @param bigint - JS::BigInt pointer */ IntType(JSContext *cx, JS::BigInt *bigint); diff --git a/include/JobQueue.hh b/include/JobQueue.hh index ce530a40..2bb07f65 100644 --- a/include/JobQueue.hh +++ b/include/JobQueue.hh @@ -49,6 +49,15 @@ public: * @return success */ bool init(JSContext *cx); + +/** + * @brief The callback for dispatching an off-thread promise to the event loop + * see https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/public/Promise.h#l580 + * https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/OffThreadPromiseRuntimeState.cpp#l160 + * @param closure - closure, currently the javascript context + * @param dispatchable - Pointer to the Dispatchable to be called + */ +static bool dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable); }; #endif \ No newline at end of file diff --git a/src/JobQueue.cc b/src/JobQueue.cc index 0944b0b1..4eb75502 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -17,9 +17,9 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, // Convert the `job` JS function to a Python function for event-loop callback // TODO (Tom Tang): assert `job` is JS::Handle by JS::GetBuiltinClass(...) == js::ESClass::Function (17) - // FIXME (Tom Tang): objects not free-ed + // FIXME (Tom Tang): memory leak, objects not free-ed auto global = new JS::RootedObject(cx, getIncumbentGlobal(cx)); - auto jobv = new JS::RootedValue(cx, JS::ObjectValue(*job.get())); + auto jobv = new JS::RootedValue(cx, JS::ObjectValue(*job)); auto callback = pyTypeFactory(cx, global, jobv)->getPyObject(); // Get the running Python event-loop @@ -71,7 +71,12 @@ js::UniquePtr JobQueue::saveJobQueue(JSContext *cx) bool JobQueue::init(JSContext *cx) { JS::SetJobQueue(cx, this); - // TODO (Tom Tang): JS::InitDispatchToEventLoop(...) - // see inside js::UseInternalJobQueues(cx); (initInternalDispatchQueue) + JS::InitDispatchToEventLoop(cx, /* callback */ dispatchToEventLoop, /* closure */ cx); + return true; +} + +bool JobQueue::dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable) { + JSContext *cx = (JSContext *)closure; // `closure` is provided in `JS::InitDispatchToEventLoop` call + // dispatchable->run(cx, JS::Dispatchable::NotShuttingDown); return true; } From 765420309e416f3aa68650ba18d76f08e2a73bc1 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 4 Apr 2023 21:42:35 +0000 Subject: [PATCH 11/73] refactor(event-loop): move `asyncio` python calls into its own `enqueueToPyEventLoop` method --- include/JobQueue.hh | 12 +++++++++++ src/JobQueue.cc | 50 ++++++++++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/include/JobQueue.hh b/include/JobQueue.hh index 2bb07f65..15e8e1e0 100644 --- a/include/JobQueue.hh +++ b/include/JobQueue.hh @@ -15,6 +15,8 @@ #include #include +#include + class JobQueue : public JS::JobQueue { // // JS::JobQueue methods. @@ -50,12 +52,22 @@ public: */ bool init(JSContext *cx); +/** + * @brief Send job to the running Python event-loop, or + * raise a Python RuntimeError if no event-loop running + * @param jobFn - The JS event-loop job converted to a Python function + * @return success + */ +bool enqueueToPyEventLoop(PyObject *jobFn); + +private: /** * @brief The callback for dispatching an off-thread promise to the event loop * see https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/public/Promise.h#l580 * https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/OffThreadPromiseRuntimeState.cpp#l160 * @param closure - closure, currently the javascript context * @param dispatchable - Pointer to the Dispatchable to be called + * @return not shutting down */ static bool dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable); }; diff --git a/src/JobQueue.cc b/src/JobQueue.cc index 4eb75502..b70029e3 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -18,36 +18,15 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, // Convert the `job` JS function to a Python function for event-loop callback // TODO (Tom Tang): assert `job` is JS::Handle by JS::GetBuiltinClass(...) == js::ESClass::Function (17) // FIXME (Tom Tang): memory leak, objects not free-ed + // FIXME (Tom Tang): `job` function is going to be GC-ed ??? auto global = new JS::RootedObject(cx, getIncumbentGlobal(cx)); auto jobv = new JS::RootedValue(cx, JS::ObjectValue(*job)); auto callback = pyTypeFactory(cx, global, jobv)->getPyObject(); - // Get the running Python event-loop - // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_running_loop - PyObject *asyncio = PyImport_ImportModule("asyncio"); - PyObject *loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); - if (loop == nullptr) { // `get_running_loop` would raise a RuntimeError if there is no running event loop - // Overwrite the error raised by `get_running_loop` - PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop to make asynchronous calls."); - // Clean up references - Py_DECREF(asyncio); - return false; - } - - // Enqueue job to the Python event-loop - // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon - auto asyncHandle = PyObject_CallMethod(loop, "call_soon", "O", callback); // https://docs.python.org/3/c-api/arg.html#other-objects - // TODO (Tom Tang): refactor python calls into its own method - // Inform the JS runtime that the job queue is no longer empty JS::JobQueueMayNotBeEmpty(cx); - // Clean up - Py_DECREF(asyncio); - Py_DECREF(loop); - Py_DECREF(asyncHandle); - - return true; + return enqueueToPyEventLoop(callback); } void JobQueue::runJobs(JSContext *cx) { @@ -75,6 +54,31 @@ bool JobQueue::init(JSContext *cx) { return true; } +bool JobQueue::enqueueToPyEventLoop(PyObject *jobFn) { + // Get the running Python event-loop + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_running_loop + PyObject *asyncio = PyImport_ImportModule("asyncio"); + PyObject *loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); + if (loop == nullptr) { // `get_running_loop` would raise a RuntimeError if there is no running event loop + // Overwrite the error raised by `get_running_loop` + PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop to make asynchronous calls."); + Py_DECREF(asyncio); // clean up + return false; + } + + // Enqueue job to the Python event-loop + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon + auto asyncHandle = PyObject_CallMethod(loop, "call_soon", "O", jobFn); // https://docs.python.org/3/c-api/arg.html#other-objects + + // Clean up references to Python objects + Py_DECREF(asyncio); + Py_DECREF(loop); + Py_DECREF(asyncHandle); + + return true; +} + +/* static */ bool JobQueue::dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable) { JSContext *cx = (JSContext *)closure; // `closure` is provided in `JS::InitDispatchToEventLoop` call // dispatchable->run(cx, JS::Dispatchable::NotShuttingDown); From 6a5edf50637bc26ca1510760aa9f20871f8bd25e Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 5 Apr 2023 17:34:01 +0000 Subject: [PATCH 12/73] refactor(event-loop): move python calls to the new `PyEventLoop` struct --- include/JobQueue.hh | 8 ------- include/PyEventLoop.hh | 49 ++++++++++++++++++++++++++++++++++++++++++ src/JobQueue.cc | 32 ++++++--------------------- src/PyEventLoop.cc | 32 +++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 33 deletions(-) create mode 100644 include/PyEventLoop.hh create mode 100644 src/PyEventLoop.cc diff --git a/include/JobQueue.hh b/include/JobQueue.hh index 15e8e1e0..5e59048b 100644 --- a/include/JobQueue.hh +++ b/include/JobQueue.hh @@ -52,14 +52,6 @@ public: */ bool init(JSContext *cx); -/** - * @brief Send job to the running Python event-loop, or - * raise a Python RuntimeError if no event-loop running - * @param jobFn - The JS event-loop job converted to a Python function - * @return success - */ -bool enqueueToPyEventLoop(PyObject *jobFn); - private: /** * @brief The callback for dispatching an off-thread promise to the event loop diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh new file mode 100644 index 00000000..d1135e81 --- /dev/null +++ b/include/PyEventLoop.hh @@ -0,0 +1,49 @@ +/** + * @file PyEventLoop.hh + * @author Tom Tang (xmader@distributive.network) + * @brief + * @version 0.1 + * @date 2023-04-05 + * + * @copyright Copyright (c) 2023 + * + */ + +#ifndef PythonMonkey_PyEventLoop_ +#define PythonMonkey_PyEventLoop_ + +#include + +struct PyEventLoop { +public: + ~PyEventLoop() { + Py_XDECREF(_loop); + } + + bool initialized() const { + return !!_loop; + } + + /** + * @brief Send job to the Python event-loop + * @param jobFn - The JS event-loop job converted to a Python function + * @return success + */ + bool enqueue(PyObject *jobFn); + bool enqueueWithDelay(PyObject *jobFn, double delaySeconds); + + /** + * @brief Get the running Python event-loop, or + * raise a Python RuntimeError if no event-loop running + * @return an instance of `PyEventLoop` + */ + static PyEventLoop getRunningLoop(); + +protected: + PyObject *_loop; + + PyEventLoop() = delete; + PyEventLoop(PyObject *loop) : _loop(loop) {}; +}; + +#endif \ No newline at end of file diff --git a/src/JobQueue.cc b/src/JobQueue.cc index b70029e3..253f9374 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -1,4 +1,5 @@ #include "include/JobQueue.hh" +#include "include/PyEventLoop.hh" #include "include/pyTypeFactory.hh" #include @@ -26,7 +27,12 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, // Inform the JS runtime that the job queue is no longer empty JS::JobQueueMayNotBeEmpty(cx); - return enqueueToPyEventLoop(callback); + // Send job to the running Python event-loop + PyEventLoop loop = PyEventLoop::getRunningLoop(); + if (!loop.initialized()) return false; + loop.enqueue(callback); + + return true; } void JobQueue::runJobs(JSContext *cx) { @@ -54,30 +60,6 @@ bool JobQueue::init(JSContext *cx) { return true; } -bool JobQueue::enqueueToPyEventLoop(PyObject *jobFn) { - // Get the running Python event-loop - // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_running_loop - PyObject *asyncio = PyImport_ImportModule("asyncio"); - PyObject *loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); - if (loop == nullptr) { // `get_running_loop` would raise a RuntimeError if there is no running event loop - // Overwrite the error raised by `get_running_loop` - PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop to make asynchronous calls."); - Py_DECREF(asyncio); // clean up - return false; - } - - // Enqueue job to the Python event-loop - // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon - auto asyncHandle = PyObject_CallMethod(loop, "call_soon", "O", jobFn); // https://docs.python.org/3/c-api/arg.html#other-objects - - // Clean up references to Python objects - Py_DECREF(asyncio); - Py_DECREF(loop); - Py_DECREF(asyncHandle); - - return true; -} - /* static */ bool JobQueue::dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable) { JSContext *cx = (JSContext *)closure; // `closure` is provided in `JS::InitDispatchToEventLoop` call diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc new file mode 100644 index 00000000..d0355469 --- /dev/null +++ b/src/PyEventLoop.cc @@ -0,0 +1,32 @@ +#include "include/PyEventLoop.hh" + +#include + +bool PyEventLoop::enqueue(PyObject *jobFn) { + // Enqueue job to the Python event-loop + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon + PyObject *asyncHandle = PyObject_CallMethod(_loop, "call_soon", "O", jobFn); // https://docs.python.org/3/c-api/arg.html#other-objects + Py_DECREF(asyncHandle); // clean up + return true; +} + +bool PyEventLoop::enqueueWithDelay(PyObject *jobFn, double delaySeconds) { + +} + +/* static */ +PyEventLoop PyEventLoop::getRunningLoop() { + // Get the running Python event-loop + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_running_loop + PyObject *asyncio = PyImport_ImportModule("asyncio"); + PyObject *loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); + + Py_DECREF(asyncio); // clean up + + if (loop == nullptr) { // `get_running_loop` would raise a RuntimeError if there is no running event loop + // Overwrite the error raised by `get_running_loop` + PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop to make asynchronous calls."); + } + + return PyEventLoop(loop); // `loop` may be null +} From d373206e623e2294fefd874a0d2dca3caa3b02ec Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 5 Apr 2023 18:08:56 +0000 Subject: [PATCH 13/73] feat(event-loop): support cancel event-loop jobs --- include/PyEventLoop.hh | 45 +++++++++++++++++++++++++++++++++++++++--- src/PyEventLoop.cc | 15 ++++++++------ 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index d1135e81..b2921624 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -24,13 +24,52 @@ public: return !!_loop; } + /** + * @brief C++ wrapper for Python `asyncio.Handle` class + * @see https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.Handle + */ + struct AsyncHandle { + public: + AsyncHandle(PyObject *handle) : _handle(handle) {}; + ~AsyncHandle() { + Py_XDECREF(_handle); + } + + /** + * @brief Cancel the scheduled event-loop job. + * If the job has already been canceled or executed, this method has no effect. + */ + void cancel(); + + /** + * @brief Get the unique `timeoutID` for JS `setTimeout`/`clearTimeout` methods + * @see https://developer.mozilla.org/en-US/docs/Web/API/setTimeout#return_value + */ + inline uint64_t getId() { + // Currently we use the address of the underlying `asyncio.Handle` object + // FIXME (Tom Tang): JS stores the `timeoutID` in a `number` (float64), may overflow + return (uint64_t)_handle; + } + static inline AsyncHandle fromId(uint64_t timeoutID) { + return AsyncHandle((PyObject *)timeoutID); + } + protected: + PyObject *_handle; + }; + /** * @brief Send job to the Python event-loop * @param jobFn - The JS event-loop job converted to a Python function - * @return success + * @return a AsyncHandle, the value can be safely ignored + */ + AsyncHandle enqueue(PyObject *jobFn); + /** + * @brief Schedule a job to the Python event-loop, with the given delay + * @param jobFn - The JS event-loop job converted to a Python function + * @param delaySeconds - The job function will be called after the given number of seconds + * @return a AsyncHandle, the value can be safely ignored */ - bool enqueue(PyObject *jobFn); - bool enqueueWithDelay(PyObject *jobFn, double delaySeconds); + AsyncHandle enqueueWithDelay(PyObject *jobFn, double delaySeconds); /** * @brief Get the running Python event-loop, or diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index d0355469..c72c4f63 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -2,17 +2,14 @@ #include -bool PyEventLoop::enqueue(PyObject *jobFn) { +PyEventLoop::AsyncHandle PyEventLoop::enqueue(PyObject *jobFn) { // Enqueue job to the Python event-loop // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon PyObject *asyncHandle = PyObject_CallMethod(_loop, "call_soon", "O", jobFn); // https://docs.python.org/3/c-api/arg.html#other-objects - Py_DECREF(asyncHandle); // clean up - return true; + return PyEventLoop::AsyncHandle(asyncHandle); } -bool PyEventLoop::enqueueWithDelay(PyObject *jobFn, double delaySeconds) { - -} +PyEventLoop::AsyncHandle PyEventLoop::enqueueWithDelay(PyObject *jobFn, double delaySeconds) {} /* static */ PyEventLoop PyEventLoop::getRunningLoop() { @@ -30,3 +27,9 @@ PyEventLoop PyEventLoop::getRunningLoop() { return PyEventLoop(loop); // `loop` may be null } + +void PyEventLoop::AsyncHandle::cancel() { + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.Handle.cancel + PyObject *ret = PyObject_CallMethod(_handle, "cancel", NULL); // returns None + Py_XDECREF(ret); +} From 65e6626bd095e732fd62c16fa89f1e624008edbc Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 5 Apr 2023 18:22:24 +0000 Subject: [PATCH 14/73] feat(event-loop): schedule job (with given delay) to the Python event-loop --- src/PyEventLoop.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index c72c4f63..5fdcd478 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -5,11 +5,16 @@ PyEventLoop::AsyncHandle PyEventLoop::enqueue(PyObject *jobFn) { // Enqueue job to the Python event-loop // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon - PyObject *asyncHandle = PyObject_CallMethod(_loop, "call_soon", "O", jobFn); // https://docs.python.org/3/c-api/arg.html#other-objects + PyObject *asyncHandle = PyObject_CallMethod(_loop, "call_soon", "O", jobFn); // https://docs.python.org/3/c-api/arg.html#c.Py_BuildValue return PyEventLoop::AsyncHandle(asyncHandle); } -PyEventLoop::AsyncHandle PyEventLoop::enqueueWithDelay(PyObject *jobFn, double delaySeconds) {} +PyEventLoop::AsyncHandle PyEventLoop::enqueueWithDelay(PyObject *jobFn, double delaySeconds) { + // Schedule job to the Python event-loop + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_later + PyObject *asyncHandle = PyObject_CallMethod(_loop, "call_later", "dO", delaySeconds, jobFn); // https://docs.python.org/3/c-api/arg.html#c.Py_BuildValue + return PyEventLoop::AsyncHandle(asyncHandle); +} /* static */ PyEventLoop PyEventLoop::getRunningLoop() { From 4fb29b5a3be2a72ee876522c1f0b171d764f62a3 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 6 Apr 2023 00:05:50 +0000 Subject: [PATCH 15/73] feat(event-loop): implement `setTimeout` and `clearTimeout` --- include/PyEventLoop.hh | 10 ++++ src/modules/pythonmonkey/pythonmonkey.cc | 59 ++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index b2921624..5fef1eca 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -46,13 +46,23 @@ public: * @see https://developer.mozilla.org/en-US/docs/Web/API/setTimeout#return_value */ inline uint64_t getId() { + Py_INCREF(_handle); // otherwise the object would be GC-ed as the AsyncHandle destructs // Currently we use the address of the underlying `asyncio.Handle` object // FIXME (Tom Tang): JS stores the `timeoutID` in a `number` (float64), may overflow return (uint64_t)_handle; } static inline AsyncHandle fromId(uint64_t timeoutID) { + // FIXME (Tom Tang): `clearTimeout` can only be applied once on the same handle because the handle is GC-ed return AsyncHandle((PyObject *)timeoutID); } + + /** + * @brief Get the underlying `asyncio.Handle` Python object + */ + inline PyObject *getHandleObject() const { + Py_INCREF(_handle); // otherwise the object would be GC-ed as the AsyncHandle destructor decreases the reference count + return _handle; + } protected: PyObject *_handle; }; diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 9352c764..fff18bbe 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -20,6 +20,7 @@ #include "include/PyType.hh" #include "include/pyTypeFactory.hh" #include "include/StrType.hh" +#include "include/PyEventLoop.hh" #include #include @@ -180,6 +181,59 @@ struct PyModuleDef pythonmonkey = PyObject *SpiderMonkeyError = NULL; +// Implement the `setTimeout` global function +// https://developer.mozilla.org/en-US/docs/Web/API/setTimeout +// https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#dom-settimeout +static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { + JS::CallArgs args = JS::CallArgsFromVp(argc, vp); + + // Get the function to be executed + // TODO (Tom Tang): `setTimeout` should allow passing additional arguments to the callback + // FIXME (Tom Tang): memory leak, not free-ed + JS::RootedObject *thisv = new JS::RootedObject(cx); + JS_ValueToObject(cx, args.thisv(), thisv); + JS::RootedValue *jobArg = new JS::RootedValue(cx, args[0]); + PyObject *job = pyTypeFactory(cx, thisv, jobArg)->getPyObject(); + + // Get the delay time + // JS `setTimeout` takes milliseconds, but Python takes seconds + double delayMs = args[1].toNumber(); + double delaySeconds = delayMs / 1000; // convert ms to s + + // Schedule job to the running Python event-loop + PyEventLoop loop = PyEventLoop::getRunningLoop(); + if (!loop.initialized()) return false; + PyEventLoop::AsyncHandle handle = loop.enqueueWithDelay(job, delaySeconds); + + // Return the `timeoutID` to use in `clearTimeout` + args.rval().setDouble((double)handle.getId()); + + return true; +} + +// Implement the `clearTimeout` global function +// https://developer.mozilla.org/en-US/docs/Web/API/clearTimeout +// https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#dom-cleartimeout +static bool clearTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { + using AsyncHandle = PyEventLoop::AsyncHandle; + JS::CallArgs args = JS::CallArgsFromVp(argc, vp); + + // Retrieve the AsyncHandle by `timeoutID` + double timeoutID = args[0].toNumber(); + AsyncHandle handle = AsyncHandle::fromId((uint64_t)timeoutID); + + // Cancel this job on Python event-loop + handle.cancel(); + + return true; +} + +static JSFunctionSpec jsGlobalFunctions[] = { + JS_FN("setTimeout", setTimeout, /* nargs */ 2, 0), + JS_FN("clearTimeout", clearTimeout, 1, 0), + JS_FS_END +}; + PyMODINIT_FUNC PyInit_pythonmonkey(void) { PyDateTime_IMPORT; @@ -218,6 +272,11 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) autoRealm = new JSAutoRealm(GLOBAL_CX, *global); + if (!JS_DefineFunctions(GLOBAL_CX, *global, jsGlobalFunctions)) { + PyErr_SetString(SpiderMonkeyError, "Spidermonkey could not define global functions."); + return NULL; + } + JS_SetGCCallback(GLOBAL_CX, handleSharedPythonMonkeyMemory, NULL); PyObject *pyModule; From dae66757e9c3da6b6a778cd81751923c0cbe6d77 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 6 Apr 2023 19:11:26 +0000 Subject: [PATCH 16/73] fix(event-loop): `setTimeout` should set `this` to the global object as spec-ed --- src/modules/pythonmonkey/pythonmonkey.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index fff18bbe..14e1ad12 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -190,14 +190,13 @@ static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { // Get the function to be executed // TODO (Tom Tang): `setTimeout` should allow passing additional arguments to the callback // FIXME (Tom Tang): memory leak, not free-ed - JS::RootedObject *thisv = new JS::RootedObject(cx); - JS_ValueToObject(cx, args.thisv(), thisv); + JS::RootedObject *thisv = new JS::RootedObject(cx, JS::GetNonCCWObjectGlobal(&args.callee())); // HTML spec requires `thisArg` to be the global object JS::RootedValue *jobArg = new JS::RootedValue(cx, args[0]); PyObject *job = pyTypeFactory(cx, thisv, jobArg)->getPyObject(); // Get the delay time // JS `setTimeout` takes milliseconds, but Python takes seconds - double delayMs = args[1].toNumber(); + double delayMs = args.hasDefined(1) ? args[1].toNumber() : 0; // use value of 0 if the delay parameter is omitted double delaySeconds = delayMs / 1000; // convert ms to s // Schedule job to the running Python event-loop From 9f21bf74bc22fa7c421f560426a1b0cda744287b Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 6 Apr 2023 19:18:52 +0000 Subject: [PATCH 17/73] fix(event-loop): `setTimeout` should set `delay` to 0 if < 0, as spec-ed --- src/modules/pythonmonkey/pythonmonkey.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 14e1ad12..292e9428 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -196,7 +196,9 @@ static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { // Get the delay time // JS `setTimeout` takes milliseconds, but Python takes seconds - double delayMs = args.hasDefined(1) ? args[1].toNumber() : 0; // use value of 0 if the delay parameter is omitted + double delayMs = 0; // use value of 0 if the delay parameter is omitted + if (args.hasDefined(1)) { JS::ToNumber(cx, args[1], &delayMs); } // implicitly do type coercion to a `number` + if (delayMs < 0) { delayMs = 0; } // as spec-ed double delaySeconds = delayMs / 1000; // convert ms to s // Schedule job to the running Python event-loop From d64b3ea9ad516d9d6ada3b7c7e438535d9344342 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 6 Apr 2023 20:55:24 +0000 Subject: [PATCH 18/73] fix(event-loop): `setTimeout` should allow passing additional arguments to the callback, as spec-ed --- src/modules/pythonmonkey/pythonmonkey.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 292e9428..6aecab34 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -188,10 +188,22 @@ static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { JS::CallArgs args = JS::CallArgsFromVp(argc, vp); // Get the function to be executed - // TODO (Tom Tang): `setTimeout` should allow passing additional arguments to the callback // FIXME (Tom Tang): memory leak, not free-ed JS::RootedObject *thisv = new JS::RootedObject(cx, JS::GetNonCCWObjectGlobal(&args.callee())); // HTML spec requires `thisArg` to be the global object JS::RootedValue *jobArg = new JS::RootedValue(cx, args[0]); + // `setTimeout` allows passing additional arguments to the callback, as spec-ed + if (args.length() > 2) { // having additional arguments + // Wrap the job function into a bound function with the given additional arguments + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/bind + JS::RootedVector bindArgs(cx); + bindArgs.append(JS::ObjectValue(**thisv)); + for (size_t i = 1, j = 2; j < args.length(); j++) { + bindArgs.append(args[j]); + } + JS::RootedObject jobArgObj = JS::RootedObject(cx, &args[0].toObject()); + JS_CallFunctionName(cx, jobArgObj, "bind", JS::HandleValueArray(bindArgs), jobArg); // jobArg = jobArg.bind(thisv, ...bindArgs) + } + // Convert to a Python function PyObject *job = pyTypeFactory(cx, thisv, jobArg)->getPyObject(); // Get the delay time From 24e53c9ec40917f057e205662bc49f7c668c7f84 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 10 Apr 2023 15:29:06 +0000 Subject: [PATCH 19/73] refactor(event-loop): use `enqueuePromiseJob`'s `incumbentGlobal` argument as the global object --- src/JobQueue.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/JobQueue.cc b/src/JobQueue.cc index 253f9374..3642f15a 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -14,13 +14,13 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, [[maybe_unused]] JS::HandleObject promise, JS::HandleObject job, [[maybe_unused]] JS::HandleObject allocationSite, - [[maybe_unused]] JS::HandleObject incumbentGlobal) { + JS::HandleObject incumbentGlobal) { // Convert the `job` JS function to a Python function for event-loop callback // TODO (Tom Tang): assert `job` is JS::Handle by JS::GetBuiltinClass(...) == js::ESClass::Function (17) // FIXME (Tom Tang): memory leak, objects not free-ed // FIXME (Tom Tang): `job` function is going to be GC-ed ??? - auto global = new JS::RootedObject(cx, getIncumbentGlobal(cx)); + auto global = new JS::RootedObject(cx, incumbentGlobal); auto jobv = new JS::RootedValue(cx, JS::ObjectValue(*job)); auto callback = pyTypeFactory(cx, global, jobv)->getPyObject(); From 815a11122f518d3f1b3e7d2388319b21564600ba Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 10 Apr 2023 21:46:28 +0000 Subject: [PATCH 20/73] feat(promise): JS Promise to Python awaitable (`asyncio.Future`) coercion --- include/PromiseType.hh | 6 ++++-- include/PyEventLoop.hh | 42 ++++++++++++++++++++++++++++++++++++++++++ src/PromiseType.cc | 42 +++++++++++++++++++++++++++++++++++++++--- src/PyEventLoop.cc | 18 ++++++++++++++++++ src/pyTypeFactory.cc | 7 +++++-- 5 files changed, 108 insertions(+), 7 deletions(-) diff --git a/include/PromiseType.hh b/include/PromiseType.hh index d6d63df6..f781e308 100644 --- a/include/PromiseType.hh +++ b/include/PromiseType.hh @@ -31,11 +31,13 @@ public: * @brief Construct a new PromiseType object from a JS::PromiseObject. * * @param cx - javascript context pointer - * @param promise - JS::PromiseObject pointer + * @param promise - JS::PromiseObject to be coerced */ - PromiseType(JSContext *cx, JS::Handle *promise); + PromiseType(JSContext *cx, JS::HandleObject promise); const TYPE returnType = TYPE::PYTHONMONKEY_PROMISE; +protected: + virtual void print(std::ostream &os) const override; }; #endif \ No newline at end of file diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index 5fef1eca..76e07f63 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -52,6 +52,7 @@ public: return (uint64_t)_handle; } static inline AsyncHandle fromId(uint64_t timeoutID) { + // FIXME: user can access arbitrary memory location // FIXME (Tom Tang): `clearTimeout` can only be applied once on the same handle because the handle is GC-ed return AsyncHandle((PyObject *)timeoutID); } @@ -81,6 +82,47 @@ public: */ AsyncHandle enqueueWithDelay(PyObject *jobFn, double delaySeconds); + /** + * @brief C++ wrapper for Python `asyncio.Future` class + * @see https://docs.python.org/3/library/asyncio-future.html#asyncio.Future + */ + struct Future { + public: + Future(PyObject *future) : _future(future) {}; + ~Future() { + Py_XDECREF(_future); + } + + /** + * @brief Mark the Future as done and set its result + * @see https://docs.python.org/3/library/asyncio-future.html#asyncio.Future.set_result + */ + void setResult(PyObject *result); + + /** + * @brief Mark the Future as done and set an exception + * @see https://docs.python.org/3/library/asyncio-future.html#asyncio.Future.set_exception + */ + void setException(PyObject *exception); + + /** + * @brief Get the underlying `asyncio.Future` Python object + */ + inline PyObject *getFutureObject() const { + Py_INCREF(_future); // otherwise the object would be GC-ed as this `PyEventLoop::Future` destructs + return _future; + } + protected: + PyObject *_future; + }; + + /** + * @brief Create a Python `asyncio.Future` object attached to this Python event-loop. + * @see https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.create_future + * @return a `Future` wrapper for the Python `asyncio.Future` object + */ + Future createFuture(); + /** * @brief Get the running Python event-loop, or * raise a Python RuntimeError if no event-loop running diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 03bd5d2a..59b0a27b 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -9,19 +9,55 @@ * */ -#include "include/modules/pythonmonkey/pythonmonkey.hh" - #include "include/PromiseType.hh" #include "include/PyType.hh" #include "include/TypeEnum.hh" +#include "include/pyTypeFactory.hh" + +#include "include/PyEventLoop.hh" #include +#include #include #include +#define PY_FUTURE_OBJ_SLOT 20 // (arbitrarily chosen) slot id to access the python object in JS callbacks + +static bool onResolvedCb(JSContext *cx, unsigned argc, JS::Value *vp) { + JS::CallArgs args = JS::CallArgsFromVp(argc, vp); + + // Convert the Promise's result to a Python object + // FIXME (Tom Tang): memory leak, not free-ed + JS::RootedObject *thisv = new JS::RootedObject(cx, &args.thisv().toObject()); + JS::RootedValue *resultArg = new JS::RootedValue(cx, args[0]); + PyObject *result = pyTypeFactory(cx, thisv, resultArg)->getPyObject(); + + // Get the `asyncio.Future` Python object from function's reserved slot + JS::Value futureObjVal = js::GetFunctionNativeReserved(&args.callee(), PY_FUTURE_OBJ_SLOT); + PyObject *futureObj = (PyObject *)(futureObjVal.toPrivate()); + + // Settle the Python asyncio.Future by the Promise's result + PyEventLoop::Future future = PyEventLoop::Future(futureObj); + future.setResult(result); +} + PromiseType::PromiseType(PyObject *object) : PyType(object) {} -PromiseType::PromiseType(JSContext *cx, JS::Handle *promise) {} +PromiseType::PromiseType(JSContext *cx, JS::HandleObject promise) { + // Create a python asyncio.Future on the running python event-loop + PyEventLoop loop = PyEventLoop::getRunningLoop(); + if (!loop.initialized()) return; + PyEventLoop::Future future = loop.createFuture(); + + // Callbacks to settle the Python asyncio.Future once the JS Promise is resolved + JS::RootedObject onFulfilled = JS::RootedObject(cx, (JSObject *)js::NewFunctionWithReserved(cx, onResolvedCb, 1, 0, NULL)); + js::SetFunctionNativeReserved(onFulfilled, PY_FUTURE_OBJ_SLOT, JS::PrivateValue(future.getFutureObject())); // put the address of the Python object in private slot so we can access it later + AddPromiseReactions(cx, promise, onFulfilled, nullptr); + // TODO (Tom Tang): JS onRejected -> Py set_exception, maybe reuse `onFulfilled` and detect if the promise is rejected + + pyObject = future.getFutureObject(); // must be a new reference +} +void PromiseType::print(std::ostream &os) const {} diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index 5fdcd478..eebce30d 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -16,6 +16,12 @@ PyEventLoop::AsyncHandle PyEventLoop::enqueueWithDelay(PyObject *jobFn, double d return PyEventLoop::AsyncHandle(asyncHandle); } +PyEventLoop::Future PyEventLoop::createFuture() { + // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.create_future + PyObject *futureObj = PyObject_CallMethod(_loop, "create_future", NULL); + return PyEventLoop::Future(futureObj); +} + /* static */ PyEventLoop PyEventLoop::getRunningLoop() { // Get the running Python event-loop @@ -38,3 +44,15 @@ void PyEventLoop::AsyncHandle::cancel() { PyObject *ret = PyObject_CallMethod(_handle, "cancel", NULL); // returns None Py_XDECREF(ret); } + +void PyEventLoop::Future::setResult(PyObject *result) { + // https://docs.python.org/3/library/asyncio-future.html#asyncio.Future.set_result + PyObject *ret = PyObject_CallMethod(_future, "set_result", "O", result); // returns None + Py_XDECREF(ret); +} + +void PyEventLoop::Future::setException(PyObject *exception) { + // https://docs.python.org/3/library/asyncio-future.html#asyncio.Future.set_exception + PyObject *ret = PyObject_CallMethod(_future, "set_exception", "O", exception); // returns None + Py_XDECREF(ret); +} diff --git a/src/pyTypeFactory.cc b/src/pyTypeFactory.cc index e44a2123..131fc94f 100644 --- a/src/pyTypeFactory.cc +++ b/src/pyTypeFactory.cc @@ -21,6 +21,7 @@ #include "include/ListType.hh" #include "include/NoneType.hh" #include "include/NullType.hh" +#include "include/PromiseType.hh" #include "include/PyType.hh" #include "include/setSpiderMonkeyException.hh" #include "include/StrType.hh" @@ -103,11 +104,13 @@ PyType *pyTypeFactory(JSContext *cx, JS::Rooted *global, JS::Rooted< break; } case js::ESClass::Date: { - JS::RootedValue unboxed(cx); - js::Unbox(cx, obj, &unboxed); returnValue = new DateType(cx, obj); break; } + case js::ESClass::Promise: { + returnValue = new PromiseType(cx, obj); + break; + } case js::ESClass::Function: { PyObject *JSCxGlobalFuncTuple = Py_BuildValue("(lll)", (uint64_t)cx, (uint64_t)global, (uint64_t)rval); PyObject *pyFunc = PyCFunction_New(&callJSFuncDef, JSCxGlobalFuncTuple); From 2e17e92f165f03654841ad004785830642c5e779 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 10 Apr 2023 21:50:03 +0000 Subject: [PATCH 21/73] fix(exception-propagation): `errorReport->filename` can be null --- src/setSpiderMonkeyException.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setSpiderMonkeyException.cc b/src/setSpiderMonkeyException.cc index 80ee6f30..236c4edc 100644 --- a/src/setSpiderMonkeyException.cc +++ b/src/setSpiderMonkeyException.cc @@ -52,7 +52,7 @@ void setSpiderMonkeyException(JSContext *cx) { std::stringstream outStrStream; JSErrorReport *errorReport = reportBuilder.report(); - if (errorReport) { + if (errorReport && errorReport->filename) { // `errorReport->filename` (the source file name) can be null std::string offsetSpaces(errorReport->tokenOffset(), ' '); // number of spaces equal to tokenOffset std::string linebuf; // the offending JS line of code (can be empty) From 3ddb0845d9146bcf7ba2aa2c0685a0bd68aab68f Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 11 Apr 2023 19:01:51 +0000 Subject: [PATCH 22/73] feat(exception-propagation): JS Error object to Python Exception object coercion --- include/ExceptionType.hh | 42 ++++++++++++++++++++++++++ include/TypeEnum.hh | 1 + include/setSpiderMonkeyException.hh | 7 +++++ src/ExceptionType.cc | 25 +++++++++++++++ src/pyTypeFactory.cc | 5 +++ src/setSpiderMonkeyException.cc | 42 +++++++++++++++----------- tests/python/test_pythonmonkey_eval.py | 16 ++++++++++ 7 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 include/ExceptionType.hh create mode 100644 src/ExceptionType.cc diff --git a/include/ExceptionType.hh b/include/ExceptionType.hh new file mode 100644 index 00000000..ead2304d --- /dev/null +++ b/include/ExceptionType.hh @@ -0,0 +1,42 @@ +/** + * @file ExceptionType.hh + * @author Tom Tang (xmader@distributive.network) + * @brief Struct for representing Python Exception objects from a corresponding JS Error object + * @version 0.1 + * @date 2023-04-11 + * + * @copyright Copyright (c) 2023 + * + */ + +#ifndef PythonMonkey_ExceptionType_ +#define PythonMonkey_ExceptionType_ + +#include "PyType.hh" +#include "TypeEnum.hh" + +#include + +#include + +/** + * @brief This struct represents a Python Exception object from the corresponding JS Error object + */ +struct ExceptionType : public PyType { +public: + ExceptionType(PyObject *object); + + /** + * @brief Construct a new SpiderMonkeyError from the JS Error object. + * + * @param cx - javascript context pointer + * @param error - JS Error object to be converted + */ + ExceptionType(JSContext *cx, JS::HandleObject error); + + const TYPE returnType = TYPE::EXCEPTION; +protected: + virtual void print(std::ostream &os) const override; +}; + +#endif \ No newline at end of file diff --git a/include/TypeEnum.hh b/include/TypeEnum.hh index 899db23e..472a4237 100644 --- a/include/TypeEnum.hh +++ b/include/TypeEnum.hh @@ -24,6 +24,7 @@ enum class TYPE { TUPLE, DATE, PYTHONMONKEY_PROMISE, + EXCEPTION, NONE, PYTHONMONKEY_NULL, }; diff --git a/include/setSpiderMonkeyException.hh b/include/setSpiderMonkeyException.hh index 4224ccfe..f838e5e4 100644 --- a/include/setSpiderMonkeyException.hh +++ b/include/setSpiderMonkeyException.hh @@ -14,6 +14,13 @@ #include +/** + * @brief Convert the given SpiderMonkey exception stack to a Python string + * + * @param cx - pointer to the JS context + * @param exceptionStack - reference to the SpiderMonkey exception stack + */ +PyObject *getExceptionString(JSContext *cx, const JS::ExceptionStack &exceptionStack); /** * @brief This function sets a python error under the assumption that a JS_* function call has failed. Do not call this function if that is not the case. diff --git a/src/ExceptionType.cc b/src/ExceptionType.cc new file mode 100644 index 00000000..e0a6a7e6 --- /dev/null +++ b/src/ExceptionType.cc @@ -0,0 +1,25 @@ +#include "include/modules/pythonmonkey/pythonmonkey.hh" +#include "include/setSpiderMonkeyException.hh" + +#include "include/ExceptionType.hh" + +#include +#include + +#include + +ExceptionType::ExceptionType(PyObject *object) : PyType(object) {} + +ExceptionType::ExceptionType(JSContext *cx, JS::HandleObject error) { + // Convert the JS Error object to a Python string + JS::RootedValue errValue(cx, JS::ObjectValue(*error)); // err + JS::RootedObject errStack(cx, JS::ExceptionStackOrNull(error)); // err.stack + PyObject *errStr = getExceptionString(cx, JS::ExceptionStack(cx, errValue, errStack)); + + // Construct a new SpiderMonkeyError python object + // pyObject = SpiderMonkeyError(errStr) + pyObject = PyObject_CallOneArg(SpiderMonkeyError, errStr); // _PyErr_CreateException, https://github.com/python/cpython/blob/3.9/Python/errors.c#L100 + Py_XDECREF(errStr); +} + +void ExceptionType::print(std::ostream &os) const {} diff --git a/src/pyTypeFactory.cc b/src/pyTypeFactory.cc index 131fc94f..cc8ee72d 100644 --- a/src/pyTypeFactory.cc +++ b/src/pyTypeFactory.cc @@ -14,6 +14,7 @@ #include "include/BoolType.hh" #include "include/DateType.hh" #include "include/DictType.hh" +#include "include/ExceptionType.hh" #include "include/FloatType.hh" #include "include/FuncType.hh" #include "include/IntType.hh" @@ -111,6 +112,10 @@ PyType *pyTypeFactory(JSContext *cx, JS::Rooted *global, JS::Rooted< returnValue = new PromiseType(cx, obj); break; } + case js::ESClass::Error: { + returnValue = new ExceptionType(cx, obj); + break; + } case js::ESClass::Function: { PyObject *JSCxGlobalFuncTuple = Py_BuildValue("(lll)", (uint64_t)cx, (uint64_t)global, (uint64_t)rval); PyObject *pyFunc = PyCFunction_New(&callJSFuncDef, JSCxGlobalFuncTuple); diff --git a/src/setSpiderMonkeyException.cc b/src/setSpiderMonkeyException.cc index 236c4edc..21e3f73d 100644 --- a/src/setSpiderMonkeyException.cc +++ b/src/setSpiderMonkeyException.cc @@ -20,23 +20,10 @@ #include -void setSpiderMonkeyException(JSContext *cx) { - if (PyErr_Occurred()) { // Check if a Python exception has already been set, otherwise `PyErr_SetString` would overwrite the exception set - return; - } - if (!JS_IsExceptionPending(cx)) { - PyErr_SetString(SpiderMonkeyError, "Spidermonkey failed, but spidermonkey did not set an exception."); - return; - } - JS::ExceptionStack exceptionStack(cx); - if (!JS::GetPendingExceptionStack(cx, &exceptionStack)) { - PyErr_SetString(SpiderMonkeyError, "Spidermonkey set an exception, but was unable to retrieve it."); - return; - } +PyObject *getExceptionString(JSContext *cx, const JS::ExceptionStack &exceptionStack) { JS::ErrorReportBuilder reportBuilder(cx); if (!reportBuilder.init(cx, exceptionStack, JS::ErrorReportBuilder::WithSideEffects /* may call the `toString` method if an object is thrown */)) { - PyErr_SetString(SpiderMonkeyError, "Spidermonkey set an exception, but could not initialize the error report."); - return; + return PyUnicode_FromString("Spidermonkey set an exception, but could not initialize the error report."); } /** @@ -78,5 +65,26 @@ void setSpiderMonkeyException(JSContext *cx) { outStrStream << "Stack Trace: \n" << StrType(cx, stackStr).getValue(); } - PyErr_SetString(SpiderMonkeyError, outStrStream.str().c_str()); -} \ No newline at end of file + return PyUnicode_FromString(outStrStream.str().c_str()); +} + +void setSpiderMonkeyException(JSContext *cx) { + if (PyErr_Occurred()) { // Check if a Python exception has already been set, otherwise `PyErr_SetString` would overwrite the exception set + return; + } + if (!JS_IsExceptionPending(cx)) { + PyErr_SetString(SpiderMonkeyError, "Spidermonkey failed, but spidermonkey did not set an exception."); + return; + } + JS::ExceptionStack exceptionStack(cx); + if (!JS::GetPendingExceptionStack(cx, &exceptionStack)) { + PyErr_SetString(SpiderMonkeyError, "Spidermonkey set an exception, but was unable to retrieve it."); + return; + } + + // `PyErr_SetString` uses `PyErr_SetObject` with `PyUnicode_FromString` under the hood + // see https://github.com/python/cpython/blob/3.9/Python/errors.c#L234-L236 + PyObject *errStr = getExceptionString(cx, exceptionStack); + PyErr_SetObject(SpiderMonkeyError, errStr); + Py_XDECREF(errStr); +} diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 18a1d7ab..f1cc23e7 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -408,10 +408,26 @@ def test_eval_exceptions(): with pytest.raises(pm.SpiderMonkeyError, match="Error: abc"): # manually by the `throw` statement pm.eval('throw new Error("abc")') + + # ANYTHING can be thrown in JS + with pytest.raises(pm.SpiderMonkeyError, match="uncaught exception: 9007199254740993"): + pm.eval('throw 9007199254740993n') # 2**53+1 + with pytest.raises(pm.SpiderMonkeyError, match="uncaught exception: null"): + pm.eval('throw null') + with pytest.raises(pm.SpiderMonkeyError, match="uncaught exception: undefined"): + pm.eval('throw undefined') with pytest.raises(pm.SpiderMonkeyError, match="uncaught exception: something from toString"): # (side effect) calls the `toString` method if an object is thrown pm.eval('throw { toString() { return "something from toString" } }') + # convert JS Error object to a Python Exception object for later use (in a `raise` statement) + js_err = pm.eval("new RangeError('to be raised in Python')") + assert isinstance(js_err, BaseException) + assert isinstance(js_err, Exception) + assert type(js_err) == pm.SpiderMonkeyError + with pytest.raises(pm.SpiderMonkeyError, match="RangeError: to be raised in Python"): + raise js_err + def test_eval_undefined(): x = pm.eval("undefined") assert x == None From 7484319f31b96148577921cefad9208b8a1746e6 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 11 Apr 2023 20:51:26 +0000 Subject: [PATCH 23/73] feat(promise): handle rejected JS Promise --- src/PromiseType.cc | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 59b0a27b..799188ec 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -10,13 +10,12 @@ */ #include "include/PromiseType.hh" +#include "include/PyEventLoop.hh" #include "include/PyType.hh" #include "include/TypeEnum.hh" #include "include/pyTypeFactory.hh" -#include "include/PyEventLoop.hh" - #include #include #include @@ -24,23 +23,35 @@ #include #define PY_FUTURE_OBJ_SLOT 20 // (arbitrarily chosen) slot id to access the python object in JS callbacks +#define PROMISE_OBJ_SLOT 21 static bool onResolvedCb(JSContext *cx, unsigned argc, JS::Value *vp) { JS::CallArgs args = JS::CallArgsFromVp(argc, vp); - // Convert the Promise's result to a Python object - // FIXME (Tom Tang): memory leak, not free-ed - JS::RootedObject *thisv = new JS::RootedObject(cx, &args.thisv().toObject()); - JS::RootedValue *resultArg = new JS::RootedValue(cx, args[0]); - PyObject *result = pyTypeFactory(cx, thisv, resultArg)->getPyObject(); + // Convert the Promise's result (either fulfilled resolution or rejection reason) to a Python object + JS::RootedObject thisv(cx); + // args.computeThis(cx, &thisv); // thisv is the global object, not the promise + JS::RootedValue resultArg(cx, args[0]); + PyObject *result = pyTypeFactory(cx, &thisv, &resultArg)->getPyObject(); // Get the `asyncio.Future` Python object from function's reserved slot JS::Value futureObjVal = js::GetFunctionNativeReserved(&args.callee(), PY_FUTURE_OBJ_SLOT); PyObject *futureObj = (PyObject *)(futureObjVal.toPrivate()); + // Get the Promise state + JS::Value promiseObjVal = js::GetFunctionNativeReserved(&args.callee(), PROMISE_OBJ_SLOT); + JS::RootedObject promise = JS::RootedObject(cx, &promiseObjVal.toObject()); + JS::PromiseState state = JS::GetPromiseState(promise); + // Settle the Python asyncio.Future by the Promise's result PyEventLoop::Future future = PyEventLoop::Future(futureObj); - future.setResult(result); + if (state == JS::PromiseState::Fulfilled) { + future.setResult(result); + } else { // state == JS::PromiseState::Rejected + future.setException(result); + } + + return true; } PromiseType::PromiseType(PyObject *object) : PyType(object) {} @@ -52,10 +63,10 @@ PromiseType::PromiseType(JSContext *cx, JS::HandleObject promise) { PyEventLoop::Future future = loop.createFuture(); // Callbacks to settle the Python asyncio.Future once the JS Promise is resolved - JS::RootedObject onFulfilled = JS::RootedObject(cx, (JSObject *)js::NewFunctionWithReserved(cx, onResolvedCb, 1, 0, NULL)); - js::SetFunctionNativeReserved(onFulfilled, PY_FUTURE_OBJ_SLOT, JS::PrivateValue(future.getFutureObject())); // put the address of the Python object in private slot so we can access it later - AddPromiseReactions(cx, promise, onFulfilled, nullptr); - // TODO (Tom Tang): JS onRejected -> Py set_exception, maybe reuse `onFulfilled` and detect if the promise is rejected + JS::RootedObject onResolved = JS::RootedObject(cx, (JSObject *)js::NewFunctionWithReserved(cx, onResolvedCb, 1, 0, NULL)); + js::SetFunctionNativeReserved(onResolved, PY_FUTURE_OBJ_SLOT, JS::PrivateValue(future.getFutureObject())); // put the address of the Python object in private slot so we can access it later + js::SetFunctionNativeReserved(onResolved, PROMISE_OBJ_SLOT, JS::ObjectValue(*promise)); + AddPromiseReactions(cx, promise, onResolved, onResolved); pyObject = future.getFutureObject(); // must be a new reference } From f0615ff279c0b8d20a391be61009bf5c3717ab5c Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 12 Apr 2023 04:02:20 +0000 Subject: [PATCH 24/73] fix(promise): handle Promise rejection with non-Error object thrown --- src/PromiseType.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 799188ec..b284235c 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -9,6 +9,8 @@ * */ +#include "include/modules/pythonmonkey/pythonmonkey.hh" + #include "include/PromiseType.hh" #include "include/PyEventLoop.hh" @@ -28,21 +30,28 @@ static bool onResolvedCb(JSContext *cx, unsigned argc, JS::Value *vp) { JS::CallArgs args = JS::CallArgsFromVp(argc, vp); + // Get the Promise state + JS::Value promiseObjVal = js::GetFunctionNativeReserved(&args.callee(), PROMISE_OBJ_SLOT); + JS::RootedObject promise = JS::RootedObject(cx, &promiseObjVal.toObject()); + JS::PromiseState state = JS::GetPromiseState(promise); + // Convert the Promise's result (either fulfilled resolution or rejection reason) to a Python object JS::RootedObject thisv(cx); // args.computeThis(cx, &thisv); // thisv is the global object, not the promise JS::RootedValue resultArg(cx, args[0]); PyObject *result = pyTypeFactory(cx, &thisv, &resultArg)->getPyObject(); + if (state == JS::PromiseState::Rejected && !PyExceptionInstance_Check(result)) { + // Wrap the result object into a SpiderMonkeyError object + // because only *Exception objects can be thrown in Python `raise` statement and alike + PyObject *wrapped = PyObject_CallOneArg(SpiderMonkeyError, result); // wrapped = SpiderMonkeyError(result) + Py_XDECREF(result); + result = wrapped; + } // Get the `asyncio.Future` Python object from function's reserved slot JS::Value futureObjVal = js::GetFunctionNativeReserved(&args.callee(), PY_FUTURE_OBJ_SLOT); PyObject *futureObj = (PyObject *)(futureObjVal.toPrivate()); - // Get the Promise state - JS::Value promiseObjVal = js::GetFunctionNativeReserved(&args.callee(), PROMISE_OBJ_SLOT); - JS::RootedObject promise = JS::RootedObject(cx, &promiseObjVal.toObject()); - JS::PromiseState state = JS::GetPromiseState(promise); - // Settle the Python asyncio.Future by the Promise's result PyEventLoop::Future future = PyEventLoop::Future(futureObj); if (state == JS::PromiseState::Fulfilled) { From d9ec062e26a9d90165bcb5a7feb9942ca8e30600 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 12 Apr 2023 18:02:07 +0000 Subject: [PATCH 25/73] feat(promise): python awaitable object check --- include/PromiseType.hh | 6 ++++++ src/PromiseType.cc | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/include/PromiseType.hh b/include/PromiseType.hh index f781e308..26010430 100644 --- a/include/PromiseType.hh +++ b/include/PromiseType.hh @@ -40,4 +40,10 @@ protected: virtual void print(std::ostream &os) const override; }; +/** + * @brief Check if the object can be used in Python await expression. + * `PyAwaitable_Check` hasn't been and has no plan to be added to the Python C API as of CPython 3.9 + */ +bool PythonAwaitable_Check(PyObject *obj); + #endif \ No newline at end of file diff --git a/src/PromiseType.cc b/src/PromiseType.cc index b284235c..177459ad 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -81,3 +81,16 @@ PromiseType::PromiseType(JSContext *cx, JS::HandleObject promise) { } void PromiseType::print(std::ostream &os) const {} + +bool PythonAwaitable_Check(PyObject *obj) { + // result = inspect.isawaitable(obj) + // see https://docs.python.org/3.9/library/inspect.html#inspect.isawaitable + // https://github.com/python/cpython/blob/3.9/Lib/inspect.py#L230-L235 + PyObject *inspect = PyImport_ImportModule("inspect"); + PyObject *result = PyObject_CallMethod(inspect, "isawaitable", "O", obj); // https://docs.python.org/3/c-api/arg.html#c.Py_BuildValue + bool isAwaitable = result == Py_True; + // cleanup + Py_DECREF(inspect); + Py_DECREF(result); + return isAwaitable; +} From b33af5b906f5db836e1b4915ceff59e4fa243976 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 13 Apr 2023 21:38:23 +0000 Subject: [PATCH 26/73] feat(promise): Python awaitable to JS Promise coercion --- include/PromiseType.hh | 7 +++++++ include/PyEventLoop.hh | 26 ++++++++++++++++++++++++++ src/PromiseType.cc | 42 ++++++++++++++++++++++++++++++++++++++++++ src/PyEventLoop.cc | 38 ++++++++++++++++++++++++++++++++++++++ src/jsTypeFactory.cc | 8 ++++++++ src/pyTypeFactory.cc | 4 ++-- 6 files changed, 123 insertions(+), 2 deletions(-) diff --git a/include/PromiseType.hh b/include/PromiseType.hh index 26010430..0874fdec 100644 --- a/include/PromiseType.hh +++ b/include/PromiseType.hh @@ -36,6 +36,13 @@ public: PromiseType(JSContext *cx, JS::HandleObject promise); const TYPE returnType = TYPE::PYTHONMONKEY_PROMISE; + + /** + * @brief Convert a Python [awaitable](https://docs.python.org/3/library/asyncio-task.html#awaitables) object to JS Promise + * + * @param cx - javascript context pointer + */ + JSObject *toJsPromise(JSContext *cx); protected: virtual void print(std::ostream &os) const override; }; diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index 76e07f63..0f611a2b 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -105,6 +105,26 @@ public: */ void setException(PyObject *exception); + /** + * @brief Add a callback to be run when the Future is done + * @see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.add_done_callback + */ + void addDoneCallback(PyObject *cb); + + /** + * @brief Get the result of the Future. + * Would raise exception if the Future is pending, cancelled, or having an exception set. + * @see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.result + */ + PyObject *getResult(); + + /** + * @brief Get the exception object that was set on this Future, or `Py_None` if no exception was set. + * Would raise an exception if the Future is pending or cancelled. + * @see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.exception + */ + PyObject *getException(); + /** * @brief Get the underlying `asyncio.Future` Python object */ @@ -123,6 +143,12 @@ public: */ Future createFuture(); + /** + * @brief Convert a Python awaitable to `asyncio.Future` attached to this Python event-loop. + * @see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.ensure_future + */ + Future ensureFuture(PyObject *awaitable); + /** * @brief Get the running Python event-loop, or * raise a Python RuntimeError if no event-loop running diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 177459ad..c10523d1 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -17,6 +17,7 @@ #include "include/PyType.hh" #include "include/TypeEnum.hh" #include "include/pyTypeFactory.hh" +#include "include/jsTypeFactory.hh" #include #include @@ -82,6 +83,47 @@ PromiseType::PromiseType(JSContext *cx, JS::HandleObject promise) { void PromiseType::print(std::ostream &os) const {} +// Callback to resolve or reject the JS Promise when the Future is done +static PyObject *futureOnDoneCallback(PyObject *futureCallbackTuple, PyObject *args) { + JSContext *cx = (JSContext *)PyLong_AsLongLong(PyTuple_GetItem(futureCallbackTuple, 0)); + JS::HandleObject promise = *(JS::RootedObject *)PyLong_AsLongLong(PyTuple_GetItem(futureCallbackTuple, 1)); + PyObject *futureObj = PyTuple_GetItem(args, 0); // the callback is called with the Future object as its only argument + // see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.add_done_callback + PyEventLoop::Future future = PyEventLoop::Future(futureObj); + + PyObject *exception = future.getException(); + if (exception == Py_None) { // no exception set on this awaitable, safe to get result, otherwise the exception will be raised when calling `futureObj.result()` + PyObject *result = future.getResult(); + JS::ResolvePromise(cx, promise, JS::RootedValue(cx, jsTypeFactory(cx, result))); + Py_DECREF(result); + } else { // having exception set, to reject the promise + JS::RejectPromise(cx, promise, JS::RootedValue(cx, jsTypeFactory(cx, exception))); + } + Py_XDECREF(exception); // cleanup + + // Py_DECREF(futureObj) // would cause bug, because `Future` constructor didn't increase futureObj's ref count, but the destructor will decrease it + Py_RETURN_NONE; +} +static PyMethodDef futureCallbackDef = {"futureOnDoneCallback", futureOnDoneCallback, METH_VARARGS, NULL}; + +JSObject *PromiseType::toJsPromise(JSContext *cx) { + // Create a new JS Promise object + JSObject *promise = JS::NewPromiseObject(cx, nullptr); + + // Convert the python awaitable to an asyncio.Future object + PyEventLoop loop = PyEventLoop::getRunningLoop(); + if (!loop.initialized()) return nullptr; + PyEventLoop::Future future = loop.ensureFuture(pyObject); + + // Resolve or Reject the JS Promise once the python awaitable is done + JS::RootedObject *rooted = new JS::RootedObject(cx, promise); // FIXME (Tom Tang): memory leak + PyObject *futureCallbackTuple = Py_BuildValue("(ll)", (uint64_t)cx, (uint64_t)rooted); + PyObject *onDoneCb = PyCFunction_New(&futureCallbackDef, futureCallbackTuple); + future.addDoneCallback(onDoneCb); + + return promise; +} + bool PythonAwaitable_Check(PyObject *obj) { // result = inspect.isawaitable(obj) // see https://docs.python.org/3.9/library/inspect.html#inspect.isawaitable diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index eebce30d..4106b07d 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -22,6 +22,28 @@ PyEventLoop::Future PyEventLoop::createFuture() { return PyEventLoop::Future(futureObj); } +PyEventLoop::Future PyEventLoop::ensureFuture(PyObject *awaitable) { + PyObject *asyncio = PyImport_ImportModule("asyncio"); + + PyObject *ensure_future_fn = PyObject_GetAttrString(asyncio, "ensure_future"); // ensure_future_fn = asyncio.ensure_future + // instead of a simpler `PyObject_CallMethod`, only the `PyObject_Call` API function can be used here because `loop` is a keyword-only argument + // see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.ensure_future + // https://docs.python.org/3/c-api/call.html#object-calling-api + PyObject *args = PyTuple_New(1); + PyTuple_SetItem(args, 0, awaitable); + PyObject *kwargs = PyDict_New(); + PyDict_SetItemString(kwargs, "loop", _loop); + PyObject *futureObj = PyObject_Call(ensure_future_fn, args, kwargs); // futureObj = ensure_future_fn(awaitable, loop=_loop) + + // clean up + Py_DECREF(asyncio); + Py_DECREF(ensure_future_fn); + Py_DECREF(args); + Py_DECREF(kwargs); + + return PyEventLoop::Future(futureObj); +} + /* static */ PyEventLoop PyEventLoop::getRunningLoop() { // Get the running Python event-loop @@ -56,3 +78,19 @@ void PyEventLoop::Future::setException(PyObject *exception) { PyObject *ret = PyObject_CallMethod(_future, "set_exception", "O", exception); // returns None Py_XDECREF(ret); } + +void PyEventLoop::Future::addDoneCallback(PyObject *cb) { + // https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.add_done_callback + PyObject *ret = PyObject_CallMethod(_future, "add_done_callback", "O", cb); // returns None + Py_XDECREF(ret); +} + +PyObject *PyEventLoop::Future::getResult() { + // https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.result + return PyObject_CallMethod(_future, "result", NULL); +} + +PyObject *PyEventLoop::Future::getException() { + // https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.exception + return PyObject_CallMethod(_future, "exception", NULL); +} diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index bff71694..29073d19 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -17,6 +17,7 @@ #include "include/pyTypeFactory.hh" #include "include/StrType.hh" #include "include/IntType.hh" +#include "include/PromiseType.hh" #include #include @@ -145,6 +146,13 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { else if (object == PythonMonkey_Null) { returnType.setNull(); } + else if (PythonAwaitable_Check(object)) { + auto p = new PromiseType(object); // FIXME (Tom Tang): get rid of `new`. The real problem is that we don't want `~PromiseType` to be called because it decreases `object`'s ref count to 0 + JSObject *promise = p->toJsPromise(cx); + returnType.setObject(*promise); + // FIXME (Tom Tang): how to tell Python to GC the object once JS is done with the Promise? + // memoizePyTypeAndGCThing(p, returnType); + } else { PyErr_SetString(PyExc_TypeError, "Python types other than bool, function, int, pythonmonkey.bigint, pythonmonkey.null, float, str, and None are not supported by pythonmonkey yet."); } diff --git a/src/pyTypeFactory.cc b/src/pyTypeFactory.cc index cc8ee72d..3d7d3eb4 100644 --- a/src/pyTypeFactory.cc +++ b/src/pyTypeFactory.cc @@ -143,12 +143,12 @@ PyType *pyTypeFactory(JSContext *cx, JS::Rooted *global, JS::Rooted< break; } default: { - printf("objects of this type are not handled by PythonMonkey yet"); + printf("objects %d of this type are not handled by PythonMonkey yet\n", cls); } } } else if (rval->isMagic()) { - printf("magic type is not handled by PythonMonkey yet"); + printf("magic type is not handled by PythonMonkey yet\n"); } return returnValue; From 204a0682ad1eef816ca9a17c3e29a7043ebb3739 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 13 Apr 2023 21:47:09 +0000 Subject: [PATCH 27/73] fix(promise): memory leak for the rooted Promise object in `PromiseType::toJsPromise` --- src/PromiseType.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/PromiseType.cc b/src/PromiseType.cc index c10523d1..3a08b8aa 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -86,7 +86,8 @@ void PromiseType::print(std::ostream &os) const {} // Callback to resolve or reject the JS Promise when the Future is done static PyObject *futureOnDoneCallback(PyObject *futureCallbackTuple, PyObject *args) { JSContext *cx = (JSContext *)PyLong_AsLongLong(PyTuple_GetItem(futureCallbackTuple, 0)); - JS::HandleObject promise = *(JS::RootedObject *)PyLong_AsLongLong(PyTuple_GetItem(futureCallbackTuple, 1)); + JS::RootedObject *rootedPtr = (JS::RootedObject *)PyLong_AsLongLong(PyTuple_GetItem(futureCallbackTuple, 1)); + JS::HandleObject promise = *rootedPtr; PyObject *futureObj = PyTuple_GetItem(args, 0); // the callback is called with the Future object as its only argument // see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.add_done_callback PyEventLoop::Future future = PyEventLoop::Future(futureObj); @@ -101,6 +102,8 @@ static PyObject *futureOnDoneCallback(PyObject *futureCallbackTuple, PyObject *a } Py_XDECREF(exception); // cleanup + delete rootedPtr; // no longer needed to be rooted, clean it up + // Py_DECREF(futureObj) // would cause bug, because `Future` constructor didn't increase futureObj's ref count, but the destructor will decrease it Py_RETURN_NONE; } @@ -116,8 +119,8 @@ JSObject *PromiseType::toJsPromise(JSContext *cx) { PyEventLoop::Future future = loop.ensureFuture(pyObject); // Resolve or Reject the JS Promise once the python awaitable is done - JS::RootedObject *rooted = new JS::RootedObject(cx, promise); // FIXME (Tom Tang): memory leak - PyObject *futureCallbackTuple = Py_BuildValue("(ll)", (uint64_t)cx, (uint64_t)rooted); + JS::RootedObject *rootedPtr = new JS::RootedObject(cx, promise); // `promise` is required to be rooted from here to the end of onDoneCallback + PyObject *futureCallbackTuple = Py_BuildValue("(ll)", (uint64_t)cx, (uint64_t)rootedPtr); PyObject *onDoneCb = PyCFunction_New(&futureCallbackDef, futureCallbackTuple); future.addDoneCallback(onDoneCb); From 42a2dbff906349374f309fccb6b456d0dc7a4879 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 13 Apr 2023 22:02:44 +0000 Subject: [PATCH 28/73] feat(promise): handle cancelled Python awaitable in to JS Promise coercion --- src/PromiseType.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 3a08b8aa..6b1c95b7 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -93,7 +93,10 @@ static PyObject *futureOnDoneCallback(PyObject *futureCallbackTuple, PyObject *a PyEventLoop::Future future = PyEventLoop::Future(futureObj); PyObject *exception = future.getException(); - if (exception == Py_None) { // no exception set on this awaitable, safe to get result, otherwise the exception will be raised when calling `futureObj.result()` + if (exception == NULL || PyErr_Occurred()) { // awaitable is cancelled, `futureObj.exception()` raises a CancelledError + // TODO (Tom Tang): get bool future.isCancelled(), and reject the promise with a CancelledError + return NULL; + } else if (exception == Py_None) { // no exception set on this awaitable, safe to get result, otherwise the exception will be raised when calling `futureObj.result()` PyObject *result = future.getResult(); JS::ResolvePromise(cx, promise, JS::RootedValue(cx, jsTypeFactory(cx, result))); Py_DECREF(result); From 08a373995b2f1b363e9758ab6e89e9e48f9129b6 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 17 Apr 2023 15:18:01 +0000 Subject: [PATCH 29/73] feat(jsTypeFactory): jsTypeFactorySafe to warn instead of setting an error in Python error stack --- include/jsTypeFactory.hh | 6 ++++++ src/PromiseType.cc | 4 ++-- src/jsTypeFactory.cc | 13 ++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/include/jsTypeFactory.hh b/include/jsTypeFactory.hh index 5f8e4b4c..6ec68412 100644 --- a/include/jsTypeFactory.hh +++ b/include/jsTypeFactory.hh @@ -28,6 +28,7 @@ struct PythonExternalString; * @return size_t - length of outStr (counting surrogate pairs as 2) */ size_t UCS4ToUTF16(const uint32_t *chars, size_t length, uint16_t *outStr); + /** * @brief Function that takes a PyObject and returns a corresponding JS::Value, doing shared memory management when necessary * @@ -36,6 +37,11 @@ size_t UCS4ToUTF16(const uint32_t *chars, size_t length, uint16_t *outStr); * @return JS::Value - A JS::Value corresponding to the PyType */ JS::Value jsTypeFactory(JSContext *cx, PyObject *object); +/** + * @brief same to jsTypeFactory, but it's guaranteed that no error would be set on the Python error stack, instead + * return JS `null` on error, and output a warning in Python-land + */ +JS::Value jsTypeFactorySafe(JSContext *cx, PyObject *object); /** * @brief Helper function for jsTypeFactory to create a JSFunction* through JS_NewFunction that knows how to call a python function. diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 6b1c95b7..03424142 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -98,10 +98,10 @@ static PyObject *futureOnDoneCallback(PyObject *futureCallbackTuple, PyObject *a return NULL; } else if (exception == Py_None) { // no exception set on this awaitable, safe to get result, otherwise the exception will be raised when calling `futureObj.result()` PyObject *result = future.getResult(); - JS::ResolvePromise(cx, promise, JS::RootedValue(cx, jsTypeFactory(cx, result))); + JS::ResolvePromise(cx, promise, JS::RootedValue(cx, jsTypeFactorySafe(cx, result))); Py_DECREF(result); } else { // having exception set, to reject the promise - JS::RejectPromise(cx, promise, JS::RootedValue(cx, jsTypeFactory(cx, exception))); + JS::RejectPromise(cx, promise, JS::RootedValue(cx, jsTypeFactorySafe(cx, exception))); } Py_XDECREF(exception); // cleanup diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index c607cea8..a269afb0 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -130,7 +130,7 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { Py_DECREF(argspec); Py_DECREF(args); } - + JSFunction *jsFunc = js::NewFunctionWithReserved(cx, callPyFunc, nargs, 0, NULL); JSObject *jsFuncObject = JS_GetFunctionObject(jsFunc); @@ -159,6 +159,17 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { } +JS::Value jsTypeFactorySafe(JSContext *cx, PyObject *object) { + JS::Value v = jsTypeFactory(cx, object); + PyObject *err = PyErr_Occurred(); // borrowed reference, don't need Py_DECREF() + if (err) { + PyErr_Clear(); // guarantees no error would be set on Python's error stack + PyErr_WarnEx(PyExc_RuntimeWarning, "jsTypeFactory raises an error", 1); + v.setNull(); + } + return v; +} + bool callPyFunc(JSContext *cx, unsigned int argc, JS::Value *vp) { JS::CallArgs callargs = JS::CallArgsFromVp(argc, vp); From 5f9e83a2cb825930a1224642d866f3038c238762 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 17 Apr 2023 15:43:01 +0000 Subject: [PATCH 30/73] feat(jsTypeFactory): jsTypeFactorySafe to warn the actual error string set --- src/jsTypeFactory.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index a269afb0..cec4e207 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -161,11 +161,13 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { JS::Value jsTypeFactorySafe(JSContext *cx, PyObject *object) { JS::Value v = jsTypeFactory(cx, object); - PyObject *err = PyErr_Occurred(); // borrowed reference, don't need Py_DECREF() - if (err) { - PyErr_Clear(); // guarantees no error would be set on Python's error stack - PyErr_WarnEx(PyExc_RuntimeWarning, "jsTypeFactory raises an error", 1); + if (PyErr_Occurred()) { + // Convert the Python error to a warning + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); // also clears Python's error stack + PyErr_WarnEx(PyExc_RuntimeWarning, PyUnicode_AsUTF8(value), 1); v.setNull(); + Py_XDECREF(type); Py_XDECREF(value); Py_XDECREF(traceback); } return v; } From 7e9c393a843ea8fd4adf3a843305b066ad3f8d06 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 17 Apr 2023 21:23:18 +0000 Subject: [PATCH 31/73] fix(promise): accessing to arbitrary unsafe memory locations on `onResolved` function obj, extended slot id must be less than 2 https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/JSFunction.h#l866 --- src/PromiseType.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 03424142..d8cbd3b5 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -25,8 +25,9 @@ #include -#define PY_FUTURE_OBJ_SLOT 20 // (arbitrarily chosen) slot id to access the python object in JS callbacks -#define PROMISE_OBJ_SLOT 21 +#define PY_FUTURE_OBJ_SLOT 0 // slot id to access the python object in JS callbacks +#define PROMISE_OBJ_SLOT 1 +// slot id must be less than 2 (https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/JSFunction.h#l866), otherwise it will access to arbitrary unsafe memory locations static bool onResolvedCb(JSContext *cx, unsigned argc, JS::Value *vp) { JS::CallArgs args = JS::CallArgsFromVp(argc, vp); From c91ce695dee4bd571d727fe10303435f47de8b32 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 17 Apr 2023 22:37:50 +0000 Subject: [PATCH 32/73] feat(exception-propagation): Python Exception object to JS Error object coercion --- include/ExceptionType.hh | 7 +++++++ src/ExceptionType.cc | 24 ++++++++++++++++++++++++ src/jsTypeFactory.cc | 5 +++++ 3 files changed, 36 insertions(+) diff --git a/include/ExceptionType.hh b/include/ExceptionType.hh index ead2304d..00174876 100644 --- a/include/ExceptionType.hh +++ b/include/ExceptionType.hh @@ -35,6 +35,13 @@ public: ExceptionType(JSContext *cx, JS::HandleObject error); const TYPE returnType = TYPE::EXCEPTION; + + /** + * @brief Convert a python [*Exception object](https://docs.python.org/3/c-api/exceptions.html#standard-exceptions) to JS Error object + * + * @param cx - javascript context pointer + */ + JSObject *toJsError(JSContext *cx); protected: virtual void print(std::ostream &os) const override; }; diff --git a/src/ExceptionType.cc b/src/ExceptionType.cc index e0a6a7e6..c5d6fb91 100644 --- a/src/ExceptionType.cc +++ b/src/ExceptionType.cc @@ -23,3 +23,27 @@ ExceptionType::ExceptionType(JSContext *cx, JS::HandleObject error) { } void ExceptionType::print(std::ostream &os) const {} + +// TODO (Tom Tang): preserve the original Python exception object somewhere in the JS obj for lossless two-way conversion +JSObject *ExceptionType::toJsError(JSContext *cx) { + PyObject *pyErrType = PyObject_Type(pyObject); + const char *pyErrTypeName = _PyType_Name((PyTypeObject *)pyErrType); + PyObject *pyErrMsg = PyObject_Str(pyObject); + // TODO (Tom Tang): Convert Python traceback and set it as the `stack` property on JS Error object + // PyObject *traceback = PyException_GetTraceback(pyObject); + + std::stringstream msgStream; + msgStream << "Python " << pyErrTypeName << ": " << PyUnicode_AsUTF8(pyErrMsg); + std::string msg = msgStream.str(); + + JS::RootedValue rval(cx); + JS::RootedObject stack(cx); + JS::RootedString filename(cx, JS_GetEmptyString(cx)); + JS::RootedString message(cx, JS_NewStringCopyZ(cx, msg.c_str())); + JS::CreateError(cx, JSExnType::JSEXN_ERR, stack, filename, 0, 0, nullptr, message, JS::NothingHandleValue, &rval); + + Py_DECREF(pyErrType); + Py_DECREF(pyErrMsg); + + return rval.toObjectOrNull(); +} diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index cec4e207..28747c14 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -18,6 +18,7 @@ #include "include/StrType.hh" #include "include/IntType.hh" #include "include/PromiseType.hh" +#include "include/ExceptionType.hh" #include #include @@ -139,6 +140,10 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { returnType.setObject(*jsFuncObject); memoizePyTypeAndGCThing(new FuncType(object), returnType); } + else if (PyExceptionInstance_Check(object)) { + JSObject *error = ExceptionType(object).toJsError(cx); + returnType.setObject(*error); + } else if (object == Py_None) { returnType.setUndefined(); } From 66d32df4f539e30919a7bb1b2271d6cb10d1192c Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 17 Apr 2023 22:39:16 +0000 Subject: [PATCH 33/73] fix(jsTypeFactory): for `jsTypeFactorySafe`, the error value may not be a Python str --- src/jsTypeFactory.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index 28747c14..001d83de 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -170,9 +170,12 @@ JS::Value jsTypeFactorySafe(JSContext *cx, PyObject *object) { // Convert the Python error to a warning PyObject *type, *value, *traceback; PyErr_Fetch(&type, &value, &traceback); // also clears Python's error stack - PyErr_WarnEx(PyExc_RuntimeWarning, PyUnicode_AsUTF8(value), 1); - v.setNull(); + PyObject *msg = PyObject_Str(value); + PyErr_WarnEx(PyExc_RuntimeWarning, PyUnicode_AsUTF8(msg), 1); + Py_DECREF(msg); Py_XDECREF(type); Py_XDECREF(value); Py_XDECREF(traceback); + // Return JS `null` on error + v.setNull(); } return v; } From 5780112550dc3d55415afa2ffcf53e2839bbe35f Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 18 Apr 2023 14:15:11 +0000 Subject: [PATCH 34/73] test(exception-propagation): write tests for Python Exception to JS Error object coercion --- tests/python/test_pythonmonkey_eval.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index ad12c84b..9723db4d 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -429,6 +429,13 @@ def test_eval_exceptions(): with pytest.raises(pm.SpiderMonkeyError, match="RangeError: to be raised in Python"): raise js_err + # convert Python Exception object to a JS Error object + get_err_msg = pm.eval("(err) => err.message") + assert "Python BufferError: ttt" == get_err_msg(BufferError("ttt")) + js_rethrow = pm.eval("(err) => { throw err }") + with pytest.raises(pm.SpiderMonkeyError, match="Error: Python BaseException: 123"): + js_rethrow(BaseException("123")) + def test_eval_undefined(): x = pm.eval("undefined") assert x == None From 186ff7a56bc52404fd65acf9290b4e42450c8588 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 18 Apr 2023 15:16:22 +0000 Subject: [PATCH 35/73] test(event-loop): write tests for `setTimeout` and `clearTimeout` --- tests/python/test_pythonmonkey_eval.py | 41 +++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 9723db4d..4d49bf24 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -4,6 +4,7 @@ import random from datetime import datetime, timedelta import math +import asyncio def test_passes(): assert True @@ -640,4 +641,42 @@ def concatenate(a, b): for j in range(length2): codepoint = random.randint(0x0000, 0xFFFF) string2 += chr(codepoint) - assert caller(concatenate, string1, string2) == string1 + string2 \ No newline at end of file + assert caller(concatenate, string1, string2) == string1 + string2 + +def test_set_clear_timeout(): + async def async_fn(): + # standalone `setTimeout` + loop = asyncio.get_running_loop() + f0 = loop.create_future() + def add(a, b, c): + f0.set_result(a + b + c) + pm.eval("setTimeout")(add, 0, 1, 2, 3) + assert 6.0 == await f0 + + # test `clearTimeout` + f1 = loop.create_future() + def to_raise(msg): + f1.set_exception(TypeError(msg)) + timeout_id0 = pm.eval("setTimeout")(to_raise, 100, "going to be there") + assert type(timeout_id0) == float + assert timeout_id0 > 0 # `setTimeout` should return a positive integer value + assert int(timeout_id0) == timeout_id0 + with pytest.raises(TypeError, match="going to be there"): + await f1 # `clearTimeout` not called + f1 = loop.create_future() + timeout_id1 = pm.eval("setTimeout")(to_raise, 100, "shouldn't be here") + pm.eval("clearTimeout")(timeout_id1) + with pytest.raises(asyncio.exceptions.TimeoutError): + await asyncio.wait_for(f1, timeout=0.5) # `clearTimeout` is called + + # `this` value in `setTimeout` callback should be the global object, as spec-ed + assert await pm.eval("new Promise(function (resolve) { setTimeout(function(){ resolve(this == globalThis) }) })") + # `setTimeout` should allow passing additional arguments to the callback, as spec-ed + assert 3.0 == await pm.eval("new Promise((resolve) => setTimeout(function(){ resolve(arguments.length) }, 100, 90, 91, 92))") + assert 92.0 == await pm.eval("new Promise((resolve) => setTimeout((...args) => { resolve(args[2]) }, 100, 90, 91, 92))") + # TODO (Tom Tang): test `setTimeout` setting delay to 0 if < 0 + # TODO (Tom Tang): test `setTimeout` accepting string as the delay, coercing to a number like parseFloat + + # making sure the async_fn is run + return True + assert asyncio.run(async_fn()) From a49cec7b37b8f402512a6216800f51785b2fb646 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 18 Apr 2023 22:18:48 +0000 Subject: [PATCH 36/73] feat(promise): handle cancelled Python awaitable --- include/PyEventLoop.hh | 7 ++++++- src/ExceptionType.cc | 2 +- src/PromiseType.cc | 8 ++++++-- src/PyEventLoop.cc | 8 ++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index 0f611a2b..034e87d3 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -111,6 +111,12 @@ public: */ void addDoneCallback(PyObject *cb); + /** + * @brief Return True if the Future is cancelled. + * @see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.cancelled + */ + bool isCancelled(); + /** * @brief Get the result of the Future. * Would raise exception if the Future is pending, cancelled, or having an exception set. @@ -156,7 +162,6 @@ public: */ static PyEventLoop getRunningLoop(); -protected: PyObject *_loop; PyEventLoop() = delete; diff --git a/src/ExceptionType.cc b/src/ExceptionType.cc index c5d6fb91..219bf07a 100644 --- a/src/ExceptionType.cc +++ b/src/ExceptionType.cc @@ -38,7 +38,7 @@ JSObject *ExceptionType::toJsError(JSContext *cx) { JS::RootedValue rval(cx); JS::RootedObject stack(cx); - JS::RootedString filename(cx, JS_GetEmptyString(cx)); + JS::RootedString filename(cx, JS_NewStringCopyZ(cx, "[python code]")); JS::RootedString message(cx, JS_NewStringCopyZ(cx, msg.c_str())); JS::CreateError(cx, JSExnType::JSEXN_ERR, stack, filename, 0, 0, nullptr, message, JS::NothingHandleValue, &rval); diff --git a/src/PromiseType.cc b/src/PromiseType.cc index d8cbd3b5..5a8383e9 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -95,8 +95,12 @@ static PyObject *futureOnDoneCallback(PyObject *futureCallbackTuple, PyObject *a PyObject *exception = future.getException(); if (exception == NULL || PyErr_Occurred()) { // awaitable is cancelled, `futureObj.exception()` raises a CancelledError - // TODO (Tom Tang): get bool future.isCancelled(), and reject the promise with a CancelledError - return NULL; + // Reject the promise with the CancelledError, or very unlikely, an InvalidStateError exception if the Future isn’t done yet + // see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.exception + PyObject *errType, *errValue, *traceback; + PyErr_Fetch(&errType, &errValue, &traceback); // also clears the Python error stack + JS::RejectPromise(cx, promise, JS::RootedValue(cx, jsTypeFactorySafe(cx, errValue))); + Py_XDECREF(errType); Py_XDECREF(errValue); Py_XDECREF(traceback); } else if (exception == Py_None) { // no exception set on this awaitable, safe to get result, otherwise the exception will be raised when calling `futureObj.result()` PyObject *result = future.getResult(); JS::ResolvePromise(cx, promise, JS::RootedValue(cx, jsTypeFactorySafe(cx, result))); diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index 4106b07d..6ec5e7ad 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -85,6 +85,14 @@ void PyEventLoop::Future::addDoneCallback(PyObject *cb) { Py_XDECREF(ret); } +bool PyEventLoop::Future::isCancelled() { + // https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.cancelled + PyObject *ret = PyObject_CallMethod(_future, "cancelled", NULL); // returns Python bool + bool cancelled = ret == Py_True; + Py_XDECREF(ret); + return cancelled; +} + PyObject *PyEventLoop::Future::getResult() { // https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.result return PyObject_CallMethod(_future, "result", NULL); From 432e9bf39e95681646712fa488271a908dfec32f Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 19 Apr 2023 19:08:01 +0000 Subject: [PATCH 37/73] fix(promise): the final GC for Python asyncio.Future objects coercion ``` Segmentation fault (core dumped) Hint: address points to the zero page. in PyObject_GC_IsFinalized in handleSharedPythonMonkeyMemory(JSContext*, JSGCStatus, JS::GCReason, void*) ``` --- src/PromiseType.cc | 4 ++-- src/jsTypeFactory.cc | 10 ++++++---- src/modules/pythonmonkey/pythonmonkey.cc | 10 +++++++--- src/setSpiderMonkeyException.cc | 1 + 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 5a8383e9..8db10b3f 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -87,7 +87,7 @@ void PromiseType::print(std::ostream &os) const {} // Callback to resolve or reject the JS Promise when the Future is done static PyObject *futureOnDoneCallback(PyObject *futureCallbackTuple, PyObject *args) { JSContext *cx = (JSContext *)PyLong_AsLongLong(PyTuple_GetItem(futureCallbackTuple, 0)); - JS::RootedObject *rootedPtr = (JS::RootedObject *)PyLong_AsLongLong(PyTuple_GetItem(futureCallbackTuple, 1)); + auto rootedPtr = (JS::PersistentRooted *)PyLong_AsLongLong(PyTuple_GetItem(futureCallbackTuple, 1)); JS::HandleObject promise = *rootedPtr; PyObject *futureObj = PyTuple_GetItem(args, 0); // the callback is called with the Future object as its only argument // see https://docs.python.org/3.9/library/asyncio-future.html#asyncio.Future.add_done_callback @@ -127,7 +127,7 @@ JSObject *PromiseType::toJsPromise(JSContext *cx) { PyEventLoop::Future future = loop.ensureFuture(pyObject); // Resolve or Reject the JS Promise once the python awaitable is done - JS::RootedObject *rootedPtr = new JS::RootedObject(cx, promise); // `promise` is required to be rooted from here to the end of onDoneCallback + JS::PersistentRooted *rootedPtr = new JS::PersistentRooted(cx, promise); // `promise` is required to be rooted from here to the end of onDoneCallback PyObject *futureCallbackTuple = Py_BuildValue("(ll)", (uint64_t)cx, (uint64_t)rootedPtr); PyObject *onDoneCb = PyCFunction_New(&futureCallbackDef, futureCallbackTuple); future.addDoneCallback(onDoneCb); diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index 001d83de..f8314372 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -99,10 +99,13 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { case (PyUnicode_1BYTE_KIND): { JSString *str = JS_NewExternalString(cx, (char16_t *)PyUnicode_1BYTE_DATA(object), PyUnicode_GET_LENGTH(object), &PythonExternalStringCallbacks); - /* @TODO (Caleb Aikens) this is a hack to set the JSString::LATIN1_CHARS_BIT, because there isnt an API for latin1 JSExternalStrings. + /* TODO (Caleb Aikens): this is a hack to set the JSString::LATIN1_CHARS_BIT, because there isnt an API for latin1 JSExternalStrings. * Ideally we submit a patch to Spidermonkey to make this part of their API with the following signature: * JS_NewExternalString(JSContext *cx, const char *chars, size_t length, const JSExternalStringCallbacks *callbacks) */ + // FIXME: JSExternalString are all treated as two-byte strings when GCed + // see https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/StringType-inl.h#l514 + // https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/StringType.h#l1808 *(std::atomic *)str |= 512; returnType.setString(str); break; @@ -151,11 +154,10 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { returnType.setNull(); } else if (PythonAwaitable_Check(object)) { - auto p = new PromiseType(object); // FIXME (Tom Tang): get rid of `new`. The real problem is that we don't want `~PromiseType` to be called because it decreases `object`'s ref count to 0 + PromiseType *p = new PromiseType(object); JSObject *promise = p->toJsPromise(cx); returnType.setObject(*promise); - // FIXME (Tom Tang): how to tell Python to GC the object once JS is done with the Promise? - // memoizePyTypeAndGCThing(p, returnType); + memoizePyTypeAndGCThing(p, returnType); } else { PyErr_SetString(PyExc_TypeError, "Python types other than bool, function, int, pythonmonkey.bigint, pythonmonkey.null, float, str, and None are not supported by pythonmonkey yet."); diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 6aecab34..93861aca 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -61,11 +61,11 @@ static PyTypeObject BigIntType = { }; static void cleanup() { - if (GLOBAL_CX) JS_DestroyContext(GLOBAL_CX); - JS_ShutDown(); - delete global; delete autoRealm; + delete global; delete JOB_QUEUE; + if (GLOBAL_CX) JS_DestroyContext(GLOBAL_CX); + JS_ShutDown(); } void memoizePyTypeAndGCThing(PyType *pyType, JS::Handle GCThing) { @@ -144,6 +144,7 @@ static PyObject *eval(PyObject *self, PyObject *args) { setSpiderMonkeyException(GLOBAL_CX); return NULL; } + delete code; // evaluate source code JS::Rooted *rval = new JS::Rooted(GLOBAL_CX); @@ -155,6 +156,9 @@ static PyObject *eval(PyObject *self, PyObject *args) { // translate to the proper python type PyType *returnValue = pyTypeFactory(GLOBAL_CX, global, rval); + // TODO: Find a better way to destroy the root when necessary (when the returned Python object is GCed). + // delete rval; // rval may be a JS function which must be kept alive. + if (returnValue) { return returnValue->getPyObject(); } diff --git a/src/setSpiderMonkeyException.cc b/src/setSpiderMonkeyException.cc index 21e3f73d..d207a517 100644 --- a/src/setSpiderMonkeyException.cc +++ b/src/setSpiderMonkeyException.cc @@ -81,6 +81,7 @@ void setSpiderMonkeyException(JSContext *cx) { PyErr_SetString(SpiderMonkeyError, "Spidermonkey set an exception, but was unable to retrieve it."); return; } + JS_ClearPendingException(cx); // `PyErr_SetString` uses `PyErr_SetObject` with `PyUnicode_FromString` under the hood // see https://github.com/python/cpython/blob/3.9/Python/errors.c#L234-L236 From 3a60f261a5a5d33404de75c271c25f559443a88c Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 19 Apr 2023 20:52:09 +0000 Subject: [PATCH 38/73] fix(promise): GC for Python asyncio.Future objects coercion --- src/jsTypeFactory.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index f8314372..2a6e9d48 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -157,7 +157,8 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { PromiseType *p = new PromiseType(object); JSObject *promise = p->toJsPromise(cx); returnType.setObject(*promise); - memoizePyTypeAndGCThing(p, returnType); + // nested awaitables would have already been GCed if finished + // memoizePyTypeAndGCThing(p, returnType); } else { PyErr_SetString(PyExc_TypeError, "Python types other than bool, function, int, pythonmonkey.bigint, pythonmonkey.null, float, str, and None are not supported by pythonmonkey yet."); From fd70e7b01a36697f4d7338ae385eb3d9ed3d9421 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 19 Apr 2023 21:09:07 +0000 Subject: [PATCH 39/73] fix(GC): python object would have already been deallocated, `pyObj->ob_type` became an invalid pointer --- src/jsTypeFactory.cc | 3 +-- src/modules/pythonmonkey/pythonmonkey.cc | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index 2a6e9d48..f8314372 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -157,8 +157,7 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { PromiseType *p = new PromiseType(object); JSObject *promise = p->toJsPromise(cx); returnType.setObject(*promise); - // nested awaitables would have already been GCed if finished - // memoizePyTypeAndGCThing(p, returnType); + memoizePyTypeAndGCThing(p, returnType); } else { PyErr_SetString(PyExc_TypeError, "Python types other than bool, function, int, pythonmonkey.bigint, pythonmonkey.null, float, str, and None are not supported by pythonmonkey yet."); diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 93861aca..3025b78b 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -88,7 +88,9 @@ void handleSharedPythonMonkeyMemory(JSContext *cx, JSGCStatus status, JS::GCReas while (pyIt != PyTypeToGCThing.end()) { // If the PyObject reference count is exactly 1, then the only reference to the object is the one // we are holding, which means the object is ready to be free'd. - if (PyObject_GC_IsFinalized(pyIt->first->getPyObject()) || pyIt->first->getPyObject()->ob_refcnt == 1) { + PyObject *pyObj = pyIt->first->getPyObject(); + bool isAlive = (intptr_t)Py_TYPE(pyObj) > 1; // object would have already been deallocated, `pyObj->ob_type` became an invalid pointer (-1) + if (isAlive && (PyObject_GC_IsFinalized(pyObj) || pyObj->ob_refcnt == 1)) { for (JS::PersistentRooted *rval: pyIt->second) { // for each related GCThing bool found = false; for (PyToGCIterator innerPyIt = PyTypeToGCThing.begin(); innerPyIt != PyTypeToGCThing.end(); innerPyIt++) { // for each other PyType pointer From 8a87d21bb3381f7bad4886beeeec7a8ee3ebf1b1 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 20 Apr 2023 17:59:56 +0000 Subject: [PATCH 40/73] fix(promise): JS Promise may resolve to a JS function, so we must keep the result value alive and rooted --- src/PromiseType.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 8db10b3f..ee9b18a0 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -38,10 +38,12 @@ static bool onResolvedCb(JSContext *cx, unsigned argc, JS::Value *vp) { JS::PromiseState state = JS::GetPromiseState(promise); // Convert the Promise's result (either fulfilled resolution or rejection reason) to a Python object - JS::RootedObject thisv(cx); - // args.computeThis(cx, &thisv); // thisv is the global object, not the promise - JS::RootedValue resultArg(cx, args[0]); - PyObject *result = pyTypeFactory(cx, &thisv, &resultArg)->getPyObject(); + // FIXME (Tom Tang): memory leak, not free-ed + // The result might be another JS function, so we must keep them alive + JS::RootedObject *thisv = new JS::RootedObject(cx); + args.computeThis(cx, thisv); // thisv is the global object, not the promise + JS::RootedValue *resultArg = new JS::RootedValue(cx, args[0]); + PyObject *result = pyTypeFactory(cx, thisv, resultArg)->getPyObject(); if (state == JS::PromiseState::Rejected && !PyExceptionInstance_Check(result)) { // Wrap the result object into a SpiderMonkeyError object // because only *Exception objects can be thrown in Python `raise` statement and alike From b3cdb2a57e07f70321e40b418b53a66cb88e87c4 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 20 Apr 2023 19:44:43 +0000 Subject: [PATCH 41/73] fix(callPyFunc): Python may raise an exception, if the number of arguments passed to the python function from JS-land is not the expected argument count by the python function Would throw one of the following: * TypeError: fn() takes x positional arguments but y was given * TypeError: fn() missing x required positional argument: 'xxx' Without this fix, it would give a segfault --- src/jsTypeFactory.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index f8314372..f3bf8854 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -194,6 +194,9 @@ bool callPyFunc(JSContext *cx, unsigned int argc, JS::Value *vp) { if (!callargs.length()) { PyObject *pyRval = PyObject_CallNoArgs(pyFunc); + if (PyErr_Occurred()) { // Check if an exception has already been set in Python error stack + return false; + } // @TODO (Caleb Aikens) need to check for python exceptions here callargs.rval().set(jsTypeFactory(cx, pyRval)); return true; @@ -208,6 +211,9 @@ bool callPyFunc(JSContext *cx, unsigned int argc, JS::Value *vp) { } PyObject *pyRval = PyObject_Call(pyFunc, pyArgs, NULL); + if (PyErr_Occurred()) { + return false; + } // @TODO (Caleb Aikens) need to check for python exceptions here callargs.rval().set(jsTypeFactory(cx, pyRval)); From a7405d1ae012a4db325e2c4539a70dbd3152dde8 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 20 Apr 2023 22:42:42 +0000 Subject: [PATCH 42/73] test(promise): write tests for JS promise <-> Python awaitable coercion --- tests/python/test_pythonmonkey_eval.py | 147 +++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 4d49bf24..88243e7e 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -680,3 +680,150 @@ def to_raise(msg): # making sure the async_fn is run return True assert asyncio.run(async_fn()) + +def test_promises(): + async def async_fn(): + # Python awaitables to JS Promise coercion + # 1. Python asyncio.Future to JS promise + loop = asyncio.get_running_loop() + f0 = loop.create_future() + f0.set_result(2561) + assert type(f0) == asyncio.Future + assert 2561 == await f0 + assert pm.eval("(p) => p instanceof Promise")(f0) is True + assert 2561 == await pm.eval("(p) => p")(f0) + del f0 + + # 2. Python asyncio.Task to JS promise + async def coro_fn(x): + await asyncio.sleep(0.01) + return x + task = loop.create_task(coro_fn("from a Task")) + assert type(task) == asyncio.Task + assert type(task) != asyncio.Future + assert isinstance(task, asyncio.Future) + assert "from a Task" == await task + assert pm.eval("(p) => p instanceof Promise")(task) is True + assert "from a Task" == await pm.eval("(p) => p")(task) + del task + + # 3. Python coroutine to JS promise + coro = coro_fn("from a Coroutine") + assert asyncio.iscoroutine(coro) + # assert "a Coroutine" == await coro # coroutines cannot be awaited more than once + # assert pm.eval("(p) => p instanceof Promise")(coro) is True # RuntimeError: cannot reuse already awaited coroutine + assert "from a Coroutine" == await pm.eval("(p) => (p instanceof Promise) && p")(coro) + del coro + + # JS Promise to Python awaitable coercion + assert 100 == await pm.eval("new Promise((r)=>{ r(100) })") + assert 10010 == await pm.eval("Promise.resolve(10010)") + with pytest.raises(pm.SpiderMonkeyError, match="TypeError: .+ is not a constructor"): + await pm.eval("Promise.resolve")(10086) + assert 10086 == await pm.eval("Promise.resolve.bind(Promise)")(10086) + + assert "promise returning a function" == (await pm.eval("Promise.resolve(() => { return 'promise returning a function' })"))() + assert "function 2" == (await pm.eval("Promise.resolve(x=>x)"))("function 2") + def aaa(n): + return n + ident0 = await (pm.eval("Promise.resolve.bind(Promise)")(aaa)) + assert "from aaa" == ident0("from aaa") + ident1 = await pm.eval("async (aaa) => x=>aaa(x)")(aaa) + assert "from ident1" == ident1("from ident1") + ident2 = await pm.eval("() => Promise.resolve(x=>x)")() + assert "from ident2" == ident2("from ident2") + ident3 = await pm.eval("(aaa) => Promise.resolve(x=>aaa(x))")(aaa) + assert "from ident3" == ident3("from ident3") + del aaa + + # promise returning a JS Promise that calls a Python function inside + def fn0(n): + return n + 100 + def fn1(): + return pm.eval("async x=>x")(fn0) + fn2 = await pm.eval("async (fn1) => { const fn0 = await fn1(); return Promise.resolve(x=>fn0(x)) }")(fn1) + assert 101.2 == fn2(1.2) + fn3 = await pm.eval("async (fn1) => { const fn0 = await fn1(); return Promise.resolve(async x => { return fn0(x) }) }")(fn1) + assert 101.3 == await fn3(1.3) + fn4 = await pm.eval("async (fn1) => { return Promise.resolve(async x => { const fn0 = await fn1(); return fn0(x) }) }")(fn1) + assert 101.4 == await fn4(1.4) + + # chained JS promises + assert "chained" == await (pm.eval("async () => new Promise((resolve) => resolve( Promise.resolve().then(()=>'chained') ))")()) + + # chained Python awaitables + async def a(): + await asyncio.sleep(0.01) + return "nested" + async def b(): + await asyncio.sleep(0.01) + return a() + async def c(): + await asyncio.sleep(0.01) + return b() + # JS `await` supports chaining. However, on Python-land, it actually requires `await (await (await c()))` + assert "nested" == await pm.eval("async (promise) => await promise")(c()) + assert "nested" == await pm.eval("async (promise) => await promise")(await c()) + assert "nested" == await pm.eval("async (promise) => await promise")(await (await c())) + assert "nested" == await pm.eval("async (promise) => await promise")(await (await (await c()))) + assert "nested" == await pm.eval("async (promise) => promise")(c()) + assert "nested" == await pm.eval("async (promise) => promise")(await c()) + assert "nested" == await pm.eval("async (promise) => promise")(await (await c())) + assert "nested" == await pm.eval("async (promise) => promise")(await (await (await c()))) + assert "nested" == await pm.eval("(promise) => Promise.resolve(promise)")(c()) + assert "nested" == await pm.eval("(promise) => Promise.resolve(promise)")(await c()) + assert "nested" == await pm.eval("(promise) => Promise.resolve(promise)")(await (await c())) + assert "nested" == await pm.eval("(promise) => Promise.resolve(promise)")(await (await (await c()))) + assert "nested" == await pm.eval("(promise) => promise")(c()) + assert "nested" == await pm.eval("(promise) => promise")(await c()) + assert "nested" == await pm.eval("(promise) => promise")(await (await c())) + with pytest.raises(TypeError, match="object str can't be used in 'await' expression"): + await pm.eval("(promise) => promise")(await (await (await c()))) + + # Python awaitable throwing exceptions + async def coro_to_throw0(): + await asyncio.sleep(0.01) + print([].non_exist) + with pytest.raises(pm.SpiderMonkeyError, match="Python AttributeError: 'list' object has no attribute 'non_exist'"): + await (pm.eval("(promise) => promise")(coro_to_throw0())) + with pytest.raises(pm.SpiderMonkeyError, match="Python AttributeError: 'list' object has no attribute 'non_exist'"): + await (pm.eval("async (promise) => promise")(coro_to_throw0())) + with pytest.raises(pm.SpiderMonkeyError, match="Python AttributeError: 'list' object has no attribute 'non_exist'"): + await (pm.eval("(promise) => Promise.resolve().then(async () => await promise)")(coro_to_throw0())) + async def coro_to_throw1(): + await asyncio.sleep(0.01) + raise TypeError("reason") + with pytest.raises(pm.SpiderMonkeyError, match="Python TypeError: reason"): + await (pm.eval("(promise) => promise")(coro_to_throw1())) + assert 'rejected ' == await pm.eval("(promise) => promise.then(()=>{}, (err)=>`rejected <${err.message}>`)")(coro_to_throw1()) + + # JS Promise throwing exceptions + with pytest.raises(pm.SpiderMonkeyError, match="nan"): + await pm.eval("Promise.reject(NaN)") # JS can throw anything + with pytest.raises(pm.SpiderMonkeyError, match="123.0"): + await (pm.eval("async () => { throw 123 }")()) + # await (pm.eval("async () => { throw {} }")()) + with pytest.raises(pm.SpiderMonkeyError, match="anything"): + await pm.eval("Promise.resolve().then(()=>{ throw 'anything' })") + # FIXME (Tom Tang): We currently handle Promise exceptions by converting the object thrown to a Python Exception object through `pyTypeFactory` + # + # await pm.eval("Promise.resolve().then(()=>{ throw {a:1,toString(){return'anything'}} })") + with pytest.raises(pm.SpiderMonkeyError, match="on line 1:\nTypeError: undefined has no properties"): # not going through the conversion + await pm.eval("Promise.resolve().then(()=>{ (undefined).prop })") + + # TODO (Tom Tang): Modify this testcase once we support ES2020-style dynamic import + pm.eval("import('some_module')") # dynamic import returns a Promise, see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/import + with pytest.raises(pm.SpiderMonkeyError, match="\nError: Dynamic module import is disabled or not supported in this context"): + await pm.eval("import('some_module')") + + # await scheduled jobs on the Python event-loop + js_sleep = pm.eval("(second) => new Promise((resolve) => setTimeout(resolve, second*1000))") + py_sleep = asyncio.sleep + both_sleep = pm.eval("(js_sleep, py_sleep) => async (second) => { await js_sleep(second); await py_sleep(second) }")(js_sleep, py_sleep) + await asyncio.wait_for(both_sleep(0.1), timeout=0.21) + with pytest.raises(asyncio.exceptions.TimeoutError): + await asyncio.wait_for(both_sleep(0.1), timeout=0.19) + + # making sure the async_fn is run + return True + assert asyncio.run(async_fn()) From 0468e9537010f29b487b77c7d0507f07cf17cf59 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Fri, 21 Apr 2023 00:21:22 +0000 Subject: [PATCH 43/73] Revert "fix(GC): python object would have already been deallocated, `pyObj->ob_type` became an invalid pointer" This reverts commit fd70e7b01a36697f4d7338ae385eb3d9ed3d9421. --- src/jsTypeFactory.cc | 3 ++- src/modules/pythonmonkey/pythonmonkey.cc | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index f3bf8854..d8187ce4 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -157,7 +157,8 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { PromiseType *p = new PromiseType(object); JSObject *promise = p->toJsPromise(cx); returnType.setObject(*promise); - memoizePyTypeAndGCThing(p, returnType); + // nested awaitables would have already been GCed if finished + // memoizePyTypeAndGCThing(p, returnType); } else { PyErr_SetString(PyExc_TypeError, "Python types other than bool, function, int, pythonmonkey.bigint, pythonmonkey.null, float, str, and None are not supported by pythonmonkey yet."); diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 3025b78b..93861aca 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -88,9 +88,7 @@ void handleSharedPythonMonkeyMemory(JSContext *cx, JSGCStatus status, JS::GCReas while (pyIt != PyTypeToGCThing.end()) { // If the PyObject reference count is exactly 1, then the only reference to the object is the one // we are holding, which means the object is ready to be free'd. - PyObject *pyObj = pyIt->first->getPyObject(); - bool isAlive = (intptr_t)Py_TYPE(pyObj) > 1; // object would have already been deallocated, `pyObj->ob_type` became an invalid pointer (-1) - if (isAlive && (PyObject_GC_IsFinalized(pyObj) || pyObj->ob_refcnt == 1)) { + if (PyObject_GC_IsFinalized(pyIt->first->getPyObject()) || pyIt->first->getPyObject()->ob_refcnt == 1) { for (JS::PersistentRooted *rval: pyIt->second) { // for each related GCThing bool found = false; for (PyToGCIterator innerPyIt = PyTypeToGCThing.begin(); innerPyIt != PyTypeToGCThing.end(); innerPyIt++) { // for each other PyType pointer From 5a2cbb41bd1a10872d3e10170fb0c6fb7db50d35 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 24 Apr 2023 13:25:45 +0000 Subject: [PATCH 44/73] refactor(event-loop): the `_loop` member in `PyEventLoop` struct should be protected --- include/PyEventLoop.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index 034e87d3..e224f850 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -162,6 +162,7 @@ public: */ static PyEventLoop getRunningLoop(); +protected: PyObject *_loop; PyEventLoop() = delete; From 471ce4b47c16f087637e4b55709cb105cdfd910b Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 24 Apr 2023 18:12:03 +0000 Subject: [PATCH 45/73] refactor(event-loop): use a timeoutID pool instead of casting a raw PyObject pointer as the timeoutID * prevents arbitrary memory access * matches the behaviour in browsers where `setTimeout` returns a small integer --- include/PyEventLoop.hh | 31 +++++++++++++++--------- src/modules/pythonmonkey/pythonmonkey.cc | 4 +-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index e224f850..30d45b62 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -13,6 +13,7 @@ #define PythonMonkey_PyEventLoop_ #include +#include struct PyEventLoop { public: @@ -30,9 +31,13 @@ public: */ struct AsyncHandle { public: - AsyncHandle(PyObject *handle) : _handle(handle) {}; + explicit AsyncHandle(PyObject *handle) : _handle(handle) {}; + AsyncHandle(const AsyncHandle &old) = delete; // forbid copy-initialization + AsyncHandle(AsyncHandle &&old) : _handle(std::exchange(old._handle, nullptr)) {}; // clear the moved-from object ~AsyncHandle() { - Py_XDECREF(_handle); + if (Py_IsInitialized()) { // the Python runtime has already been finalized when `_timeoutIdMap` is cleared at exit + Py_XDECREF(_handle); + } } /** @@ -45,16 +50,13 @@ public: * @brief Get the unique `timeoutID` for JS `setTimeout`/`clearTimeout` methods * @see https://developer.mozilla.org/en-US/docs/Web/API/setTimeout#return_value */ - inline uint64_t getId() { - Py_INCREF(_handle); // otherwise the object would be GC-ed as the AsyncHandle destructs - // Currently we use the address of the underlying `asyncio.Handle` object - // FIXME (Tom Tang): JS stores the `timeoutID` in a `number` (float64), may overflow - return (uint64_t)_handle; + static inline uint32_t getUniqueId(AsyncHandle &&handle) { + // TODO (Tom Tang): mutex lock + _timeoutIdMap.push_back(std::move(handle)); + return _timeoutIdMap.size() - 1; // the index in `_timeoutIdMap` } - static inline AsyncHandle fromId(uint64_t timeoutID) { - // FIXME: user can access arbitrary memory location - // FIXME (Tom Tang): `clearTimeout` can only be applied once on the same handle because the handle is GC-ed - return AsyncHandle((PyObject *)timeoutID); + static inline AsyncHandle &fromId(uint32_t timeoutID) { + return _timeoutIdMap.at(timeoutID); } /** @@ -66,6 +68,9 @@ public: } protected: PyObject *_handle; + + // TODO (Tom Tang): use separate pools of IDs for different global objects + static inline std::vector _timeoutIdMap; }; /** @@ -88,7 +93,9 @@ public: */ struct Future { public: - Future(PyObject *future) : _future(future) {}; + explicit Future(PyObject *future) : _future(future) {}; + Future(const Future &old) = delete; // forbid copy-initialization + Future(Future &&old) : _future(std::exchange(old._future, nullptr)) {}; // clear the moved-from object ~Future() { Py_XDECREF(_future); } diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 93861aca..a01ab37c 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -223,7 +223,7 @@ static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { PyEventLoop::AsyncHandle handle = loop.enqueueWithDelay(job, delaySeconds); // Return the `timeoutID` to use in `clearTimeout` - args.rval().setDouble((double)handle.getId()); + args.rval().setDouble((double)PyEventLoop::AsyncHandle::getUniqueId(std::move(handle))); return true; } @@ -237,7 +237,7 @@ static bool clearTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { // Retrieve the AsyncHandle by `timeoutID` double timeoutID = args[0].toNumber(); - AsyncHandle handle = AsyncHandle::fromId((uint64_t)timeoutID); + AsyncHandle &handle = AsyncHandle::fromId((uint32_t)timeoutID); // Cancel this job on Python event-loop handle.cancel(); From b155c8c1a08f74fca6af68fc71cd9c7038ba0c48 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 24 Apr 2023 19:18:39 +0000 Subject: [PATCH 46/73] fix(event-loop): no exception should be thrown when an invalid timeoutID is passed to `clearTimeout`, silently do nothing instead --- include/PyEventLoop.hh | 8 ++++++-- src/modules/pythonmonkey/pythonmonkey.cc | 15 ++++++++++++--- tests/python/test_pythonmonkey_eval.py | 8 ++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index 30d45b62..1c9eac33 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -55,8 +55,12 @@ public: _timeoutIdMap.push_back(std::move(handle)); return _timeoutIdMap.size() - 1; // the index in `_timeoutIdMap` } - static inline AsyncHandle &fromId(uint32_t timeoutID) { - return _timeoutIdMap.at(timeoutID); + static inline AsyncHandle *fromId(uint32_t timeoutID) { + try { + return &_timeoutIdMap.at(timeoutID); + } catch (...) { // std::out_of_range& + return nullptr; // invalid timeoutID + } } /** diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index a01ab37c..b82bfde5 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -234,13 +234,22 @@ static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { static bool clearTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { using AsyncHandle = PyEventLoop::AsyncHandle; JS::CallArgs args = JS::CallArgsFromVp(argc, vp); + JS::HandleValue timeoutIdArg = args.get(0); + + args.rval().setUndefined(); + + // silently does nothing when an invalid timeoutID is passed in + if (!timeoutIdArg.isNumber()) { + return true; + } // Retrieve the AsyncHandle by `timeoutID` - double timeoutID = args[0].toNumber(); - AsyncHandle &handle = AsyncHandle::fromId((uint32_t)timeoutID); + double timeoutID = timeoutIdArg.toNumber(); + AsyncHandle *handle = AsyncHandle::fromId((uint32_t)timeoutID); + if (!handle) return true; // does nothing on invalid timeoutID // Cancel this job on Python event-loop - handle.cancel(); + handle->cancel(); return true; } diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 88243e7e..dc79b5b6 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -677,6 +677,14 @@ def to_raise(msg): # TODO (Tom Tang): test `setTimeout` setting delay to 0 if < 0 # TODO (Tom Tang): test `setTimeout` accepting string as the delay, coercing to a number like parseFloat + # passing an invalid ID to `clearTimeout` should silently do nothing; no exception is thrown. + pm.eval("clearTimeout(NaN)") + pm.eval("clearTimeout(999)") + pm.eval("clearTimeout(-1)") + pm.eval("clearTimeout('a')") + pm.eval("clearTimeout(undefined)") + pm.eval("clearTimeout()") + # making sure the async_fn is run return True assert asyncio.run(async_fn()) From b064d87586675df6d14ad9e852b2a0c2f5da8e7f Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 24 Apr 2023 19:26:00 +0000 Subject: [PATCH 47/73] =?UTF-8?q?fix(event-loop):=20`std::exchange`=20comp?= =?UTF-8?q?ile-time=20error:=20=E2=80=98exchange=E2=80=99=20is=20not=20a?= =?UTF-8?q?=20member=20of=20=E2=80=98std=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/PyEventLoop.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index 1c9eac33..1a954e69 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -14,6 +14,7 @@ #include #include +#include struct PyEventLoop { public: From ee75a56b6cc8cd5992eaf3d246db884935171629 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 24 Apr 2023 19:36:40 +0000 Subject: [PATCH 48/73] fix(event-loop): the timeoutID should always be an integer --- src/modules/pythonmonkey/pythonmonkey.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index b82bfde5..5298c0be 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -239,12 +239,12 @@ static bool clearTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { args.rval().setUndefined(); // silently does nothing when an invalid timeoutID is passed in - if (!timeoutIdArg.isNumber()) { + if (!timeoutIdArg.isInt32()) { return true; } // Retrieve the AsyncHandle by `timeoutID` - double timeoutID = timeoutIdArg.toNumber(); + int32_t timeoutID = timeoutIdArg.toInt32(); AsyncHandle *handle = AsyncHandle::fromId((uint32_t)timeoutID); if (!handle) return true; // does nothing on invalid timeoutID From 5f0c2bbbf54cdc80f9661328e25827c96433a9c3 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 24 Apr 2023 21:00:34 +0000 Subject: [PATCH 49/73] feat(event-loop): throw a TypeError when the first parameter to `setTimeout` is not a function --- src/modules/pythonmonkey/pythonmonkey.cc | 14 ++++++++++++-- tests/python/test_pythonmonkey_eval.py | 8 ++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 5298c0be..6c0589da 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -191,10 +192,19 @@ PyObject *SpiderMonkeyError = NULL; static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { JS::CallArgs args = JS::CallArgsFromVp(argc, vp); + // Ensure the first parameter is a function + // We don't support passing a `code` string to `setTimeout` (yet) + JS::HandleValue jobArgVal = args.get(0); + bool jobArgIsFunction = jobArgVal.isObject() && js::IsFunctionObject(&jobArgVal.toObject()); + if (!jobArgIsFunction) { + JS_ReportErrorNumberASCII(cx, nullptr, nullptr, JSErrNum::JSMSG_NOT_FUNCTION, "The first parameter to setTimeout()"); + return false; + } + // Get the function to be executed // FIXME (Tom Tang): memory leak, not free-ed JS::RootedObject *thisv = new JS::RootedObject(cx, JS::GetNonCCWObjectGlobal(&args.callee())); // HTML spec requires `thisArg` to be the global object - JS::RootedValue *jobArg = new JS::RootedValue(cx, args[0]); + JS::RootedValue *jobArg = new JS::RootedValue(cx, jobArgVal); // `setTimeout` allows passing additional arguments to the callback, as spec-ed if (args.length() > 2) { // having additional arguments // Wrap the job function into a bound function with the given additional arguments @@ -204,7 +214,7 @@ static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { for (size_t i = 1, j = 2; j < args.length(); j++) { bindArgs.append(args[j]); } - JS::RootedObject jobArgObj = JS::RootedObject(cx, &args[0].toObject()); + JS::RootedObject jobArgObj = JS::RootedObject(cx, &jobArgVal.toObject()); JS_CallFunctionName(cx, jobArgObj, "bind", JS::HandleValueArray(bindArgs), jobArg); // jobArg = jobArg.bind(thisv, ...bindArgs) } // Convert to a Python function diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index dc79b5b6..8cc5ba76 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -685,6 +685,14 @@ def to_raise(msg): pm.eval("clearTimeout(undefined)") pm.eval("clearTimeout()") + # should throw a TypeError when the first parameter to `setTimeout` is not a function + with pytest.raises(pm.SpiderMonkeyError, match="TypeError: The first parameter to setTimeout\\(\\) is not a function"): + pm.eval("setTimeout()") + with pytest.raises(pm.SpiderMonkeyError, match="TypeError: The first parameter to setTimeout\\(\\) is not a function"): + pm.eval("setTimeout(undefined)") + with pytest.raises(pm.SpiderMonkeyError, match="TypeError: The first parameter to setTimeout\\(\\) is not a function"): + pm.eval("setTimeout(1)") + # making sure the async_fn is run return True assert asyncio.run(async_fn()) From 271a3c453828d485d78fecdc89839ff893467677 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 25 Apr 2023 19:12:00 +0000 Subject: [PATCH 50/73] feat(event-loop): schedule off-thread promises to the event-loop on main thread --- include/PyEventLoop.hh | 13 +++++- src/JobQueue.cc | 31 +++++++++++++- src/PyEventLoop.cc | 59 +++++++++++++++++++++++++- tests/python/test_pythonmonkey_eval.py | 24 +++++++++++ 4 files changed, 123 insertions(+), 4 deletions(-) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index 1a954e69..ce3f2995 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -168,17 +168,28 @@ public: Future ensureFuture(PyObject *awaitable); /** - * @brief Get the running Python event-loop, or + * @brief Get the running Python event-loop on the current thread, or * raise a Python RuntimeError if no event-loop running * @return an instance of `PyEventLoop` */ static PyEventLoop getRunningLoop(); + /** + * @brief Get the running Python event-loop on main thread, or + * raise a Python RuntimeError if no event-loop running + * @return an instance of `PyEventLoop` + */ + static PyEventLoop getMainLoop(); + protected: PyObject *_loop; PyEventLoop() = delete; PyEventLoop(PyObject *loop) : _loop(loop) {}; +private: + static PyEventLoop _mainLoopNotFound(); + static PyEventLoop _getLoopOnThread(PyThreadState *tstate); + static PyThreadState *_getMainThread(); }; #endif \ No newline at end of file diff --git a/src/JobQueue.cc b/src/JobQueue.cc index 3642f15a..c70de25c 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -60,9 +60,36 @@ bool JobQueue::init(JSContext *cx) { return true; } +static PyObject *callDispatchFunc(PyObject *dispatchFuncTuple, PyObject *Py_UNUSED(unused)) { + JSContext *cx = (JSContext *)PyLong_AsLongLong(PyTuple_GetItem(dispatchFuncTuple, 0)); + JS::Dispatchable *dispatchable = (JS::Dispatchable *)PyLong_AsLongLong(PyTuple_GetItem(dispatchFuncTuple, 1)); + dispatchable->run(cx, JS::Dispatchable::NotShuttingDown); + Py_RETURN_NONE; +} +static PyMethodDef callDispatchFuncDef = {"JsDispatchCallable", callDispatchFunc, METH_NOARGS, NULL}; + /* static */ bool JobQueue::dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable) { JSContext *cx = (JSContext *)closure; // `closure` is provided in `JS::InitDispatchToEventLoop` call - // dispatchable->run(cx, JS::Dispatchable::NotShuttingDown); - return true; + + // The `dispatchToEventLoop` function is running in a helper thread, so + // we must acquire the Python GIL (global interpreter lock) + // see https://docs.python.org/3/c-api/init.html#non-python-created-threads + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + PyObject *dispatchFuncTuple = Py_BuildValue("(ll)", (uint64_t)cx, (uint64_t)dispatchable); + PyObject *pyFunc = PyCFunction_New(&callDispatchFuncDef, dispatchFuncTuple); + + // Send job to the running Python event-loop on `cx`'s thread (the main thread) + PyEventLoop loop = PyEventLoop::getMainLoop(); + if (!loop.initialized()) { + PyErr_Print(); // Python RuntimeError is thrown if no event-loop running on main thread + PyGILState_Release(gstate); + return false; + } + loop.enqueue(pyFunc); + + PyGILState_Release(gstate); + return true; // dispatchable must eventually run } diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index 6ec5e7ad..c04588c8 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -5,7 +5,7 @@ PyEventLoop::AsyncHandle PyEventLoop::enqueue(PyObject *jobFn) { // Enqueue job to the Python event-loop // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_soon - PyObject *asyncHandle = PyObject_CallMethod(_loop, "call_soon", "O", jobFn); // https://docs.python.org/3/c-api/arg.html#c.Py_BuildValue + PyObject *asyncHandle = PyObject_CallMethod(_loop, "call_soon_threadsafe", "O", jobFn); // https://docs.python.org/3/c-api/arg.html#c.Py_BuildValue return PyEventLoop::AsyncHandle(asyncHandle); } @@ -13,6 +13,9 @@ PyEventLoop::AsyncHandle PyEventLoop::enqueueWithDelay(PyObject *jobFn, double d // Schedule job to the Python event-loop // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_later PyObject *asyncHandle = PyObject_CallMethod(_loop, "call_later", "dO", delaySeconds, jobFn); // https://docs.python.org/3/c-api/arg.html#c.Py_BuildValue + if (asyncHandle == nullptr) { + PyErr_Print(); // RuntimeError: Non-thread-safe operation invoked on an event loop other than the current one + } return PyEventLoop::AsyncHandle(asyncHandle); } @@ -44,8 +47,62 @@ PyEventLoop::Future PyEventLoop::ensureFuture(PyObject *awaitable) { return PyEventLoop::Future(futureObj); } +/* static */ +PyEventLoop PyEventLoop::_mainLoopNotFound() { + PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop on the main thread."); + return PyEventLoop(nullptr); +} + +/* static */ +PyEventLoop PyEventLoop::_getLoopOnThread(PyThreadState *tstate) { + // Modified from Python 3.9 `get_running_loop` https://github.com/python/cpython/blob/7cb3a44/Modules/_asynciomodule.c#L241-L278 + PyObject *ts_dict = _PyThreadState_GetDict(tstate); // borrowed reference + if (ts_dict == NULL) { + return _mainLoopNotFound(); + } + + PyObject *rl = PyDict_GetItemString(ts_dict, "__asyncio_running_event_loop__"); // borrowed reference + if (rl == NULL) { + return _mainLoopNotFound(); + } + + using PyRunningLoopHolder = struct { + PyObject_HEAD + PyObject *rl_loop; + }; + + PyObject *running_loop = ((PyRunningLoopHolder *)rl)->rl_loop; + if (running_loop == Py_None) { + return _mainLoopNotFound(); + } + + Py_INCREF(running_loop); + return PyEventLoop(running_loop); +} + +/* static */ +PyThreadState *PyEventLoop::_getMainThread() { + // The last element in the linked-list of threads associated with the main interpreter should be the main thread + // (The first element is the current thread, see https://github.com/python/cpython/blob/7cb3a44/Python/pystate.c#L291-L293) + PyInterpreterState *interp = PyInterpreterState_Main(); + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); // https://docs.python.org/3/c-api/init.html#c.PyInterpreterState_ThreadHead + while (PyThreadState_Next(tstate) != nullptr) { + tstate = PyThreadState_Next(tstate); + } + return tstate; +} + +/* static */ +PyEventLoop PyEventLoop::getMainLoop() { + return _getLoopOnThread(_getMainThread()); +} + /* static */ PyEventLoop PyEventLoop::getRunningLoop() { + // TODO: refactor into the C API `_getLoopOnThread(PyThreadState_Get())` + // We don't use the C API here yet because the Python method caches the PyRunningLoopHolder (Is it worth caching?), + // see https://github.com/python/cpython/blob/7cb3a44/Modules/_asynciomodule.c#L234-L239 + // Get the running Python event-loop // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_running_loop PyObject *asyncio = PyImport_ImportModule("asyncio"); diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 8cc5ba76..be1ad69d 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -692,6 +692,8 @@ def to_raise(msg): pm.eval("setTimeout(undefined)") with pytest.raises(pm.SpiderMonkeyError, match="TypeError: The first parameter to setTimeout\\(\\) is not a function"): pm.eval("setTimeout(1)") + with pytest.raises(pm.SpiderMonkeyError, match="TypeError: The first parameter to setTimeout\\(\\) is not a function"): + pm.eval("setTimeout('a', 100)") # making sure the async_fn is run return True @@ -843,3 +845,25 @@ async def coro_to_throw1(): # making sure the async_fn is run return True assert asyncio.run(async_fn()) + +def test_webassembly(): + async def async_fn(): + # off-thread promises can run + assert 'instantiated' == await pm.eval(""" + // https://github.com/mdn/webassembly-examples/blob/main/js-api-examples/simple.wasm + var code = new Uint8Array([ + 0, 97, 115, 109, 1, 0, 0, 0, 1, 8, 2, 96, + 1, 127, 0, 96, 0, 0, 2, 25, 1, 7, 105, 109, + 112, 111, 114, 116, 115, 13, 105, 109, 112, 111, 114, 116, + 101, 100, 95, 102, 117, 110, 99, 0, 0, 3, 2, 1, + 1, 7, 17, 1, 13, 101, 120, 112, 111, 114, 116, 101, + 100, 95, 102, 117, 110, 99, 0, 1, 10, 8, 1, 6, + 0, 65, 42, 16, 0, 11 + ]); + + WebAssembly.instantiate(code, { imports: { imported_func() {} } }).then(() => 'instantiated') + """) + + # making sure the async_fn is run + return True + assert asyncio.run(async_fn()) From 5113e86f33822ef6eff9ab77a4fb729cad554397 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 25 Apr 2023 20:22:58 +0000 Subject: [PATCH 51/73] refactor(event-loop): rename `PyEventLoop::_mainLoopNotFound` to `_loopNotFound` We want to generalize this method. --- include/PyEventLoop.hh | 12 +++++++++++- src/PyEventLoop.cc | 10 +++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index ce3f2995..8faa9213 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -187,8 +187,18 @@ protected: PyEventLoop() = delete; PyEventLoop(PyObject *loop) : _loop(loop) {}; private: - static PyEventLoop _mainLoopNotFound(); + /** + * @brief Convenient method to raise Python RuntimeError for no event-loop running, and + * create a null instance of `PyEventLoop` + */ + static PyEventLoop _loopNotFound(); + + /** + * @brief Get the running Python event-loop on a specific thread, or + * raise a Python RuntimeError if no event-loop running on that thread + */ static PyEventLoop _getLoopOnThread(PyThreadState *tstate); + static PyThreadState *_getMainThread(); }; diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index c04588c8..105c6a7d 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -48,8 +48,8 @@ PyEventLoop::Future PyEventLoop::ensureFuture(PyObject *awaitable) { } /* static */ -PyEventLoop PyEventLoop::_mainLoopNotFound() { - PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop on the main thread."); +PyEventLoop PyEventLoop::_loopNotFound() { + PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop to make asynchronous calls."); return PyEventLoop(nullptr); } @@ -58,12 +58,12 @@ PyEventLoop PyEventLoop::_getLoopOnThread(PyThreadState *tstate) { // Modified from Python 3.9 `get_running_loop` https://github.com/python/cpython/blob/7cb3a44/Modules/_asynciomodule.c#L241-L278 PyObject *ts_dict = _PyThreadState_GetDict(tstate); // borrowed reference if (ts_dict == NULL) { - return _mainLoopNotFound(); + return _loopNotFound(); } PyObject *rl = PyDict_GetItemString(ts_dict, "__asyncio_running_event_loop__"); // borrowed reference if (rl == NULL) { - return _mainLoopNotFound(); + return _loopNotFound(); } using PyRunningLoopHolder = struct { @@ -73,7 +73,7 @@ PyEventLoop PyEventLoop::_getLoopOnThread(PyThreadState *tstate) { PyObject *running_loop = ((PyRunningLoopHolder *)rl)->rl_loop; if (running_loop == Py_None) { - return _mainLoopNotFound(); + return _loopNotFound(); } Py_INCREF(running_loop); From f0f79a06635d070a00c98e151b29097bb10121fc Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 25 Apr 2023 21:07:00 +0000 Subject: [PATCH 52/73] refactor(event-loop): use only Python C APIs to get the running event-loop on the current thread --- include/PyEventLoop.hh | 4 +++- src/PyEventLoop.cc | 27 ++++++++++---------------- tests/python/test_pythonmonkey_eval.py | 8 ++++++++ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/include/PyEventLoop.hh b/include/PyEventLoop.hh index 8faa9213..8875350f 100644 --- a/include/PyEventLoop.hh +++ b/include/PyEventLoop.hh @@ -170,12 +170,13 @@ public: /** * @brief Get the running Python event-loop on the current thread, or * raise a Python RuntimeError if no event-loop running + * @see https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_running_loop * @return an instance of `PyEventLoop` */ static PyEventLoop getRunningLoop(); /** - * @brief Get the running Python event-loop on main thread, or + * @brief Get the running Python event-loop on **main thread**, or * raise a Python RuntimeError if no event-loop running * @return an instance of `PyEventLoop` */ @@ -200,6 +201,7 @@ private: static PyEventLoop _getLoopOnThread(PyThreadState *tstate); static PyThreadState *_getMainThread(); + static inline PyThreadState *_getCurrentThread(); }; #endif \ No newline at end of file diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index 105c6a7d..8393fb31 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -61,6 +61,8 @@ PyEventLoop PyEventLoop::_getLoopOnThread(PyThreadState *tstate) { return _loopNotFound(); } + // TODO: Python `get_running_loop` caches the PyRunningLoopHolder, should we do it as well for the main thread? + // see https://github.com/python/cpython/blob/7cb3a44/Modules/_asynciomodule.c#L234-L239 PyObject *rl = PyDict_GetItemString(ts_dict, "__asyncio_running_event_loop__"); // borrowed reference if (rl == NULL) { return _loopNotFound(); @@ -92,6 +94,13 @@ PyThreadState *PyEventLoop::_getMainThread() { return tstate; } +/* static inline */ +PyThreadState *PyEventLoop::_getCurrentThread() { + // `PyThreadState_Get` is used under the hood of the Python `asyncio.get_running_loop` method, + // see https://github.com/python/cpython/blob/7cb3a44/Modules/_asynciomodule.c#L234 + return PyThreadState_Get(); // https://docs.python.org/3/c-api/init.html#c.PyThreadState_Get +} + /* static */ PyEventLoop PyEventLoop::getMainLoop() { return _getLoopOnThread(_getMainThread()); @@ -99,23 +108,7 @@ PyEventLoop PyEventLoop::getMainLoop() { /* static */ PyEventLoop PyEventLoop::getRunningLoop() { - // TODO: refactor into the C API `_getLoopOnThread(PyThreadState_Get())` - // We don't use the C API here yet because the Python method caches the PyRunningLoopHolder (Is it worth caching?), - // see https://github.com/python/cpython/blob/7cb3a44/Modules/_asynciomodule.c#L234-L239 - - // Get the running Python event-loop - // https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_running_loop - PyObject *asyncio = PyImport_ImportModule("asyncio"); - PyObject *loop = PyObject_CallMethod(asyncio, "get_running_loop", NULL); - - Py_DECREF(asyncio); // clean up - - if (loop == nullptr) { // `get_running_loop` would raise a RuntimeError if there is no running event loop - // Overwrite the error raised by `get_running_loop` - PyErr_SetString(PyExc_RuntimeError, "PythonMonkey cannot find a running Python event-loop to make asynchronous calls."); - } - - return PyEventLoop(loop); // `loop` may be null + return _getLoopOnThread(_getCurrentThread()); } void PyEventLoop::AsyncHandle::cancel() { diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index be1ad69d..05b2164c 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -644,6 +644,10 @@ def concatenate(a, b): assert caller(concatenate, string1, string2) == string1 + string2 def test_set_clear_timeout(): + # throw RuntimeError outside a coroutine + with pytest.raises(RuntimeError, match="PythonMonkey cannot find a running Python event-loop to make asynchronous calls."): + pm.eval("setTimeout")(print) + async def async_fn(): # standalone `setTimeout` loop = asyncio.get_running_loop() @@ -699,6 +703,10 @@ def to_raise(msg): return True assert asyncio.run(async_fn()) + # throw RuntimeError outside a coroutine (the event-loop has ended) + with pytest.raises(RuntimeError, match="PythonMonkey cannot find a running Python event-loop to make asynchronous calls."): + pm.eval("setTimeout")(print) + def test_promises(): async def async_fn(): # Python awaitables to JS Promise coercion From d58f6d5a43b2ef2b525d465a6b97a2083bb37b9e Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 25 Apr 2023 21:42:36 +0000 Subject: [PATCH 53/73] fix(pyTypeFactory): segfault if `pyObject` is not set yet in constructors of `PyType` subclasses because of an early return --- include/PyType.hh | 2 +- src/modules/pythonmonkey/pythonmonkey.cc | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/PyType.hh b/include/PyType.hh index a641aed8..57d34145 100644 --- a/include/PyType.hh +++ b/include/PyType.hh @@ -35,6 +35,6 @@ public: protected: virtual void print(std::ostream &os) const = 0; - PyObject *pyObject; + PyObject *pyObject = nullptr; }; #endif \ No newline at end of file diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 6c0589da..16858ea9 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -156,6 +156,9 @@ static PyObject *eval(PyObject *self, PyObject *args) { // translate to the proper python type PyType *returnValue = pyTypeFactory(GLOBAL_CX, global, rval); + if (PyErr_Occurred()) { + return NULL; + } // TODO: Find a better way to destroy the root when necessary (when the returned Python object is GCed). // delete rval; // rval may be a JS function which must be kept alive. From 59295511ce4386b0bf3e9b76f88a8de44b1284e6 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 25 Apr 2023 21:54:22 +0000 Subject: [PATCH 54/73] test(promise): a python RuntimeError should be thrown if JS Promise is created outside a coroutine --- tests/python/test_pythonmonkey_eval.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 05b2164c..c2e1ad68 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -708,7 +708,14 @@ def to_raise(msg): pm.eval("setTimeout")(print) def test_promises(): + # should throw RuntimeError if Promises are created outside a coroutine + create_promise = pm.eval("() => Promise.resolve(1)") + with pytest.raises(RuntimeError, match="PythonMonkey cannot find a running Python event-loop to make asynchronous calls."): + create_promise() + async def async_fn(): + create_promise() # inside a coroutine, no error + # Python awaitables to JS Promise coercion # 1. Python asyncio.Future to JS promise loop = asyncio.get_running_loop() @@ -854,6 +861,10 @@ async def coro_to_throw1(): return True assert asyncio.run(async_fn()) + # should throw a RuntimeError if created outside a coroutine (the event-loop has ended) + with pytest.raises(RuntimeError, match="PythonMonkey cannot find a running Python event-loop to make asynchronous calls."): + pm.eval("new Promise(() => { })") + def test_webassembly(): async def async_fn(): # off-thread promises can run From 9d043873240b9a00c5cfaab0477c97622f6da032 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 26 Apr 2023 18:27:30 +0000 Subject: [PATCH 55/73] feat(event-loop): make sure promise jobs are JS functions, using `MOZ_RELEASE_ASSERT` --- src/JobQueue.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JobQueue.cc b/src/JobQueue.cc index c70de25c..1758643c 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -17,7 +17,7 @@ bool JobQueue::enqueuePromiseJob(JSContext *cx, JS::HandleObject incumbentGlobal) { // Convert the `job` JS function to a Python function for event-loop callback - // TODO (Tom Tang): assert `job` is JS::Handle by JS::GetBuiltinClass(...) == js::ESClass::Function (17) + MOZ_RELEASE_ASSERT(js::IsFunctionObject(job)); // FIXME (Tom Tang): memory leak, objects not free-ed // FIXME (Tom Tang): `job` function is going to be GC-ed ??? auto global = new JS::RootedObject(cx, incumbentGlobal); From ceab871c160d411834e29a58f3eca6c8a6cb2ac0 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Fri, 28 Apr 2023 19:19:39 +0000 Subject: [PATCH 56/73] feat(buffers): Python bytearray to JS ArrayBuffer coercion --- include/BufferType.hh | 40 +++++++++++++++++++++++++++++++++++ include/TypeEnum.hh | 7 ++++--- src/BufferType.cc | 49 +++++++++++++++++++++++++++++++++++++++++++ src/jsTypeFactory.cc | 9 ++++++-- 4 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 include/BufferType.hh create mode 100644 src/BufferType.cc diff --git a/include/BufferType.hh b/include/BufferType.hh new file mode 100644 index 00000000..bd6db455 --- /dev/null +++ b/include/BufferType.hh @@ -0,0 +1,40 @@ +/** + * @file BufferType.hh + * @author Tom Tang (xmader@distributive.network) + * @brief Struct for representing ArrayBuffers + * @version 0.1 + * @date 2023-04-27 + * + * @copyright Copyright (c) 2023 Distributive Corp. + * + */ + +#ifndef PythonMonkey_BufferType_ +#define PythonMonkey_BufferType_ + +#include "PyType.hh" +#include "TypeEnum.hh" + +#include + +#include + +struct BufferType : public PyType { +public: + BufferType(PyObject *object); + + const TYPE returnType = TYPE::BUFFER; + + /** + * @brief Convert a Python object that [provides the buffer interface](https://docs.python.org/3.9/c-api/typeobj.html#buffer-object-structures) to JS ArrayBuffer + * + * @param cx - javascript context pointer + */ + JSObject *toJsArrayBuffer(JSContext *cx); +protected: + virtual void print(std::ostream &os) const override; + + static void _releasePyBuffer(void *, void *bufView); // JS::BufferContentsFreeFunc +}; + +#endif \ No newline at end of file diff --git a/include/TypeEnum.hh b/include/TypeEnum.hh index 472a4237..a2d2c875 100644 --- a/include/TypeEnum.hh +++ b/include/TypeEnum.hh @@ -1,11 +1,11 @@ /** * @file TypeEnum.hh - * @author Caleb Aikens (caleb@distributive.network) & Giovanni Tedesco (giovanni@distributive.network) + * @author Caleb Aikens (caleb@distributive.network) & Giovanni Tedesco (giovanni@distributive.network) & Tom Tang (xmader@distributive.network) * @brief Enum for every PyType * @version 0.1 - * @date 2022-0-08 + * @date 2022-08-08 * - * @copyright Copyright (c) 2022 + * @copyright Copyright (c) 2023 Distributive Corp. * */ @@ -24,6 +24,7 @@ enum class TYPE { TUPLE, DATE, PYTHONMONKEY_PROMISE, + BUFFER, EXCEPTION, NONE, PYTHONMONKEY_NULL, diff --git a/src/BufferType.cc b/src/BufferType.cc new file mode 100644 index 00000000..59de879a --- /dev/null +++ b/src/BufferType.cc @@ -0,0 +1,49 @@ +/** + * @file BufferType.hh + * @author Tom Tang (xmader@distributive.network) + * @brief Struct for representing ArrayBuffers + * @version 0.1 + * @date 2023-04-27 + * + * @copyright Copyright (c) 2023 Distributive Corp. + * + */ + +#include "include/BufferType.hh" + +#include "include/PyType.hh" +#include "include/TypeEnum.hh" + +#include +#include + +#include + +BufferType::BufferType(PyObject *object) : PyType(object) {} + +void BufferType::print(std::ostream &os) const {} + +JSObject *BufferType::toJsArrayBuffer(JSContext *cx) { + // Get the pyObject's underlying buffer pointer and size + Py_buffer *view = new Py_buffer{}; + if (PyObject_GetBuffer(pyObject, view, PyBUF_WRITABLE /* C-contiguous and writable */) < 0) { + // The exporter (pyObject) cannot provide a C-contiguous buffer, + // also raises a PyExc_BufferError + return nullptr; + } + + // Create a new ExternalArrayBuffer object + // data is copied instead of transferring the ownership when this ArrayBuffer is "transferred" to a worker thread. + // see https://hg.mozilla.org/releases/mozilla-esr102/file/a03fde6/js/public/ArrayBuffer.h#l86 + return JS::NewExternalArrayBuffer(cx, + view->len, view->buf, + BufferType::_releasePyBuffer, view + ); +} + +/* static */ +void BufferType::_releasePyBuffer(void *, void *bufView) { + Py_buffer *view = (Py_buffer *)bufView; + PyBuffer_Release(view); + delete view; +} \ No newline at end of file diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index d8187ce4..4c3a85f0 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -19,6 +19,7 @@ #include "include/IntType.hh" #include "include/PromiseType.hh" #include "include/ExceptionType.hh" +#include "include/BufferType.hh" #include #include @@ -147,6 +148,10 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { JSObject *error = ExceptionType(object).toJsError(cx); returnType.setObject(*error); } + else if (PyObject_CheckBuffer(object)) { + JSObject *arrayBuffer = BufferType(object).toJsArrayBuffer(cx); // may return null + returnType.setObjectOrNull(arrayBuffer); + } else if (object == Py_None) { returnType.setUndefined(); } @@ -155,8 +160,8 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { } else if (PythonAwaitable_Check(object)) { PromiseType *p = new PromiseType(object); - JSObject *promise = p->toJsPromise(cx); - returnType.setObject(*promise); + JSObject *promise = p->toJsPromise(cx); // may return null + returnType.setObjectOrNull(promise); // nested awaitables would have already been GCed if finished // memoizePyTypeAndGCThing(p, returnType); } From 0b19619dd9bdd23d192859513760bbaae575f5f9 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 1 May 2023 16:33:30 +0000 Subject: [PATCH 57/73] feat(buffers): Python buffer objects to JS TypedArray coercion, the TypedArray subtype is determined by the buffer's type code --- include/BufferType.hh | 15 ++++++-- src/BufferType.cc | 81 +++++++++++++++++++++++++++++++++++++------ src/jsTypeFactory.cc | 4 +-- 3 files changed, 86 insertions(+), 14 deletions(-) diff --git a/include/BufferType.hh b/include/BufferType.hh index bd6db455..77007916 100644 --- a/include/BufferType.hh +++ b/include/BufferType.hh @@ -16,6 +16,7 @@ #include "TypeEnum.hh" #include +#include #include @@ -26,15 +27,25 @@ public: const TYPE returnType = TYPE::BUFFER; /** - * @brief Convert a Python object that [provides the buffer interface](https://docs.python.org/3.9/c-api/typeobj.html#buffer-object-structures) to JS ArrayBuffer + * @brief Convert a Python object that [provides the buffer interface](https://docs.python.org/3.9/c-api/typeobj.html#buffer-object-structures) to JS TypedArray. + * The subtype (Uint8Array, Float64Array, ...) is automatically determined by the Python buffer's [format](https://docs.python.org/3.9/c-api/buffer.html#c.Py_buffer.format) * * @param cx - javascript context pointer */ - JSObject *toJsArrayBuffer(JSContext *cx); + JSObject *toJsTypedArray(JSContext *cx); protected: virtual void print(std::ostream &os) const override; static void _releasePyBuffer(void *, void *bufView); // JS::BufferContentsFreeFunc + + static JS::Scalar::Type _getPyBufferType(Py_buffer *bufView); + + /** + * @brief Create a new typed array using up the given ArrayBuffer or SharedArrayBuffer for storage. + * @see https://hg.mozilla.org/releases/mozilla-esr102/file/a03fde6/js/public/experimental/TypedData.h#l80 + * There's no SpiderMonkey API to assign the subtype at execution time + */ + static JSObject *_newTypedArrayWithBuffer(JSContext *cx, JS::Scalar::Type subtype, JS::HandleObject arrayBuffer); }; #endif \ No newline at end of file diff --git a/src/BufferType.cc b/src/BufferType.cc index 59de879a..9a9c622d 100644 --- a/src/BufferType.cc +++ b/src/BufferType.cc @@ -16,6 +16,8 @@ #include #include +#include +#include #include @@ -23,22 +25,27 @@ BufferType::BufferType(PyObject *object) : PyType(object) {} void BufferType::print(std::ostream &os) const {} -JSObject *BufferType::toJsArrayBuffer(JSContext *cx) { +JSObject *BufferType::toJsTypedArray(JSContext *cx) { // Get the pyObject's underlying buffer pointer and size Py_buffer *view = new Py_buffer{}; - if (PyObject_GetBuffer(pyObject, view, PyBUF_WRITABLE /* C-contiguous and writable */) < 0) { - // The exporter (pyObject) cannot provide a C-contiguous buffer, - // also raises a PyExc_BufferError - return nullptr; + if (PyObject_GetBuffer(pyObject, view, PyBUF_WRITABLE /* C-contiguous and writable 1-dimensional array */ | PyBUF_FORMAT) < 0) { + // The exporter (pyObject) cannot provide a contiguous 1-dimensional buffer, or + // the buffer is immutable (read-only) + return nullptr; // raises a PyExc_BufferError } // Create a new ExternalArrayBuffer object - // data is copied instead of transferring the ownership when this ArrayBuffer is "transferred" to a worker thread. + // Note: data will be copied instead of transferring the ownership when this external ArrayBuffer is "transferred" to a worker thread. // see https://hg.mozilla.org/releases/mozilla-esr102/file/a03fde6/js/public/ArrayBuffer.h#l86 - return JS::NewExternalArrayBuffer(cx, - view->len, view->buf, - BufferType::_releasePyBuffer, view + JSObject *arrayBuffer = JS::NewExternalArrayBuffer(cx, + view->len /* byteLength */, view->buf /* data pointer */, + BufferType::_releasePyBuffer, view /* the `bufView` argument to `_releasePyBuffer` */ ); + JS::RootedObject arrayBufferRooted(cx, arrayBuffer); + + // Determine the TypedArray's subtype (Uint8Array, Float64Array, ...) + JS::Scalar::Type subtype = _getPyBufferType(view); + return _newTypedArrayWithBuffer(cx, subtype, arrayBufferRooted); } /* static */ @@ -46,4 +53,58 @@ void BufferType::_releasePyBuffer(void *, void *bufView) { Py_buffer *view = (Py_buffer *)bufView; PyBuffer_Release(view); delete view; -} \ No newline at end of file +} + +/* static */ +JS::Scalar::Type BufferType::_getPyBufferType(Py_buffer *bufView) { + if (!bufView->format) { // If `format` is NULL, "B" (unsigned bytes) is assumed. https://docs.python.org/3.9/c-api/buffer.html#c.Py_buffer.format + return JS::Scalar::Uint8; + } + if (std::char_traits::length(bufView->format) != 1) { // the type code should be a single character + return JS::Scalar::MaxTypedArrayViewType; // invalid + } + + char typeCode = bufView->format[0]; + // floating point types + if (typeCode == 'f') { + return JS::Scalar::Float32; + } else if (typeCode == 'd') { + return JS::Scalar::Float64; + } + + // integer types + // We can't rely on the type codes alone since the typecodes are mapped to C types and would have different sizes on different architectures + // see https://docs.python.org/3.9/library/array.html#module-array + // https://github.com/python/cpython/blob/7cb3a44/Modules/arraymodule.c#L550-L570 + // TODO (Tom Tang): refactor to something like `switch (typeCode) case 'Q': return [compile-time] intType` + bool isSigned = std::islower(typeCode); // e.g. 'b' for signed char, 'B' for unsigned char + uint8_t byteSize = bufView->itemsize; + switch (byteSize) { + case 1: + return isSigned ? JS::Scalar::Int8 : JS::Scalar::Uint8; // TODO (Tom Tang): Uint8Clamped + case 2: + return isSigned ? JS::Scalar::Int16 : JS::Scalar::Uint16; + case 4: + return isSigned ? JS::Scalar::Int32 : JS::Scalar::Uint32; + case 8: + return isSigned ? JS::Scalar::BigInt64 : JS::Scalar::BigUint64; + default: + return JS::Scalar::MaxTypedArrayViewType; // invalid byteSize + } +} + +JSObject *BufferType::_newTypedArrayWithBuffer(JSContext *cx, JS::Scalar::Type subtype, JS::HandleObject arrayBuffer) { + switch (subtype) { +#define NEW_TYPED_ARRAY_WITH_BUFFER(ExternalType, NativeType, Name) \ +case JS::Scalar::Name: \ + return JS_New ## Name ## ArrayWithBuffer(cx, arrayBuffer, 0, -1 /* use up the ArrayBuffer */); + + JS_FOR_EACH_TYPED_ARRAY(NEW_TYPED_ARRAY_WITH_BUFFER) +#undef NEW_TYPED_ARRAY_WITH_BUFFER + default: // invalid + PyErr_SetString(PyExc_TypeError, "Invalid Python buffer type."); + return nullptr; + } +} + + diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index 4c3a85f0..72470f91 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -149,8 +149,8 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { returnType.setObject(*error); } else if (PyObject_CheckBuffer(object)) { - JSObject *arrayBuffer = BufferType(object).toJsArrayBuffer(cx); // may return null - returnType.setObjectOrNull(arrayBuffer); + JSObject *typedArray = BufferType(object).toJsTypedArray(cx); // may return null + returnType.setObjectOrNull(typedArray); } else if (object == Py_None) { returnType.setUndefined(); From 64bc024fd6306c3812d09606b813f5ade06e43b6 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Tue, 2 May 2023 20:43:24 +0000 Subject: [PATCH 58/73] feat(buffers): JS TypedArray/ArrayBuffer to Python `memoryview` coercion --- include/BufferType.hh | 19 ++++++++ src/BufferType.cc | 108 ++++++++++++++++++++++++++++++++++++++++-- src/pyTypeFactory.cc | 9 +++- 3 files changed, 131 insertions(+), 5 deletions(-) diff --git a/include/BufferType.hh b/include/BufferType.hh index 77007916..d5f48341 100644 --- a/include/BufferType.hh +++ b/include/BufferType.hh @@ -24,6 +24,14 @@ struct BufferType : public PyType { public: BufferType(PyObject *object); + /** + * @brief Construct a new BufferType object from a JS TypedArray or ArrayBuffer, as a Python [memoryview](https://docs.python.org/3.9/c-api/memoryview.html) object + * + * @param cx - javascript context pointer + * @param bufObj - JS object to be coerced + */ + BufferType(JSContext *cx, JS::HandleObject bufObj); + const TYPE returnType = TYPE::BUFFER; /** @@ -33,12 +41,23 @@ public: * @param cx - javascript context pointer */ JSObject *toJsTypedArray(JSContext *cx); + + /** + * @returns Is the given JS object either a TypedArray or an ArrayBuffer? + */ + static bool isSupportedJsTypes(JSObject *obj); + protected: virtual void print(std::ostream &os) const override; + static PyObject *fromJsTypedArray(JSContext *cx, JS::HandleObject typedArray); + static PyObject *fromJsArrayBuffer(JSContext *cx, JS::HandleObject arrayBuffer); + +private: static void _releasePyBuffer(void *, void *bufView); // JS::BufferContentsFreeFunc static JS::Scalar::Type _getPyBufferType(Py_buffer *bufView); + static const char *_toPyBufferFormatCode(JS::Scalar::Type subtype); /** * @brief Create a new typed array using up the given ArrayBuffer or SharedArrayBuffer for storage. diff --git a/src/BufferType.cc b/src/BufferType.cc index 9a9c622d..e8ab195e 100644 --- a/src/BufferType.cc +++ b/src/BufferType.cc @@ -23,6 +23,76 @@ BufferType::BufferType(PyObject *object) : PyType(object) {} +BufferType::BufferType(JSContext *cx, JS::HandleObject bufObj) { + if (JS_IsTypedArrayObject(bufObj)) { + pyObject = fromJsTypedArray(cx, bufObj); + } else if (JS::IsArrayBufferObject(bufObj)) { + pyObject = fromJsArrayBuffer(cx, bufObj); + } else { + // TODO (Tom Tang): Add support for JS [DataView](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView) + PyErr_SetString(PyExc_TypeError, "`bufObj` is neither a TypedArray object nor an ArraryBuffer object."); + pyObject = nullptr; + } +} + +/* static */ +bool BufferType::isSupportedJsTypes(JSObject *obj) { + return JS::IsArrayBufferObject(obj) || JS_IsTypedArrayObject(obj); +} + +/* static */ +PyObject *BufferType::fromJsTypedArray(JSContext *cx, JS::HandleObject typedArray) { + JS::Scalar::Type subtype = JS_GetArrayBufferViewType(typedArray); + auto byteLength = JS_GetTypedArrayByteLength(typedArray); + + // Retrieve/Create the underlying ArrayBuffer object for side-effect. + // + // If byte length is less than `JS_MaxMovableTypedArraySize()`, + // the ArrayBuffer object would be created lazily and the data is stored inline in the TypedArray. + // We don't want inline data because the data pointer would be invalidated during a GC as the TypedArray object is moved. + bool isSharedMemory; + if (!JS_GetArrayBufferViewBuffer(cx, typedArray, &isSharedMemory)) return nullptr; + + uint8_t __destBuf[0] = {}; // we don't care about its value as it's used only if the TypedArray still having inline data + uint8_t *data = JS_GetArrayBufferViewFixedData(typedArray, __destBuf, 0 /* making sure we don't copy inline data */); + if (data == nullptr) { // shared memory or still having inline data + PyErr_SetString(PyExc_TypeError, "PythonMonkey cannot coerce TypedArrays backed by shared memory."); + return nullptr; + } + + Py_buffer bufInfo = { + .buf = data, + .obj = NULL /* the exporter PyObject */, + .len = (Py_ssize_t)byteLength, + .itemsize = (uint8_t)JS::Scalar::byteSize(subtype), + .readonly = false, + .ndim = 1 /* 1-dimensional array */, + .format = (char *)_toPyBufferFormatCode(subtype), + }; + return PyMemoryView_FromBuffer(&bufInfo); +} + +/* static */ +PyObject *BufferType::fromJsArrayBuffer(JSContext *cx, JS::HandleObject arrayBuffer) { + auto byteLength = JS::GetArrayBufferByteLength(arrayBuffer); + + // TODO (Tom Tang): handle SharedArrayBuffers or disallow them completely + bool isSharedMemory; // `JS::GetArrayBufferData` always sets this to `false` + JS::AutoCheckCannotGC autoNoGC(cx); // we don't really care about this + uint8_t *data = JS::GetArrayBufferData(arrayBuffer, &isSharedMemory, autoNoGC); + + Py_buffer bufInfo = { + .buf = data, + .obj = NULL /* the exporter PyObject */, + .len = (Py_ssize_t)byteLength, + .itemsize = 1 /* each element is 1 byte */, + .readonly = false, + .ndim = 1 /* 1-dimensional array */, + .format = "B" /* uint8 array */, + }; + return PyMemoryView_FromBuffer(&bufInfo); +} + void BufferType::print(std::ostream &os) const {} JSObject *BufferType::toJsTypedArray(JSContext *cx) { @@ -30,7 +100,7 @@ JSObject *BufferType::toJsTypedArray(JSContext *cx) { Py_buffer *view = new Py_buffer{}; if (PyObject_GetBuffer(pyObject, view, PyBUF_WRITABLE /* C-contiguous and writable 1-dimensional array */ | PyBUF_FORMAT) < 0) { // The exporter (pyObject) cannot provide a contiguous 1-dimensional buffer, or - // the buffer is immutable (read-only) + // the buffer is immutable (Python `bytes` type is read-only) return nullptr; // raises a PyExc_BufferError } @@ -93,11 +163,43 @@ JS::Scalar::Type BufferType::_getPyBufferType(Py_buffer *bufView) { } } +/* static */ +const char *BufferType::_toPyBufferFormatCode(JS::Scalar::Type subtype) { + // floating point types + if (subtype == JS::Scalar::Float32) { + return "f"; + } else if (subtype == JS::Scalar::Float64) { + return "d"; + } + + // integer types + bool isSigned = JS::Scalar::isSignedIntType(subtype); + uint8_t byteSize = JS::Scalar::byteSize(subtype); + // Python `array` type codes are strictly mapped to basic C types (e.g., `int`), widths may vary on different architectures, + // but JS TypedArray uses fixed-width integer types (e.g., `uint32_t`) + switch (byteSize) { + case sizeof(char): + return isSigned ? "b" : "B"; + case sizeof(short): + return isSigned ? "h" : "H"; + case sizeof(int): + return isSigned ? "i" : "I"; + // case sizeof(long): // compile error: duplicate case value + // // And this is usually where the bit widths on 32/64-bit systems don't agree, + // // see https://en.wikipedia.org/wiki/64-bit_computing#64-bit_data_models + // return isSigned ? "l" : "L"; + case sizeof(long long): + return isSigned ? "q" : "Q"; + default: // invalid + return "x"; // type code for pad bytes, no value + } +} + JSObject *BufferType::_newTypedArrayWithBuffer(JSContext *cx, JS::Scalar::Type subtype, JS::HandleObject arrayBuffer) { switch (subtype) { #define NEW_TYPED_ARRAY_WITH_BUFFER(ExternalType, NativeType, Name) \ case JS::Scalar::Name: \ - return JS_New ## Name ## ArrayWithBuffer(cx, arrayBuffer, 0, -1 /* use up the ArrayBuffer */); + return JS_New ## Name ## ArrayWithBuffer(cx, arrayBuffer, 0 /* byteOffset */, -1 /* use up the ArrayBuffer */); JS_FOR_EACH_TYPED_ARRAY(NEW_TYPED_ARRAY_WITH_BUFFER) #undef NEW_TYPED_ARRAY_WITH_BUFFER @@ -106,5 +208,3 @@ case JS::Scalar::Name: \ return nullptr; } } - - diff --git a/src/pyTypeFactory.cc b/src/pyTypeFactory.cc index a149bde2..de26cce6 100644 --- a/src/pyTypeFactory.cc +++ b/src/pyTypeFactory.cc @@ -12,6 +12,7 @@ #include "include/pyTypeFactory.hh" #include "include/BoolType.hh" +#include "include/BufferType.hh" #include "include/DateType.hh" #include "include/DictType.hh" #include "include/ExceptionType.hh" @@ -143,7 +144,13 @@ PyType *pyTypeFactory(JSContext *cx, JS::Rooted *thisObj, JS::Rooted break; } default: { - printf("objects %d of this type are not handled by PythonMonkey yet\n", cls); + if (BufferType::isSupportedJsTypes(obj)) { // TypedArray or ArrayBuffer + // TODO (Tom Tang): ArrayBuffers have cls == js::ESClass::String + returnValue = new BufferType(cx, obj); + if (returnValue->getPyObject() != nullptr) memoizePyTypeAndGCThing(returnValue, *rval); + } else { + printf("objects of this type (%d) are not handled by PythonMonkey yet\n", cls); + } } } } From ddd6bb97a4b2a86b79d954500b8e718a885b901a Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 3 May 2023 17:25:45 +0000 Subject: [PATCH 59/73] fix(buffers): handle empty Python buffers --- src/BufferType.cc | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/BufferType.cc b/src/BufferType.cc index e8ab195e..5116922a 100644 --- a/src/BufferType.cc +++ b/src/BufferType.cc @@ -104,17 +104,24 @@ JSObject *BufferType::toJsTypedArray(JSContext *cx) { return nullptr; // raises a PyExc_BufferError } - // Create a new ExternalArrayBuffer object - // Note: data will be copied instead of transferring the ownership when this external ArrayBuffer is "transferred" to a worker thread. - // see https://hg.mozilla.org/releases/mozilla-esr102/file/a03fde6/js/public/ArrayBuffer.h#l86 - JSObject *arrayBuffer = JS::NewExternalArrayBuffer(cx, - view->len /* byteLength */, view->buf /* data pointer */, - BufferType::_releasePyBuffer, view /* the `bufView` argument to `_releasePyBuffer` */ - ); - JS::RootedObject arrayBufferRooted(cx, arrayBuffer); - // Determine the TypedArray's subtype (Uint8Array, Float64Array, ...) JS::Scalar::Type subtype = _getPyBufferType(view); + + JSObject *arrayBuffer = nullptr; + if (view->len > 0) { + // Create a new ExternalArrayBuffer object + // Note: data will be copied instead of transferring the ownership when this external ArrayBuffer is "transferred" to a worker thread. + // see https://hg.mozilla.org/releases/mozilla-esr102/file/a03fde6/js/public/ArrayBuffer.h#l86 + arrayBuffer = JS::NewExternalArrayBuffer(cx, + view->len /* byteLength */, view->buf /* data pointer */, + BufferType::_releasePyBuffer, view /* the `bufView` argument to `_releasePyBuffer` */ + ); + } else { // empty buffer + arrayBuffer = JS::NewArrayBuffer(cx, 0); + BufferType::_releasePyBuffer(nullptr /* data pointer */, view); // the buffer is no longer needed since we are creating a brand new empty ArrayBuffer + } + JS::RootedObject arrayBufferRooted(cx, arrayBuffer); + return _newTypedArrayWithBuffer(cx, subtype, arrayBufferRooted); } From da40a78c31ac3b6c852cb93385385f077067c969 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 3 May 2023 18:21:39 +0000 Subject: [PATCH 60/73] feat(buffers): disallow multidimensional arrays, only 1-D array is supported --- src/BufferType.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/BufferType.cc b/src/BufferType.cc index 5116922a..9c510e12 100644 --- a/src/BufferType.cc +++ b/src/BufferType.cc @@ -98,11 +98,14 @@ void BufferType::print(std::ostream &os) const {} JSObject *BufferType::toJsTypedArray(JSContext *cx) { // Get the pyObject's underlying buffer pointer and size Py_buffer *view = new Py_buffer{}; - if (PyObject_GetBuffer(pyObject, view, PyBUF_WRITABLE /* C-contiguous and writable 1-dimensional array */ | PyBUF_FORMAT) < 0) { - // The exporter (pyObject) cannot provide a contiguous 1-dimensional buffer, or - // the buffer is immutable (Python `bytes` type is read-only) + if (PyObject_GetBuffer(pyObject, view, PyBUF_ND | PyBUF_WRITABLE /* C-contiguous and writable */ | PyBUF_FORMAT) < 0) { + // the buffer is immutable (e.g., Python `bytes` type is read-only) return nullptr; // raises a PyExc_BufferError } + if (view->ndim != 1) { + PyErr_SetString(PyExc_BufferError, "multidimensional arrays are not allowed"); + return nullptr; + } // Determine the TypedArray's subtype (Uint8Array, Float64Array, ...) JS::Scalar::Type subtype = _getPyBufferType(view); From bd6ab2f4a5e8208919cf799cfe0803cc681c2179 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 3 May 2023 18:54:32 +0000 Subject: [PATCH 61/73] chore: install numpy for our tests --- .github/workflows/tests.yaml | 4 ++-- setup.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2460cc32..06c9db68 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -21,8 +21,8 @@ jobs: python-version: ${{ matrix.python_version }} - name: Setup cmake run: | - sudo apt-get install -y cmake doxygen graphviz gcovr llvm python3-dev python3-pytest - pip install pytest + sudo apt-get install -y cmake doxygen graphviz gcovr llvm python3-dev python3-pytest python3-numpy + pip install pytest numpy - name: Cache spidermonkey build id: cache-spidermonkey uses: actions/cache@v3 diff --git a/setup.sh b/setup.sh index 21528a3d..7878342e 100755 --- a/setup.sh +++ b/setup.sh @@ -3,7 +3,7 @@ CPUS=$(getconf _NPROCESSORS_ONLN 2>/dev/null || getconf NPROCESSORS_ONLN 2>/dev/ sudo apt-get update --yes sudo apt-get upgrade --yes -sudo apt-get install cmake python3-dev python3-pytest doxygen graphviz gcovr llvm g++ pkg-config m4 --yes +sudo apt-get install cmake python3-dev python3-pytest python3-numpy doxygen graphviz gcovr llvm g++ pkg-config m4 --yes sudo curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y #install rust compiler wget -q https://ftp.mozilla.org/pub/firefox/releases/102.2.0esr/source/firefox-102.2.0esr.source.tar.xz tar xf firefox-102.2.0esr.source.tar.xz From 59ee761f699d95c6920c91f137c58a26b454051c Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 4 May 2023 15:15:19 +0000 Subject: [PATCH 62/73] refactor(buffers): `_releasePyBuffer` accepts a single `Py_buffer *` argument --- include/BufferType.hh | 3 ++- src/BufferType.cc | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/include/BufferType.hh b/include/BufferType.hh index d5f48341..1b8816af 100644 --- a/include/BufferType.hh +++ b/include/BufferType.hh @@ -54,7 +54,8 @@ protected: static PyObject *fromJsArrayBuffer(JSContext *cx, JS::HandleObject arrayBuffer); private: - static void _releasePyBuffer(void *, void *bufView); // JS::BufferContentsFreeFunc + static void _releasePyBuffer(Py_buffer *bufView); + static void _releasePyBuffer(void *, void *bufView); // JS::BufferContentsFreeFunc callback for JS::NewExternalArrayBuffer static JS::Scalar::Type _getPyBufferType(Py_buffer *bufView); static const char *_toPyBufferFormatCode(JS::Scalar::Type subtype); diff --git a/src/BufferType.cc b/src/BufferType.cc index 9c510e12..0120ffef 100644 --- a/src/BufferType.cc +++ b/src/BufferType.cc @@ -104,6 +104,7 @@ JSObject *BufferType::toJsTypedArray(JSContext *cx) { } if (view->ndim != 1) { PyErr_SetString(PyExc_BufferError, "multidimensional arrays are not allowed"); + BufferType::_releasePyBuffer(view); return nullptr; } @@ -121,18 +122,22 @@ JSObject *BufferType::toJsTypedArray(JSContext *cx) { ); } else { // empty buffer arrayBuffer = JS::NewArrayBuffer(cx, 0); - BufferType::_releasePyBuffer(nullptr /* data pointer */, view); // the buffer is no longer needed since we are creating a brand new empty ArrayBuffer + BufferType::_releasePyBuffer(view); // the buffer is no longer needed since we are creating a brand new empty ArrayBuffer } JS::RootedObject arrayBufferRooted(cx, arrayBuffer); return _newTypedArrayWithBuffer(cx, subtype, arrayBufferRooted); } +/* static */ +void BufferType::_releasePyBuffer(Py_buffer *bufView) { + PyBuffer_Release(bufView); + delete bufView; +} + /* static */ void BufferType::_releasePyBuffer(void *, void *bufView) { - Py_buffer *view = (Py_buffer *)bufView; - PyBuffer_Release(view); - delete view; + return _releasePyBuffer((Py_buffer *)bufView); } /* static */ From 5caaabbc3ea3782dc626d5eff23f8ed200d7ce70 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 4 May 2023 15:30:04 +0000 Subject: [PATCH 63/73] fix(buffers): segfault on final GC --- src/jsTypeFactory.cc | 4 +++- src/pyTypeFactory.cc | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index 72470f91..f8da1d09 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -149,8 +149,10 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { returnType.setObject(*error); } else if (PyObject_CheckBuffer(object)) { - JSObject *typedArray = BufferType(object).toJsTypedArray(cx); // may return null + BufferType *pmBuffer = new BufferType(object); + JSObject *typedArray = pmBuffer->toJsTypedArray(cx); // may return null returnType.setObjectOrNull(typedArray); + memoizePyTypeAndGCThing(pmBuffer, returnType); } else if (object == Py_None) { returnType.setUndefined(); diff --git a/src/pyTypeFactory.cc b/src/pyTypeFactory.cc index de26cce6..6906a15d 100644 --- a/src/pyTypeFactory.cc +++ b/src/pyTypeFactory.cc @@ -147,7 +147,7 @@ PyType *pyTypeFactory(JSContext *cx, JS::Rooted *thisObj, JS::Rooted if (BufferType::isSupportedJsTypes(obj)) { // TypedArray or ArrayBuffer // TODO (Tom Tang): ArrayBuffers have cls == js::ESClass::String returnValue = new BufferType(cx, obj); - if (returnValue->getPyObject() != nullptr) memoizePyTypeAndGCThing(returnValue, *rval); + // if (returnValue->getPyObject() != nullptr) memoizePyTypeAndGCThing(returnValue, *rval); } else { printf("objects of this type (%d) are not handled by PythonMonkey yet\n", cls); } From 7fa42326827296f4c1fb8c681b78d64c598c5ac6 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 4 May 2023 15:43:14 +0000 Subject: [PATCH 64/73] test(buffers): write tests for coercing buffers of numpy array to JS TypedArray, although we don't officially support numpy yet --- tests/python/test_pythonmonkey_eval.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index c2e1ad68..79491ec6 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta import math import asyncio +import numpy def test_passes(): assert True @@ -886,3 +887,26 @@ async def async_fn(): # making sure the async_fn is run return True assert asyncio.run(async_fn()) + +def test_py_buffer_to_js_typed_array(): + # should work for simple 1-D numpy array as well + numpy_int16_array = numpy.array([0, 1, 2, 3], dtype=numpy.int16) + assert "0,1,2,3" == pm.eval("(typedArray) => typedArray.toString()")(numpy_int16_array) + assert 3.0 == pm.eval("(typedArray) => typedArray[3]")(numpy_int16_array) + assert True == pm.eval("(typedArray) => typedArray instanceof Int16Array")(numpy_int16_array) + numpy_memoryview = pm.eval("(typedArray) => typedArray")(numpy_int16_array) + assert 2 == numpy_memoryview[2] + assert 4 * 2 == numpy_memoryview.nbytes # 4 elements * sizeof(int16_t) + assert "h" == numpy_memoryview.format # the type code for int16 is 'h', see https://docs.python.org/3.9/library/array.html + with pytest.raises(IndexError, match="index out of bounds on dimension 1"): + numpy_memoryview[4] + + # buffer should be in C order (row major) + fortran_order_arr = numpy.array([[1, 2], [3, 4]], order="F") # 1-D array is always considered C-contiguous because it doesn't matter if it's row or column major in 1-D + with pytest.raises(ValueError, match="ndarray is not C-contiguous"): + pm.eval("(typedArray) => {}")(fortran_order_arr) + + # disallow multidimensional array + numpy_2d_array = numpy.array([[1, 2], [3, 4]], order="C") + with pytest.raises(BufferError, match="multidimensional arrays are not allowed"): + pm.eval("(typedArray) => {}")(numpy_2d_array) From bce13f29a4dbb36215d06ab446d250babf197f6d Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 4 May 2023 20:42:06 +0000 Subject: [PATCH 65/73] test(buffers): write tests for JS TypedArray/ArrayBuffer <-> Python memoryview coercion --- tests/python/test_pythonmonkey_eval.py | 104 ++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 79491ec6..190a1903 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta import math import asyncio -import numpy +import numpy, array, struct def test_passes(): assert True @@ -889,7 +889,50 @@ async def async_fn(): assert asyncio.run(async_fn()) def test_py_buffer_to_js_typed_array(): - # should work for simple 1-D numpy array as well + # JS TypedArray/ArrayBuffer should coerce to Python memoryview type + def assert_js_to_py_memoryview(buf: memoryview): + assert type(buf) is memoryview + assert None == buf.obj # https://docs.python.org/3.9/c-api/buffer.html#c.Py_buffer.obj + assert 2 * 4 == buf.nbytes # 2 elements * sizeof(int32_t) + assert "02000000ffffffff" == buf.hex() # little endian + buf1 = pm.eval("new Int32Array([2,-1])") + buf2 = pm.eval("new Int32Array([2,-1]).buffer") + assert_js_to_py_memoryview(buf1) + assert_js_to_py_memoryview(buf2) + assert [2, -1] == buf1.tolist() + assert [2, 0, 0, 0, 255, 255, 255, 255] == buf2.tolist() + assert -1 == buf1[1] + assert 255 == buf2[7] + with pytest.raises(IndexError, match="index out of bounds on dimension 1"): + buf1[2] + with pytest.raises(IndexError, match="index out of bounds on dimension 1"): + buf2[8] + + # Python buffers should coerce to JS TypedArray + # and the typecode maps to TypedArray subtype (Uint8Array, Float64Array, ...) + assert True == pm.eval("(arr)=>arr instanceof Uint8Array")( bytearray([1,2,3]) ) + assert True == pm.eval("(arr)=>arr instanceof Uint8Array")( numpy.array([1], dtype=numpy.uint8) ) + assert True == pm.eval("(arr)=>arr instanceof Uint16Array")( numpy.array([1], dtype=numpy.uint16) ) + assert True == pm.eval("(arr)=>arr instanceof Uint32Array")( numpy.array([1], dtype=numpy.uint32) ) + assert True == pm.eval("(arr)=>arr instanceof BigUint64Array")( numpy.array([1], dtype=numpy.uint64) ) + assert True == pm.eval("(arr)=>arr instanceof Int8Array")( numpy.array([1], dtype=numpy.int8) ) + assert True == pm.eval("(arr)=>arr instanceof Int16Array")( numpy.array([1], dtype=numpy.int16) ) + assert True == pm.eval("(arr)=>arr instanceof Int32Array")( numpy.array([1], dtype=numpy.int32) ) + assert True == pm.eval("(arr)=>arr instanceof BigInt64Array")( numpy.array([1], dtype=numpy.int64) ) + assert True == pm.eval("(arr)=>arr instanceof Float32Array")( numpy.array([1], dtype=numpy.float32) ) + assert True == pm.eval("(arr)=>arr instanceof Float64Array")( numpy.array([1], dtype=numpy.float64) ) + assert pm.eval("new Uint8Array([1])").format == "B" + assert pm.eval("new Uint16Array([1])").format == "H" + assert pm.eval("new Uint32Array([1])").format == "I" # FIXME (Tom Tang): this is "L" on 32-bit systems + assert pm.eval("new BigUint64Array([1n])").format == "Q" + assert pm.eval("new Int8Array([1])").format == "b" + assert pm.eval("new Int16Array([1])").format == "h" + assert pm.eval("new Int32Array([1])").format == "i" + assert pm.eval("new BigInt64Array([1n])").format == "q" + assert pm.eval("new Float32Array([1])").format == "f" + assert pm.eval("new Float64Array([1])").format == "d" + + # simple 1-D numpy array should just work as well numpy_int16_array = numpy.array([0, 1, 2, 3], dtype=numpy.int16) assert "0,1,2,3" == pm.eval("(typedArray) => typedArray.toString()")(numpy_int16_array) assert 3.0 == pm.eval("(typedArray) => typedArray[3]")(numpy_int16_array) @@ -901,6 +944,63 @@ def test_py_buffer_to_js_typed_array(): with pytest.raises(IndexError, match="index out of bounds on dimension 1"): numpy_memoryview[4] + # can work for empty Python buffer + def assert_empty_py_buffer(buf, type: str): + assert 0 == pm.eval("(typedArray) => typedArray.length")(buf) + assert None == pm.eval("(typedArray) => typedArray[0]")(buf) # `undefined` + assert True == pm.eval("(typedArray) => typedArray instanceof "+type)(buf) + assert_empty_py_buffer(bytearray(b''), "Uint8Array") + assert_empty_py_buffer(numpy.array([], dtype=numpy.uint64), "BigUint64Array") + assert_empty_py_buffer(array.array('d', []), "Float64Array") + + # can work for empty TypedArray + def assert_empty_typedarray(buf: memoryview, typecode: str): + assert typecode == buf.format + assert struct.calcsize(typecode) == buf.itemsize + assert 0 == buf.nbytes + assert "" == buf.hex() + assert b"" == buf.tobytes() + assert [] == buf.tolist() + buf.release() + assert_empty_typedarray(pm.eval("new BigInt64Array()"), "q") + assert_empty_typedarray(pm.eval("new Float32Array(new ArrayBuffer(4), 4 /*byteOffset*/)"), "f") + assert_empty_typedarray(pm.eval("(arr)=>arr")( bytearray([]) ), "B") + assert_empty_typedarray(pm.eval("(arr)=>arr")( numpy.array([], dtype=numpy.uint16) ),"H") + assert_empty_typedarray(pm.eval("(arr)=>arr")( array.array("d", []) ),"d") + + # can work for empty ArrayBuffer + def assert_empty_arraybuffer(buf): + assert "B" == buf.format + assert 1 == buf.itemsize + assert 0 == buf.nbytes + assert "" == buf.hex() + assert b"" == buf.tobytes() + assert [] == buf.tolist() + buf.release() + assert_empty_arraybuffer(pm.eval("new ArrayBuffer()")) + assert_empty_arraybuffer(pm.eval("new Uint8Array().buffer")) + assert_empty_arraybuffer(pm.eval("new Float64Array().buffer")) + assert_empty_arraybuffer(pm.eval("(arr)=>arr.buffer")( bytearray([]) )) + assert_empty_arraybuffer(pm.eval("(arr)=>arr.buffer")( pm.eval("(arr)=>arr.buffer")(bytearray()) )) + assert_empty_arraybuffer(pm.eval("(arr)=>arr.buffer")( numpy.array([], dtype=numpy.uint64) )) + assert_empty_arraybuffer(pm.eval("(arr)=>arr.buffer")( array.array("d", []) )) + + # TODO (Tom Tang): shared ArrayBuffer should be disallowed + # pm.eval("new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }).buffer") + + # TODO (Tom Tang): once a JS ArrayBuffer is transferred to a worker thread, it should be invalidated in Python-land as well + + # TODO (Tom Tang): error for detached ArrayBuffer, or should it be considered as empty? + + # should error on immutable Python buffers + # Note: Python `bytes` type must be converted to a (mutable) `bytearray` because there's no such a concept of read-only ArrayBuffer in JS + with pytest.raises(BufferError, match="Object is not writable."): + pm.eval("(typedArray) => {}")(b'') + immutable_numpy_array = numpy.arange(10) + immutable_numpy_array.setflags(write=False) + with pytest.raises(ValueError, match="buffer source array is read-only"): + pm.eval("(typedArray) => {}")(immutable_numpy_array) + # buffer should be in C order (row major) fortran_order_arr = numpy.array([[1, 2], [3, 4]], order="F") # 1-D array is always considered C-contiguous because it doesn't matter if it's row or column major in 1-D with pytest.raises(ValueError, match="ndarray is not C-contiguous"): From afb5fe977e326d2522ca0836c19a8c8fc4272d34 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 4 May 2023 22:52:21 +0000 Subject: [PATCH 66/73] fix(callPyFunc): capture error for the return value as well because `jsTypeFactory` may also raise exceptions --- src/jsTypeFactory.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index f8da1d09..f8868d31 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -224,6 +224,9 @@ bool callPyFunc(JSContext *cx, unsigned int argc, JS::Value *vp) { } // @TODO (Caleb Aikens) need to check for python exceptions here callargs.rval().set(jsTypeFactory(cx, pyRval)); + if (PyErr_Occurred()) { + return false; + } return true; } \ No newline at end of file From c85727025d4703a498af187830c7a7a1c033b088 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 4 May 2023 22:54:11 +0000 Subject: [PATCH 67/73] =?UTF-8?q?refactor(buffers):=20suppress=20compile-t?= =?UTF-8?q?ime=20warning=20`ISO=20C++=20forbids=20converting=20a=20string?= =?UTF-8?q?=20constant=20to=20=E2=80=98char*=E2=80=99`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/BufferType.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BufferType.cc b/src/BufferType.cc index 0120ffef..30b66af7 100644 --- a/src/BufferType.cc +++ b/src/BufferType.cc @@ -88,7 +88,7 @@ PyObject *BufferType::fromJsArrayBuffer(JSContext *cx, JS::HandleObject arrayBuf .itemsize = 1 /* each element is 1 byte */, .readonly = false, .ndim = 1 /* 1-dimensional array */, - .format = "B" /* uint8 array */, + .format = (char *)"B" /* uint8 array */, }; return PyMemoryView_FromBuffer(&bufInfo); } From 0b59bb034969cb41b4dbf1e942c3e8d63a53478f Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 8 May 2023 20:56:17 +0000 Subject: [PATCH 68/73] test(buffers): write tests for JS TypedArray/ArrayBuffer <-> Python memoryview coercion --- tests/python/test_pythonmonkey_eval.py | 47 +++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 190a1903..55f56113 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -894,7 +894,7 @@ def assert_js_to_py_memoryview(buf: memoryview): assert type(buf) is memoryview assert None == buf.obj # https://docs.python.org/3.9/c-api/buffer.html#c.Py_buffer.obj assert 2 * 4 == buf.nbytes # 2 elements * sizeof(int32_t) - assert "02000000ffffffff" == buf.hex() # little endian + assert "02000000ffffffff" == buf.hex() # native (little) endian buf1 = pm.eval("new Int32Array([2,-1])") buf2 = pm.eval("new Int32Array([2,-1]).buffer") assert_js_to_py_memoryview(buf1) @@ -907,6 +907,17 @@ def assert_js_to_py_memoryview(buf: memoryview): buf1[2] with pytest.raises(IndexError, match="index out of bounds on dimension 1"): buf2[8] + del buf1, buf2 + + # test element value ranges + buf3 = pm.eval("new Uint8Array(1)") + with pytest.raises(ValueError, match="memoryview: invalid value for format 'B'"): + buf3[0] = 256 + with pytest.raises(ValueError, match="memoryview: invalid value for format 'B'"): + buf3[0] = -1 + with pytest.raises(IndexError, match="index out of bounds on dimension 1"): # no automatic resize + buf3[1] = 0 + del buf3 # Python buffers should coerce to JS TypedArray # and the typecode maps to TypedArray subtype (Uint8Array, Float64Array, ...) @@ -932,6 +943,40 @@ def assert_js_to_py_memoryview(buf: memoryview): assert pm.eval("new Float32Array([1])").format == "f" assert pm.eval("new Float64Array([1])").format == "d" + # not enough bytes to populate an element of the TypedArray + with pytest.raises(pm.SpiderMonkeyError, match="RangeError: buffer length for BigInt64Array should be a multiple of 8"): + pm.eval("(arr) => new BigInt64Array(arr.buffer)")(array.array('i', [-11111111])) + + # TypedArray with `byteOffset` and `length` + arr1 = array.array('i', [-11111111, 22222222, -33333333, 44444444]) + with pytest.raises(pm.SpiderMonkeyError, match="RangeError: invalid or out-of-range index"): + pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ -4)")(arr1) + with pytest.raises(pm.SpiderMonkeyError, match="RangeError: start offset of Int32Array should be a multiple of 4"): + pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 1)")(arr1) + with pytest.raises(pm.SpiderMonkeyError, match="RangeError: size of buffer is too small for Int32Array with byteOffset"): + pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 20)")(arr1) + with pytest.raises(pm.SpiderMonkeyError, match="RangeError: invalid or out-of-range index"): + pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 4, /*length*/ -1)")(arr1) + with pytest.raises(pm.SpiderMonkeyError, match="RangeError: attempting to construct out-of-bounds Int32Array on ArrayBuffer"): + pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 4, /*length*/ 4)")(arr1) + arr2 = pm.eval("(arr) => new Int32Array(arr.buffer, /*byteOffset*/ 4, /*length*/ 2)")(arr1) + assert 2 * 4 == arr2.nbytes # 2 elements * sizeof(int32_t) + assert [22222222, -33333333] == arr2.tolist() + assert "8e155301ab5f03fe" == arr2.hex() # native (little) endian + assert 22222222 == arr2[0] # offset 1 int32 + with pytest.raises(IndexError, match="index out of bounds on dimension 1"): + arr2[2] + arr3 = pm.eval("(arr) => new Int32Array(arr.buffer, 16 /* byteOffset */)")(arr1) # empty Int32Array + assert 0 == arr3.nbytes + del arr3 + + # test GC + del arr1 + gc.collect(), pm.collect() + gc.collect(), pm.collect() + # TODO (Tom Tang): the 0th element in the underlying buffer is still accessible after GC, even is not referenced by the JS TypedArray with byteOffset + del arr2 + # simple 1-D numpy array should just work as well numpy_int16_array = numpy.array([0, 1, 2, 3], dtype=numpy.int16) assert "0,1,2,3" == pm.eval("(typedArray) => typedArray.toString()")(numpy_int16_array) From dca5d9bbe10332095376c2cde77768fc5b823320 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 8 May 2023 22:49:36 +0000 Subject: [PATCH 69/73] test(buffers): write tests for buffers mutation --- tests/python/test_pythonmonkey_eval.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index 55f56113..f7440810 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -977,6 +977,32 @@ def assert_js_to_py_memoryview(buf: memoryview): # TODO (Tom Tang): the 0th element in the underlying buffer is still accessible after GC, even is not referenced by the JS TypedArray with byteOffset del arr2 + # mutation + mut_arr_original = bytearray(4) + pm.eval(""" + (/* @type Uint8Array */ arr) => { + // 2.25 in float32 little endian + arr[2] = 0x10 + arr[3] = 0x40 + } + """)(mut_arr_original) + assert 0x10 == mut_arr_original[2] + assert 0x40 == mut_arr_original[3] + # mutation to a different TypedArray accessing the same underlying data block will also change the original buffer + def do_mutation(mut_arr_js): + assert 2.25 == mut_arr_js[0] + mut_arr_js[0] = 225.50048828125 # float32 little endian: 0x 20 80 61 43 + assert "20806143" == mut_arr_original.hex() + assert 225.50048828125 == array.array("f", mut_arr_original)[0] + mut_arr_new = pm.eval(""" + (/* @type Uint8Array */ arr, do_mutation) => { + const mut_arr_js = new Float32Array(arr.buffer) + do_mutation(mut_arr_js) + return arr + } + """)(mut_arr_original, do_mutation) + assert [0x20, 0x80, 0x61, 0x43] == mut_arr_new.tolist() + # simple 1-D numpy array should just work as well numpy_int16_array = numpy.array([0, 1, 2, 3], dtype=numpy.int16) assert "0,1,2,3" == pm.eval("(typedArray) => typedArray.toString()")(numpy_int16_array) From 45e4b4c85cb914c0219713cc2e8534fce3ca7100 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Fri, 12 May 2023 14:56:43 +0000 Subject: [PATCH 70/73] refactor(buffers): use `sizeof(int8_t)` instead of hard-coded byte size `1` --- src/BufferType.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/BufferType.cc b/src/BufferType.cc index 30b66af7..30c04f44 100644 --- a/src/BufferType.cc +++ b/src/BufferType.cc @@ -165,13 +165,13 @@ JS::Scalar::Type BufferType::_getPyBufferType(Py_buffer *bufView) { bool isSigned = std::islower(typeCode); // e.g. 'b' for signed char, 'B' for unsigned char uint8_t byteSize = bufView->itemsize; switch (byteSize) { - case 1: + case sizeof(int8_t): return isSigned ? JS::Scalar::Int8 : JS::Scalar::Uint8; // TODO (Tom Tang): Uint8Clamped - case 2: + case sizeof(int16_t): return isSigned ? JS::Scalar::Int16 : JS::Scalar::Uint16; - case 4: + case sizeof(int32_t): return isSigned ? JS::Scalar::Int32 : JS::Scalar::Uint32; - case 8: + case sizeof(int64_t): return isSigned ? JS::Scalar::BigInt64 : JS::Scalar::BigUint64; default: return JS::Scalar::MaxTypedArrayViewType; // invalid byteSize From b4e73adaf1c08c01d631202ae8e9fd78499a24a8 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Fri, 26 May 2023 06:39:08 +0000 Subject: [PATCH 71/73] ci: force GC whenever a test function finishes, to locate GC-related errors --- tests/python/test_pythonmonkey_eval.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index f7440810..83f11337 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -7,6 +7,15 @@ import asyncio import numpy, array, struct +# https://doc.pytest.org/en/latest/how-to/xunit_setup.html#method-and-function-level-setup-teardown +def teardown_function(function): + """ + Forcing garbage collection (twice) whenever a test function finishes, + to locate GC-related errors + """ + gc.collect(), pm.collect() + gc.collect(), pm.collect() + def test_passes(): assert True From fbf3628e9e1b8232a3ee72a9eaf245f7d3d86fb1 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Fri, 26 May 2023 06:46:55 +0000 Subject: [PATCH 72/73] fix(GC): in Python 3.11+, python function object would be double-freed on GC in `handleSharedPythonMonkeyMemory` --- src/jsTypeFactory.cc | 1 + src/modules/pythonmonkey/pythonmonkey.cc | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jsTypeFactory.cc b/src/jsTypeFactory.cc index f8868d31..353a17d5 100644 --- a/src/jsTypeFactory.cc +++ b/src/jsTypeFactory.cc @@ -143,6 +143,7 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) { js::SetFunctionNativeReserved(jsFuncObject, 0, JS::PrivateValue((void *)object)); returnType.setObject(*jsFuncObject); memoizePyTypeAndGCThing(new FuncType(object), returnType); + Py_INCREF(object); // otherwise the python function object would be double-freed on GC in Python 3.11+ } else if (PyExceptionInstance_Check(object)) { JSObject *error = ExceptionType(object).toJsError(cx); diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 16858ea9..0f4e3e39 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -87,9 +87,10 @@ void handleSharedPythonMonkeyMemory(JSContext *cx, JSGCStatus status, JS::GCReas if (status == JSGCStatus::JSGC_BEGIN) { PyToGCIterator pyIt = PyTypeToGCThing.begin(); while (pyIt != PyTypeToGCThing.end()) { + PyObject *pyObj = pyIt->first->getPyObject(); // If the PyObject reference count is exactly 1, then the only reference to the object is the one // we are holding, which means the object is ready to be free'd. - if (PyObject_GC_IsFinalized(pyIt->first->getPyObject()) || pyIt->first->getPyObject()->ob_refcnt == 1) { + if (PyObject_GC_IsFinalized(pyObj) || pyObj->ob_refcnt == 1) { for (JS::PersistentRooted *rval: pyIt->second) { // for each related GCThing bool found = false; for (PyToGCIterator innerPyIt = PyTypeToGCThing.begin(); innerPyIt != PyTypeToGCThing.end(); innerPyIt++) { // for each other PyType pointer From 298fcb367e69dea60a225bd581895ad089ebf9d5 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Fri, 16 Jun 2023 22:12:53 +0000 Subject: [PATCH 73/73] fix(event-loop): `PyRunningLoopHolder` is removed in Python 3.12+ --- src/PyEventLoop.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/PyEventLoop.cc b/src/PyEventLoop.cc index 8393fb31..38395d3a 100644 --- a/src/PyEventLoop.cc +++ b/src/PyEventLoop.cc @@ -68,12 +68,18 @@ PyEventLoop PyEventLoop::_getLoopOnThread(PyThreadState *tstate) { return _loopNotFound(); } +#if PY_VERSION_HEX < 0x030c0000 // Python version is less than 3.12 using PyRunningLoopHolder = struct { PyObject_HEAD PyObject *rl_loop; }; PyObject *running_loop = ((PyRunningLoopHolder *)rl)->rl_loop; +#else + // The running loop is simply the `rl` object in Python 3.12+ + // see https://github.com/python/cpython/blob/v3.12.0b2/Modules/_asynciomodule.c#L301 + PyObject *running_loop = rl; +#endif if (running_loop == Py_None) { return _loopNotFound(); }