Skip to content

Commit

Permalink
NFC Use initialization function to load _pyodide_core (#3333)
Browse files Browse the repository at this point in the history 
This adds `_pyodide_core` to the init tab and initializes it using that. This is a small step toward unvendoring the Pyodide ffi.
  • Loading branch information
@hoodmane
hoodmane committed Dec 10, 2022
1 parent 8560319 commit 653310a
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 89 deletions.
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ dist/pyodide.asm.js: \
src/core/docstring.o \
src/core/error_handling.o \
src/core/hiwire.o \
src/core/_pyodide_core.o \
src/core/js2python.o \
src/core/jsproxy.o \
src/core/main.o \
Expand Down
89 changes: 89 additions & 0 deletions src/core/_pyodide_core.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "hiwire.h"
#include "python2js.h"
#include <emscripten.h>
#include <stdbool.h>

#define FATAL_ERROR(args...) \
do { \
PyErr_Format(PyExc_ImportError, args); \
FAIL(); \
} while (0)

#define FAIL_IF_STATUS_EXCEPTION(status) \
if (PyStatus_Exception(status)) { \
goto finally; \
}

#define TRY_INIT(mod) \
do { \
int mod##_init(); \
if (mod##_init()) { \
FATAL_ERROR("Failed to initialize module %s.", #mod); \
} \
} while (0)

#define TRY_INIT_WITH_CORE_MODULE(mod) \
do { \
int mod##_init(PyObject* mod); \
if (mod##_init(core_module)) { \
FATAL_ERROR("Failed to initialize module %s.", #mod); \
} \
} while (0)

static struct PyModuleDef core_module_def = {
PyModuleDef_HEAD_INIT,
.m_name = "_pyodide_core",
.m_doc = "Pyodide C builtins",
.m_size = -1,
};

PyObject*
PyInit__pyodide_core(void)
{
bool success = false;
PyObject* _pyodide = NULL;
PyObject* core_module = NULL;
JsRef _pyodide_proxy = NULL;

_pyodide = PyImport_ImportModule("_pyodide");
if (_pyodide == NULL) {
FATAL_ERROR("Failed to import _pyodide module.");
}

core_module = PyModule_Create(&core_module_def);
if (core_module == NULL) {
FATAL_ERROR("Failed to create core module.");
}

TRY_INIT_WITH_CORE_MODULE(error_handling);
TRY_INIT(hiwire);
TRY_INIT(docstring);
TRY_INIT(js2python);
TRY_INIT_WITH_CORE_MODULE(python2js);
TRY_INIT(python2js_buffer);
TRY_INIT_WITH_CORE_MODULE(JsProxy);
TRY_INIT_WITH_CORE_MODULE(pyproxy);

PyObject* module_dict = PyImport_GetModuleDict(); /* borrowed */
if (PyDict_SetItemString(module_dict, "_pyodide_core", core_module)) {
FATAL_ERROR("Failed to add '_pyodide_core' module to modules dict.");
FAIL();
}

// Enable JavaScript access to the _pyodide module.
_pyodide_proxy = python2js(_pyodide);
if (_pyodide_proxy == NULL) {
FATAL_ERROR("Failed to create _pyodide proxy.");
}
EM_ASM({ API._pyodide = Hiwire.pop_value($0); }, _pyodide_proxy);

success = true;
finally:
Py_CLEAR(_pyodide);
if (!success) {
Py_CLEAR(core_module);
}
return core_module;
}
86 changes: 4 additions & 82 deletions src/core/main.c
View file
Original file line number Diff line number Diff line change
@@ -1,45 +1,13 @@
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include <assert.h>
#include <emscripten.h>
#include <stdalign.h>

#include "docstring.h"
#include "error_handling.h"
#include "hiwire.h"
#include "js2python.h"
#include "jsproxy.h"
#include "pyproxy.h"
#include "python2js.h"
#include "python2js_buffer.h"

#define FATAL_ERROR(args...) \
do { \
EM_ASM(API.fatal_loading_error(args)); \
return -1; \
} while (0)
#include <stdbool.h>

#define FAIL_IF_STATUS_EXCEPTION(status) \
if (PyStatus_Exception(status)) { \
goto finally; \
}

#define TRY_INIT(mod) \
do { \
int mod##_init(); \
if (mod##_init()) { \
FATAL_ERROR("Failed to initialize module ", #mod, "."); \
} \
} while (0)

#define TRY_INIT_WITH_CORE_MODULE(mod) \
do { \
int mod##_init(PyObject* mod); \
if (mod##_init(core_module)) { \
FATAL_ERROR("Failed to initialize module", #mod, "."); \
} \
} while (0)

// Initialize python. exit() and print message to stderr on failure.
static void
initialize_python(int argc, char** argv)
Expand Down Expand Up @@ -74,12 +42,8 @@ initialize_python(int argc, char** argv)
}
}

static struct PyModuleDef core_module_def = {
PyModuleDef_HEAD_INIT,
.m_name = "_pyodide_core",
.m_doc = "Pyodide C builtins",
.m_size = -1,
};
PyObject*
PyInit__pyodide_core(void);

/**
* Bootstrap steps here:
Expand All @@ -96,54 +60,12 @@ main(int argc, char** argv)
{
// This exits and prints a message to stderr on failure,
// no status code to check.
PyImport_AppendInittab("_pyodide_core", PyInit__pyodide_core);
initialize_python(argc, argv);
emscripten_exit_with_live_runtime();
return 0;
}

int
pyodide_init(void)
{
PyObject* _pyodide = NULL;
PyObject* core_module = NULL;
JsRef _pyodide_proxy = NULL;

_pyodide = PyImport_ImportModule("_pyodide");
if (_pyodide == NULL) {
FATAL_ERROR("Failed to import _pyodide module.");
}

core_module = PyModule_Create(&core_module_def);
if (core_module == NULL) {
FATAL_ERROR("Failed to create core module.");
}

TRY_INIT_WITH_CORE_MODULE(error_handling);
TRY_INIT(hiwire);
TRY_INIT(docstring);
TRY_INIT(js2python);
TRY_INIT_WITH_CORE_MODULE(python2js);
TRY_INIT(python2js_buffer);
TRY_INIT_WITH_CORE_MODULE(JsProxy);
TRY_INIT_WITH_CORE_MODULE(pyproxy);

PyObject* module_dict = PyImport_GetModuleDict(); /* borrowed */
if (PyDict_SetItemString(module_dict, "_pyodide_core", core_module)) {
FATAL_ERROR("Failed to add '_pyodide_core' module to modules dict.");
}

// Enable JavaScript access to the _pyodide module.
_pyodide_proxy = python2js(_pyodide);
if (_pyodide_proxy == NULL) {
FATAL_ERROR("Failed to create _pyodide proxy.");
}
EM_ASM({ API._pyodide = Hiwire.pop_value($0); }, _pyodide_proxy);

Py_CLEAR(_pyodide);
Py_CLEAR(core_module);
return 0;
}

void
pymain_run_python(int* exitcode);

Expand Down
12 changes: 12 additions & 0 deletions src/js/api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,18 @@ export let pyodide_py: PyProxy; // actually defined in loadPyodide (see pyodide.
*/
export let globals: PyProxy; // actually defined in loadPyodide (see pyodide.js)

/**
* Runs code after python vm has been initialized but prior to any bootstrapping.
*/
API.rawRun = function rawRun(code: string): [number, string] {
const code_ptr = Module.stringToNewUTF8(code);
Module.API.capture_stderr();
let errcode = Module._PyRun_SimpleString(code_ptr);
Module._free(code_ptr);
const captured_stderr = Module.API.restore_stderr().trim();
return [errcode, captured_stderr];
};

/**
* Just like `runPython` except uses a different globals dict and gets
* `eval_code` from `_pyodide` so that it can work before `pyodide` is imported.
Expand Down
12 changes: 5 additions & 7 deletions src/js/pyodide.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ function unpackPyodidePy(Module: any, pyodide_py_tar: Uint8Array) {
true,
);
Module.FS.close(stream);
const code_ptr = Module.stringToNewUTF8(`

const code = `
from sys import version_info
pyversion = f"python{version_info.major}.{version_info.minor}"
import shutil
Expand All @@ -89,17 +90,14 @@ del shutil
import importlib
importlib.invalidate_caches()
del importlib
`);
Module.API.capture_stderr();
let errcode = Module._PyRun_SimpleString(code_ptr);
const captured_stderr = Module.API.restore_stderr().trim();
`;
let [errcode, captured_stderr] = Module.API.rawRun(code);
if (errcode) {
Module.API.fatal_loading_error(
"Failed to unpack standard library.\n",
captured_stderr,
);
}
Module._free(code_ptr);
Module.FS.unlink(fileName);
}

Expand Down Expand Up @@ -348,7 +346,7 @@ If you updated the Pyodide version, make sure you also updated the 'indexURL' pa

const pyodide_py_tar = await pyodide_py_tar_promise;
unpackPyodidePy(Module, pyodide_py_tar);
Module._pyodide_init();
API.rawRun("import _pyodide_core");

const pyodide = finalizeBootstrap(API, config);

Expand Down

0 comments on commit 653310a

Please sign in to comment.