From 0f6a10256e617f1b2d02f9cec1bfb25f321d9a27 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 22 Jan 2022 23:08:30 +1100 Subject: [PATCH] WIP: Make mp_module_context_t derive from mp_obj_module_t. Signed-off-by: Damien George --- mpy-cross/main.c | 3 ++- ports/unix/main.c | 4 +++- py/bc.c | 4 ++-- py/bc.h | 32 +++++++++++++++++++++++++------- py/builtinevex.c | 2 +- py/builtinimport.c | 17 +++++++++-------- py/compile.c | 14 +++++++------- py/compile.h | 4 ++-- py/emit.h | 27 +++++++++++++++------------ py/emitnative.c | 9 ++++----- py/frozenmod.c | 2 +- py/gc.c | 2 ++ py/objfun.c | 10 +++++----- py/objfun.h | 2 +- py/objgenerator.c | 4 ++-- py/objmodule.c | 9 +++++---- py/persistentcode.c | 22 +++++++++++----------- py/persistentcode.h | 6 +++--- py/runtime.c | 4 +++- py/showbc.c | 18 +++++++++--------- py/vm.c | 16 ++++++++-------- shared/runtime/pyexec.c | 12 ++++++++---- tools/mpy-tool.py | 38 ++++++++++++++++++++++++-------------- 23 files changed, 152 insertions(+), 109 deletions(-) diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 25b068ab361c7..a2daf7b22fd69 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -72,7 +72,8 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha #endif mp_parse_tree_t parse_tree = mp_parse(lex, MP_PARSE_FILE_INPUT); - mp_compiled_module_t cm = mp_compile_to_raw_code(&parse_tree, source_name, false); + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + mp_compiled_module_t cm = mp_compile_to_raw_code(&parse_tree, source_name, false, ctx); vstr_t vstr; vstr_init(&vstr, 16); diff --git a/ports/unix/main.c b/ports/unix/main.c index b2790791af856..6aa8ac5a3297a 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -138,7 +138,9 @@ STATIC int execute_from_lexer(int source_kind, const void *source, mp_parse_inpu } #endif - mp_obj_t module_fun = mp_compile(&parse_tree, source_name, is_repl); + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + ctx->module.globals = mp_globals_get(); + mp_obj_t module_fun = mp_compile(&parse_tree, source_name, is_repl, ctx); if (!compile_only) { // execute it diff --git a/py/bc.c b/py/bc.c index 3056c80cfa3ca..43136b3cb94d3 100644 --- a/py/bc.c +++ b/py/bc.c @@ -202,7 +202,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw for (size_t j = 0; j < n_pos_args + n_kwonly_args; j++) { qstr arg_qstr = mp_decode_uint(&arg_names); #if MICROPY_PERSISTENT_CODE - arg_qstr = self->mc->qstr_table[arg_qstr]; + arg_qstr = self->ctx->constants.qstr_table[arg_qstr]; #endif if (wanted_arg_name == MP_OBJ_NEW_QSTR(arg_qstr)) { if (code_state->state[n_state - 1 - j] != MP_OBJ_NULL) { @@ -258,7 +258,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw for (size_t i = 0; i < n_kwonly_args; i++) { qstr arg_qstr = mp_decode_uint(&arg_names); #if MICROPY_PERSISTENT_CODE - arg_qstr = self->mc->qstr_table[arg_qstr]; + arg_qstr = self->ctx->constants.qstr_table[arg_qstr]; #endif if (code_state->state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) { mp_map_elem_t *elem = NULL; diff --git a/py/bc.h b/py/bc.h index d0a77ae1f3031..31ede9e71cb31 100644 --- a/py/bc.h +++ b/py/bc.h @@ -197,15 +197,21 @@ typedef struct _mp_exc_stack_t { } mp_exc_stack_t; // State associated with a module, to interface bytecode with runtime. -// Outer level struct defining a compiled module. +typedef struct _mp_module_constants_t { + qstr_short_t *qstr_table; + mp_obj_t *obj_table; +} mp_module_constants_t; + +// State associated with a module, to interface bytecode with runtime. typedef struct _mp_module_context_t { - mp_obj_dict_t **globals_ref; - mp_obj_t *const_table; - qstr_short_t qstr_table[]; + mp_obj_module_t module; + mp_module_constants_t constants; } mp_module_context_t; + +// Outer level struct defining a compiled module. typedef struct _mp_compiled_module_t { + const mp_module_context_t *ctx; const struct _mp_raw_code_t *rc; - const mp_module_context_t *mc; #if MICROPY_PERSISTENT_CODE // placed here, this is part of the ABI b/c native code assumes offsets of entries below bool has_native; @@ -214,6 +220,18 @@ typedef struct _mp_compiled_module_t { #endif } mp_compiled_module_t; +// Outer level struct defining a compiled module. +typedef struct _mp_frozen_module_t { + const mp_module_constants_t constants; + const struct _mp_raw_code_t *rc; + #if MICROPY_PERSISTENT_CODE + // placed here, this is part of the ABI b/c native code assumes offsets of entries below + bool has_native; + size_t n_qstr; + size_t n_obj; + #endif +} mp_frozen_module_t; + typedef struct _mp_code_state_t { // The fun_bc entry points to the underlying function object that is being executed. // It is needed to access the start of bytecode and the const_table. @@ -245,8 +263,8 @@ const byte *mp_decode_uint_skip(const byte *ptr); mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp_obj_t inject_exc); mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, size_t n_args, size_t n_kw, const mp_obj_t *args); void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); -void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *code, mp_uint_t len, const mp_module_context_t *cm); -void mp_bytecode_print2(const mp_print_t *print, const byte *code, size_t len, const mp_module_context_t *cm); +void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *code, mp_uint_t len, const mp_module_constants_t *cm); +void mp_bytecode_print2(const mp_print_t *print, const byte *code, size_t len, const mp_module_constants_t *cm); const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip); #define mp_bytecode_print_inst(print, code, x_table) mp_bytecode_print2(print, code, 1, x_table) diff --git a/py/builtinevex.c b/py/builtinevex.c index f4f8914cd32fb..5af74d1731d40 100644 --- a/py/builtinevex.c +++ b/py/builtinevex.c @@ -54,7 +54,7 @@ STATIC mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj // the correct one if (mp_obj_is_type(self->module_fun, &mp_type_fun_bc)) { mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(self->module_fun); - *((const mp_module_context_t *)fun_bc->mc)->globals_ref = globals; + ((mp_module_context_t *)fun_bc->ctx)->module.globals = globals; } // execute code diff --git a/py/builtinimport.c b/py/builtinimport.c index c7c5f4d1692a4..91de9f0421cb0 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -168,7 +168,6 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, const mp_raw_code_t *rc, co // execute the module in its context mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj); - *mc->globals_ref = mod_globals; // save context mp_obj_dict_t *volatile old_globals = mp_globals_get(); @@ -196,7 +195,7 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, const mp_raw_code_t *rc, co } #endif -STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { +STATIC void do_load(mp_module_context_t *module_obj, vstr_t *file) { #if MICROPY_MODULE_FROZEN || MICROPY_ENABLE_COMPILER || (MICROPY_PERSISTENT_CODE_LOAD && MICROPY_HAS_FILE_READER) const char *file_str = vstr_null_terminated_str(file); #endif @@ -223,8 +222,10 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // its data) in the list of frozen files, execute it. #if MICROPY_MODULE_FROZEN_MPY if (frozen_type == MP_FROZEN_MPY) { - const mp_compiled_module_t *cm = modref; - do_execute_raw_code(module_obj, cm->rc, cm->mc, file_str + frozen_path_prefix_len); + const mp_frozen_module_t *frozen = modref; + module_obj->constants = frozen->constants; + module_obj->module.globals = mp_obj_module_get_globals(module_obj); + do_execute_raw_code(module_obj, frozen->rc, module_obj, file_str + frozen_path_prefix_len); return; } #endif @@ -236,8 +237,8 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // the correct format and, if so, load and execute the file. #if MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD if (file_str[file->len - 3] == 'm') { - mp_compiled_module_t cm = mp_raw_code_load_file(file_str); - do_execute_raw_code(module_obj, cm.rc, cm.mc, file_str); + mp_compiled_module_t cm = mp_raw_code_load_file(file_str, module_obj); + do_execute_raw_code(module_obj, cm.rc, cm.ctx, file_str); return; } #endif @@ -436,7 +437,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, size_t orig_path_len = path->len; vstr_add_str(path, PATH_SEP_CHAR "__init__.py"); if (stat_file_py_or_mpy(path) == MP_IMPORT_STAT_FILE) { - do_load(module_obj, path); + do_load(MP_OBJ_TO_PTR(module_obj), path); } else { // No-op. Nothing to load. // mp_warning("%s is imported as namespace package", vstr_str(&path)); @@ -445,7 +446,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, path->len = orig_path_len; } else { // MP_IMPORT_STAT_FILE // File -- execute "path.(m)py". - do_load(module_obj, path); + do_load(MP_OBJ_TO_PTR(module_obj), path); // Note: This should be the last component in the import path. If // there are remaining components then it's an ImportError // because the current path(the module that was just loaded) is diff --git a/py/compile.c b/py/compile.c index 692aea83f1c18..aa4bd85ac630d 100644 --- a/py/compile.c +++ b/py/compile.c @@ -3417,7 +3417,7 @@ STATIC void scope_compute_things(scope_t *scope, mp_emit_common_t *emit_common) #if !MICROPY_PERSISTENT_CODE_SAVE STATIC #endif -mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) { +mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *ctx) { // put compiler state on the stack, it's relatively small compiler_t comp_state = {0}; compiler_t *comp = &comp_state; @@ -3567,17 +3567,17 @@ mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr so // Construct the global qstr/const table for this module. mp_compiled_module_t cm; cm.rc = module_scope->raw_code; - cm.mc = NULL; + cm.ctx = ctx; #if MICROPY_PERSISTENT_CODE_SAVE cm.has_native = has_native_code; #endif if (comp->compile_error == MP_OBJ_NULL) { - mp_emit_common_create_module_context(&comp->emit_common, &cm); + mp_emit_common_create_module_context(&comp->emit_common, &cm, ctx); #if MICROPY_DEBUG_PRINTERS if (mp_verbose_flag >= 2) { for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) { mp_raw_code_t *rc = s->raw_code; - mp_bytecode_print(&mp_plat_print, rc, rc->fun_data, rc->fun_data_len, cm.mc); + mp_bytecode_print(&mp_plat_print, rc, rc->fun_data, rc->fun_data_len, &cm.ctx->constants); } } #endif @@ -3614,10 +3614,10 @@ mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr so return cm; } -mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) { - mp_compiled_module_t cm = mp_compile_to_raw_code(parse_tree, source_file, is_repl); +mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *ctx) { + mp_compiled_module_t cm = mp_compile_to_raw_code(parse_tree, source_file, is_repl, ctx); // return function that executes the outer module - return mp_make_function_from_raw_code(cm.rc, cm.mc, NULL); + return mp_make_function_from_raw_code(cm.rc, cm.ctx, NULL); } #endif // MICROPY_ENABLE_COMPILER diff --git a/py/compile.h b/py/compile.h index dc69ba5fe011f..7555993d30bc5 100644 --- a/py/compile.h +++ b/py/compile.h @@ -32,11 +32,11 @@ // the compiler will raise an exception if an error occurred // the compiler will clear the parse tree before it returns -mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); +mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *globals); #if MICROPY_PERSISTENT_CODE_SAVE // this has the same semantics as mp_compile -mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); +mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *globals); #endif // this is implemented in runtime.c diff --git a/py/emit.h b/py/emit.h index 47b8f718e0518..227c588296dc6 100644 --- a/py/emit.h +++ b/py/emit.h @@ -229,28 +229,31 @@ static inline void mp_emit_common_finalise(mp_emit_common_t *emit, bool has_nati } #include -static inline mp_module_context_t *mp_module_context_new(size_t n_qstr, size_t n_obj) { +static inline mp_module_context_t *mp_module_context_new(mp_module_context_t *ctx, size_t n_qstr, size_t n_obj) { + //size_t nctx = (sizeof(mp_module_context_t) + sizeof(mp_uint_t) - 1) / sizeof(mp_uint_t); + size_t nctx = 0; size_t nq = (n_qstr * sizeof(qstr_short_t) + sizeof(mp_uint_t) - 1) / sizeof(mp_uint_t); - size_t nc = n_obj; - mp_module_context_t *mc = m_new_obj_var(mp_module_context_t, mp_uint_t, nq + 1 + nc); - mc->globals_ref = (mp_obj_dict_t **)&((mp_uint_t *)&mc->qstr_table[0])[nq]; - *mc->globals_ref = mp_globals_get(); - mc->const_table = (mp_obj_t *)&((mp_uint_t *)&mc->qstr_table[0])[nq + 1]; - return mc; + size_t no = n_obj; + mp_uint_t *mem = m_new(mp_uint_t, nctx + nq + no); + //mp_module_context_t *ctx = (void *)mem; + ctx->constants.qstr_table = (void *)(mem + nctx); + ctx->constants.obj_table = (void *)(mem + nctx + nq); + //ctx->globals = globals; + return ctx; } -static inline void mp_emit_common_create_module_context(mp_emit_common_t *emit, mp_compiled_module_t *cm) { +static inline void mp_emit_common_create_module_context(mp_emit_common_t *emit, mp_compiled_module_t *cm, mp_module_context_t *ctx) { assert(emit->ct_cur_obj == emit->ct_num_obj); size_t qstr_map_used = emit->qstr_map.used; - mp_module_context_t *mc = mp_module_context_new(qstr_map_used, emit->ct_cur_obj); + mp_module_context_new(ctx, qstr_map_used, emit->ct_cur_obj); for (size_t i = 0; i < emit->qstr_map.alloc; ++i) { if (mp_map_slot_is_filled(&emit->qstr_map, i)) { size_t idx = MP_OBJ_SMALL_INT_VALUE(emit->qstr_map.table[i].value); qstr qst = MP_OBJ_QSTR_VALUE(emit->qstr_map.table[i].key); - mc->qstr_table[idx] = qst; + ctx->constants.qstr_table[idx] = qst; } } - memcpy(mc->const_table, emit->const_table, emit->ct_cur_obj * sizeof(mp_uint_t)); + memcpy(ctx->constants.obj_table, emit->const_table, emit->ct_cur_obj * sizeof(mp_uint_t)); #if 0 printf("table %d\n", (int)emit->ct_cur_obj); printf(" %p\n", mc->const_table[0]); @@ -258,7 +261,7 @@ static inline void mp_emit_common_create_module_context(mp_emit_common_t *emit, printf(" %p\n", mc->const_table[2]); #endif - cm->mc = mc; + cm->ctx = ctx; #if MICROPY_PERSISTENT_CODE_SAVE cm->n_qstr = emit->qstr_map.used; cm->n_obj = emit->ct_cur_obj; diff --git a/py/emitnative.c b/py/emitnative.c index d7ca4ef747461..c1741a1f963d4 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -95,10 +95,10 @@ #define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_t, sp) / sizeof(uintptr_t)) //#define OFFSETOF_OBJ_FUN_BC_GLOBALS (offsetof(mp_obj_fun_bc_t, globals) / sizeof(uintptr_t)) #define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t)) -#define OFFSETOF_OBJ_FUN_BC_MC (offsetof(mp_obj_fun_bc_t, mc) / sizeof(uintptr_t)) +#define OFFSETOF_OBJ_FUN_BC_MC (offsetof(mp_obj_fun_bc_t, ctx) / sizeof(uintptr_t)) #define OFFSETOF_OBJ_FUN_BC_RAW_CODE_TABLE (offsetof(mp_obj_fun_bc_t, raw_code_table) / sizeof(uintptr_t)) -#define OFFSETOF_MODULE_CONTEXT_CONST_TABLE (offsetof(mp_module_context_t, const_table) / sizeof(uintptr_t)) -#define OFFSETOF_MODULE_CONTEXT_GLOBALS_REF (offsetof(mp_module_context_t, globals_ref) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_CONST_TABLE (offsetof(mp_module_context_t, constants.obj_table) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_GLOBALS (offsetof(mp_module_context_t, module.globals) / sizeof(uintptr_t)) #define INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE (0) // If not already defined, set parent args to same as child call registers @@ -1185,8 +1185,7 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) { // Set new globals emit_native_mov_reg_state(emit, REG_ARG_1, LOCAL_IDX_FUN_OBJ(emit)); ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_MC); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_MODULE_CONTEXT_GLOBALS_REF); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, 0); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_MODULE_CONTEXT_GLOBALS); emit_call(emit, MP_F_NATIVE_SWAP_GLOBALS); // Save old globals (or NULL if globals didn't change) diff --git a/py/frozenmod.c b/py/frozenmod.c index 1562588bf6e7a..61c2f20aa1de3 100644 --- a/py/frozenmod.c +++ b/py/frozenmod.c @@ -57,7 +57,7 @@ extern const char mp_frozen_str_content[]; #include "py/emitglue.h" -extern const mp_compiled_module_t *const mp_frozen_mpy_content[]; +extern const mp_frozen_module_t *const mp_frozen_mpy_content[]; #endif // MICROPY_MODULE_FROZEN_MPY diff --git a/py/gc.c b/py/gc.c index b14f00968797d..39c3439672708 100644 --- a/py/gc.c +++ b/py/gc.c @@ -339,12 +339,14 @@ void gc_collect_start(void) { size_t root_end = offsetof(mp_state_ctx_t, vm.qstr_last_chunk); gc_collect_root(ptrs + root_start / sizeof(void *), (root_end - root_start) / sizeof(void *)); + #if 0 #if MICROPY_MODULE_FROZEN_MPY // Trace root pointers from frozen modules. extern void *mp_frozen_root_pointers[]; extern size_t mp_frozen_root_pointers_len; gc_collect_root(mp_frozen_root_pointers, mp_frozen_root_pointers_len); #endif + #endif #if MICROPY_ENABLE_PYSTACK // Trace root pointers from the Python stack. diff --git a/py/objfun.c b/py/objfun.c index 671db0f1fb87f..b92b248ed13ec 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -167,7 +167,7 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) { const byte *bc = fun->bytecode; MP_BC_PRELUDE_SIG_DECODE(bc); - return mp_obj_code_get_name(bc, fun->mc->qstr_table); + return mp_obj_code_get_name(bc, fun->ctx->constants.qstr_table); } #if MICROPY_CPYTHON_COMPAT @@ -285,7 +285,7 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args); // execute the byte code with the correct globals context - mp_globals_set(*self->mc->globals_ref); + mp_globals_set(self->ctx->module.globals); mp_vm_return_kind_t vm_return_kind = mp_execute_bytecode(code_state, MP_OBJ_NULL); mp_globals_set(code_state->old_globals); @@ -358,7 +358,7 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { } if (attr == MP_QSTR___globals__) { mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); - dest[0] = MP_OBJ_FROM_PTR(*self->mc->globals_ref); + dest[0] = MP_OBJ_FROM_PTR(self->ctx->module.globals); } } #endif @@ -377,7 +377,7 @@ const mp_obj_type_t mp_type_fun_bc = { #endif }; -mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *mc, struct _mp_raw_code_t *const *raw_code_table) { +mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *ctx, struct _mp_raw_code_t *const *raw_code_table) { size_t n_def_args = 0; size_t n_extra_args = 0; mp_obj_tuple_t *def_pos_args = NULL; @@ -396,7 +396,7 @@ mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_ mp_obj_fun_bc_t *o = m_new_obj_var(mp_obj_fun_bc_t, mp_obj_t, n_extra_args); o->base.type = &mp_type_fun_bc; o->bytecode = code; - o->mc = mc; + o->ctx = ctx; o->raw_code_table = raw_code_table; if (def_pos_args != NULL) { memcpy(o->extra_args, def_pos_args->items, n_def_args * sizeof(mp_obj_t)); diff --git a/py/objfun.h b/py/objfun.h index 516a5bab3905e..0364c7f627a7a 100644 --- a/py/objfun.h +++ b/py/objfun.h @@ -31,7 +31,7 @@ typedef struct _mp_obj_fun_bc_t { mp_obj_base_t base; - const mp_module_context_t *mc; // the context within which this function was defined + const mp_module_context_t *ctx; // the context within which this function was defined struct _mp_raw_code_t *const *raw_code_table; const byte *bytecode; // bytecode for the function #if MICROPY_PY_SYS_SETTRACE diff --git a/py/objgenerator.c b/py/objgenerator.c index 0aaa232375adc..9e268af6d9e60 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -95,7 +95,7 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k uintptr_t prelude_offset = ((uintptr_t *)self_fun->bytecode)[0]; #if MICROPY_EMIT_NATIVE_PRELUDE_AS_BYTES_OBJ // Prelude is in bytes object in const_table, at index prelude_offset - mp_obj_str_t *prelude_bytes = MP_OBJ_TO_PTR(self_fun->mc->const_table[prelude_offset]); + mp_obj_str_t *prelude_bytes = MP_OBJ_TO_PTR(self_fun->ctx->constants.obj_table[prelude_offset]); prelude_offset = (const byte *)prelude_bytes->data - self_fun->bytecode; #endif const uint8_t *ip = self_fun->bytecode + prelude_offset; @@ -184,7 +184,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ // Set up the correct globals context for the generator and execute it self->code_state.old_globals = mp_globals_get(); - mp_globals_set(*self->code_state.fun_bc->mc->globals_ref); + mp_globals_set(self->code_state.fun_bc->ctx->module.globals); mp_vm_return_kind_t ret_kind; diff --git a/py/objmodule.c b/py/objmodule.c index 6b06740e4dabd..98c8488f10bcc 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -112,6 +112,7 @@ const mp_obj_type_t mp_type_module = { .attr = module_attr, }; +#include "py/bc.h" mp_obj_t mp_obj_new_module(qstr module_name) { mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; mp_map_elem_t *el = mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); @@ -122,12 +123,12 @@ mp_obj_t mp_obj_new_module(qstr module_name) { } // create new module object - mp_obj_module_t *o = m_new_obj(mp_obj_module_t); - o->base.type = &mp_type_module; - o->globals = MP_OBJ_TO_PTR(mp_obj_new_dict(MICROPY_MODULE_DICT_SIZE)); + mp_module_context_t *o = m_new_obj(mp_module_context_t); + o->module.base.type = &mp_type_module; + o->module.globals = MP_OBJ_TO_PTR(mp_obj_new_dict(MICROPY_MODULE_DICT_SIZE)); // store __name__ entry in the module - mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(module_name)); + mp_obj_dict_store(MP_OBJ_FROM_PTR(o->module.globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(module_name)); // store the new module into the slot in the global dict holding all modules el->value = MP_OBJ_FROM_PTR(o); diff --git a/py/persistentcode.c b/py/persistentcode.c index f8c53038f91ba..a975a7c8969eb 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -413,7 +413,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { } #include "py/emit.h" // for mp_module_context_new -mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader) { +mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *ctx) { byte header[4]; read_bytes(reader, header, sizeof(header)); if (header[0] != 'M' @@ -431,22 +431,22 @@ mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader) { size_t n_qstr = read_uint(reader, NULL); size_t n_obj = read_uint(reader, NULL); - mp_module_context_t *cm = mp_module_context_new(n_qstr, n_obj); + mp_module_context_new(ctx, n_qstr, n_obj); // Load qstrs. for (size_t i = 0; i < n_qstr; ++i) { - cm->qstr_table[i] = load_qstr(reader); + ctx->constants.qstr_table[i] = load_qstr(reader); } // Load constant objects. for (size_t i = 0; i < n_obj; ++i) { - cm->const_table[i] = load_obj(reader); + ctx->constants.obj_table[i] = load_obj(reader); } // Load top-level module. mp_compiled_module_t cm2; cm2.rc = load_raw_code(reader); - cm2.mc = cm; + cm2.ctx = ctx; cm2.has_native = false; // TODO cm2.n_qstr = n_qstr; cm2.n_obj = n_obj; @@ -456,18 +456,18 @@ mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader) { return cm2; } -mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len) { +mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len, mp_module_context_t *ctx) { mp_reader_t reader; mp_reader_new_mem(&reader, buf, len, 0); - return mp_raw_code_load(&reader); + return mp_raw_code_load(&reader, ctx); } #if MICROPY_HAS_FILE_READER -mp_compiled_module_t mp_raw_code_load_file(const char *filename) { +mp_compiled_module_t mp_raw_code_load_file(const char *filename, mp_module_context_t *ctx) { mp_reader_t reader; mp_reader_new_file(&reader, filename); - return mp_raw_code_load(&reader); + return mp_raw_code_load(&reader, ctx); } #endif // MICROPY_HAS_FILE_READER @@ -620,12 +620,12 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { // Save function argument names (initial entries in const_table) // (viper has n_pos_args=n_kwonly_args=0 so doesn't save any qstrs here) for (size_t i = 0; i < cm->n_qstr; ++i) { - save_qstr(print, cm->mc->qstr_table[i]); + save_qstr(print, cm->ctx->constants.qstr_table[i]); } // Save constant objects. for (size_t i = 0; i < cm->n_obj; ++i) { - save_obj(print, (mp_obj_t)cm->mc->const_table[i]); + save_obj(print, (mp_obj_t)cm->ctx->constants.obj_table[i]); } // Save outer raw code, which will save all its child raw codes. diff --git a/py/persistentcode.h b/py/persistentcode.h index 5a8ff1a841370..8cb4e170d172e 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -102,9 +102,9 @@ enum { MP_NATIVE_ARCH_XTENSAWIN, }; -mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader); -mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len); -mp_compiled_module_t mp_raw_code_load_file(const char *filename); +mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *ctx); +mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len, mp_module_context_t *ctx); +mp_compiled_module_t mp_raw_code_load_file(const char *filename, mp_module_context_t *ctx); void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print); void mp_raw_code_save_file(mp_compiled_module_t *cm, const char *filename); diff --git a/py/runtime.c b/py/runtime.c index 8c93f539e04e6..e1229dcbb2b12 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1508,7 +1508,9 @@ mp_obj_t mp_parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t parse_i if (nlr_push(&nlr) == 0) { qstr source_name = lex->source_name; mp_parse_tree_t parse_tree = mp_parse(lex, parse_input_kind); - mp_obj_t module_fun = mp_compile(&parse_tree, source_name, parse_input_kind == MP_PARSE_SINGLE_INPUT); + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + ctx->module.globals = globals; + mp_obj_t module_fun = mp_compile(&parse_tree, source_name, parse_input_kind == MP_PARSE_SINGLE_INPUT, ctx); mp_obj_t ret; if (MICROPY_PY_BUILTINS_COMPILE && globals == NULL) { diff --git a/py/showbc.c b/py/showbc.c index 360427fd8ec6f..3659e0384efea 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -45,7 +45,7 @@ #define DECODE_QSTR \ DECODE_UINT; \ - qst = QSTR_MAP(mp_showbc_cm, unum) + qst = QSTR_MAP(mp_showbc_constants, unum) #else #define DECODE_QSTR \ @@ -55,19 +55,19 @@ #endif #define DECODE_PTR \ - DECODE_UINT; \ - unum = (mp_uint_t)mp_showbc_cm->const_table[unum] + DECODE_UINT; + #define DECODE_OBJ \ DECODE_UINT; \ - unum = (mp_uint_t)mp_showbc_cm->const_table[unum] + unum = (mp_uint_t)mp_showbc_constants->obj_table[unum] -#define QSTR_MAP(table, i) (table->qstr_table[(i)]) +#define QSTR_MAP(constants, i) (constants->qstr_table[(i)]) const byte * mp_showbc_code_start; -const mp_module_context_t *mp_showbc_cm; +const mp_module_constants_t *mp_showbc_constants; #include "py/emitglue.h" -void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *ip, mp_uint_t len, const mp_module_context_t *cm) { +void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *ip, mp_uint_t len, const mp_module_constants_t *cm) { mp_showbc_code_start = ip; // Decode prelude @@ -527,9 +527,9 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { return ip; } -void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, const mp_module_context_t *cm) { +void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, const mp_module_constants_t *cm) { mp_showbc_code_start = ip; - mp_showbc_cm = cm; + mp_showbc_constants = cm; while (ip < len + mp_showbc_code_start) { mp_printf(print, "%02u ", (uint)(ip - mp_showbc_code_start)); ip = mp_bytecode_print_str(print, ip); diff --git a/py/vm.c b/py/vm.c index 087ca0b8cfef5..e448671f73528 100644 --- a/py/vm.c +++ b/py/vm.c @@ -82,7 +82,7 @@ #define DECODE_QSTR \ DECODE_UINT; \ - qstr qst = code_state->fun_bc->mc->qstr_table[unum] + qstr qst = code_state->fun_bc->ctx->constants.qstr_table[unum] #else #define DECODE_QSTR \ @@ -96,7 +96,7 @@ void *ptr = (void *)(uintptr_t)code_state->fun_bc->raw_code_table[unum] #define DECODE_OBJ \ DECODE_UINT; \ - mp_obj_t obj = (mp_obj_t)code_state->fun_bc->mc->const_table[unum] + mp_obj_t obj = (mp_obj_t)code_state->fun_bc->ctx->constants.obj_table[unum] #define PUSH(val) *++sp = (val) #define POP() (*sp--) @@ -865,7 +865,7 @@ unwind_jump:; ENTRY(MP_BC_MAKE_FUNCTION): { DECODE_PTR; - PUSH(mp_make_function_from_raw_code(ptr, code_state->fun_bc->mc, NULL)); + PUSH(mp_make_function_from_raw_code(ptr, code_state->fun_bc->ctx, NULL)); DISPATCH(); } @@ -873,7 +873,7 @@ unwind_jump:; DECODE_PTR; // Stack layout: def_tuple def_dict <- TOS sp -= 1; - SET_TOP(mp_make_function_from_raw_code(ptr, code_state->fun_bc->mc, sp)); + SET_TOP(mp_make_function_from_raw_code(ptr, code_state->fun_bc->ctx, sp)); DISPATCH(); } @@ -882,7 +882,7 @@ unwind_jump:; size_t n_closed_over = *ip++; // Stack layout: closed_overs <- TOS sp -= n_closed_over - 1; - SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->mc, n_closed_over, sp)); + SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->ctx, n_closed_over, sp)); DISPATCH(); } @@ -891,7 +891,7 @@ unwind_jump:; size_t n_closed_over = *ip++; // Stack layout: def_tuple def_dict closed_overs <- TOS sp -= 2 + n_closed_over - 1; - SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->mc, 0x100 | n_closed_over, sp)); + SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->ctx, 0x100 | n_closed_over, sp)); DISPATCH(); } @@ -1399,9 +1399,9 @@ unwind_jump:; } #if MICROPY_PERSISTENT_CODE // TODO use DECODE_QSTR? - block_name = code_state->fun_bc->mc->qstr_table[block_name]; + block_name = code_state->fun_bc->ctx->constants.qstr_table[block_name]; #endif - qstr source_file = code_state->fun_bc->mc->qstr_table[0]; + qstr source_file = code_state->fun_bc->ctx->constants.qstr_table[0]; size_t source_line = mp_bytecode_get_source_line(ip, line_info_top, bc); mp_obj_exception_add_traceback(MP_OBJ_FROM_PTR(nlr.ret_val), source_file, source_line, block_name); } diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 70b5c20026775..f4a51385e395d 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -83,9 +83,11 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input #if MICROPY_MODULE_FROZEN_MPY if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) { // source is a raw_code object, create the function - const mp_compiled_module_t *cm = source; - module_fun = mp_make_function_from_raw_code(cm->rc, cm->mc, NULL); - *cm->mc->globals_ref = mp_globals_get(); + const mp_frozen_module_t *frozen = source; + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + ctx->module.globals = mp_globals_get(); + ctx->constants = frozen->constants; + module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL); } else #endif { @@ -104,7 +106,9 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input // source is a lexer, parse and compile the script qstr source_name = lex->source_name; mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); - module_fun = mp_compile(&parse_tree, source_name, exec_flags & EXEC_FLAG_IS_REPL); + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + ctx->module.globals = mp_globals_get(); + module_fun = mp_compile(&parse_tree, source_name, exec_flags & EXEC_FLAG_IS_REPL, ctx); #else mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("script compilation not supported")); #endif diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 6a8325a909606..8816388cddb20 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -514,21 +514,19 @@ def freeze(self, compiled_module_index): self.freeze_constants() - print("static const mp_module_context_t module_context_%s = {" % self.escaped_name) - print(" .globals_ref = (mp_obj_dict_t **)&mp_frozen_root_pointers[%u]," % compiled_module_index) + print() + print("static const mp_frozen_module_t frozen_module_%s = {" % self.escaped_name) + print(" .constants = {") + if len(self.qstr_table): + print(" .qstr_table = (qstr_short_t *)&const_qstr_table_data_%s," % self.escaped_name) + else: + print(" .qstr_table = NULL,") if len(self.obj_table): - print(" .const_table = (mp_obj_t *)&const_table_data_%s," % self.escaped_name) + print(" .obj_table = (mp_obj_t *)&const_obj_table_data_%s," % self.escaped_name) else: - print(" .const_table = NULL,") - print(" .qstr_table = {") - for q in self.qstr_table: - print(" %s," % q.qstr_id) + print(" .obj_table = NULL,") print(" },") - print("};") - print() - print("static const mp_compiled_module_t compiled_module_%s = {" % self.escaped_name) print(" .rc = &raw_code_%s," % self.raw_code.escaped_name) - print(" .mc = (void *)&module_context_%s," % self.escaped_name) print(" #if MICROPY_PERSISTENT_CODE_SAVE") print(" .has_native = false,") print(" .n_qstr = %u," % len(self.qstr_table)) @@ -538,6 +536,16 @@ def freeze(self, compiled_module_index): def freeze_constants(self): global const_str_content, const_int_content, const_obj_content + + if len(self.qstr_table): + print( + "static const qstr_short_t const_qstr_table_data_%s[%u] = {" + % (self.escaped_name, len(self.qstr_table)) + ) + for q in self.qstr_table: + print(" %s," % q.qstr_id) + print("};") + if not len(self.obj_table): return @@ -617,7 +625,7 @@ def freeze_constants(self): print() print("// constant table") print( - "static const mp_rom_obj_t const_table_data_%s[%u] = {" + "static const mp_rom_obj_t const_obj_table_data_%s[%u] = {" % (self.escaped_name, len(self.obj_table)) ) for i in range(len(self.obj_table)): @@ -1341,10 +1349,12 @@ def freeze_mpy(base_qstrs, compiled_modules): print(" },") print("};") + """ # Define the array of root pointers, to hold pointers to global dict for each module. print() print("void *mp_frozen_root_pointers[%u];" % len(compiled_modules)) print("const size_t mp_frozen_root_pointers_len = %u;" % len(compiled_modules)) + """ # Freeze all modules. for idx, cm in enumerate(compiled_modules): @@ -1372,9 +1382,9 @@ def freeze_mpy(base_qstrs, compiled_modules): # Define the array of pointers to frozen module content. print() - print("const mp_compiled_module_t *const mp_frozen_mpy_content[] = {") + print("const mp_frozen_module_t *const mp_frozen_mpy_content[] = {") for cm in compiled_modules: - print(" &compiled_module_%s," % cm.escaped_name) + print(" &frozen_module_%s," % cm.escaped_name) print("};") mp_frozen_mpy_content_size = len(compiled_modules * 4)