From 1f28413e840b6f688dbc9deda93ff4e33984019e Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 24 Jan 2015 16:08:51 +0000 Subject: [PATCH] py: Integrate extern native loading into import system. Can now "import" native module as you would a normal .py module: >>> import modx >>> modx.data >>> modx.add1(1) where modx.mpy (compiled from extmod/modx) is in your path. --- extmod/modx/modx.c | 8 +++++--- extmod/modx/modx.py | 4 ---- extmod/modx/mpconfigport.h | 2 +- py/builtinimport.c | 28 +++++++++++++++++++++++++++- py/modmicropython.c | 3 --- py/mpconfig.h | 10 +++++----- py/mpextern.c | 22 ++++++++++------------ py/mpextern.h | 13 ++++++++++--- py/qstrdefs.h | 4 ---- stmhal/mpconfigport.h | 2 +- unix/main.c | 6 ++++-- unix/mpconfigport.h | 2 +- 12 files changed, 64 insertions(+), 40 deletions(-) delete mode 100644 extmod/modx/modx.py diff --git a/extmod/modx/modx.c b/extmod/modx/modx.c index 674cd9d3bb5d..1d710e405e7a 100644 --- a/extmod/modx/modx.c +++ b/extmod/modx/modx.c @@ -7,7 +7,9 @@ STATIC mp_obj_t modx_add1(const mp_ext_table_t *et, mp_obj_t x) { MP_EXT_HEADER MP_EXT_INIT -mp_obj_t init(const mp_ext_table_t *et) { - mp_obj_t list[6] = {MP_OBJ_NEW_SMALL_INT(1), MP_OBJ_NEW_SMALL_INT(2), MP_OBJ_NEW_SMALL_INT(3), et->mp_const_true_, MP_OBJ_NEW_QSTR(et->qstr_from_str("modx")), et->mp_obj_new_fun_extern(false, 1, 1, modx_add1)}; - return et->mp_obj_new_list(6, list); +void init(const mp_ext_table_t *et) { + mp_obj_t f_add1 = et->mp_obj_new_fun_extern(false, 1, 1, modx_add1); + mp_obj_t list[6] = {MP_OBJ_NEW_SMALL_INT(1), MP_OBJ_NEW_SMALL_INT(2), MP_OBJ_NEW_SMALL_INT(3), et->mp_const_true_, MP_OBJ_NEW_QSTR(et->qstr_from_str("modx")), f_add1}; + et->mp_store_global(et->qstr_from_str("data"), et->mp_obj_new_list(6, list)); + et->mp_store_global(et->qstr_from_str("add1"), f_add1); } diff --git a/extmod/modx/modx.py b/extmod/modx/modx.py deleted file mode 100644 index ddde4579d5f8..000000000000 --- a/extmod/modx/modx.py +++ /dev/null @@ -1,4 +0,0 @@ -import micropython -ret = micropython.load('modx.mpy') -print(ret) -print(ret[5](42)) diff --git a/extmod/modx/mpconfigport.h b/extmod/modx/mpconfigport.h index 3bf94ea8d4bc..4a179331fe19 100644 --- a/extmod/modx/mpconfigport.h +++ b/extmod/modx/mpconfigport.h @@ -42,12 +42,12 @@ #define MICROPY_OPT_COMPUTED_GOTO (1) #define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (1) #define MICROPY_CAN_OVERRIDE_BUILTINS (1) +#define MICROPY_MODULE_EXTERN (1) #define MICROPY_PY_BUILTINS_STR_UNICODE (1) #define MICROPY_PY_BUILTINS_MEMORYVIEW (1) #define MICROPY_PY_BUILTINS_FROZENSET (1) #define MICROPY_PY_BUILTINS_COMPILE (1) #define MICROPY_PY_MICROPYTHON_MEM_INFO (1) -#define MICROPY_PY_MICROPYTHON_LOAD (1) #define MICROPY_PY_SYS_EXIT (1) #define MICROPY_PY_SYS_PLATFORM "linux" #define MICROPY_PY_SYS_MAXSIZE (1) diff --git a/py/builtinimport.c b/py/builtinimport.c index 73bf0b4722a1..17f14e0d7524 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -35,6 +35,7 @@ #include "py/runtime.h" #include "py/builtin.h" #include "py/frozenmod.h" +#include "py/mpextern.h" #if 0 // print debugging info #define DEBUG_PRINT (1) @@ -71,6 +72,12 @@ STATIC mp_import_stat_t stat_dir_or_file(vstr_t *path) { if (stat == MP_IMPORT_STAT_FILE) { return stat; } + vstr_cut_tail_bytes(path, 2); + vstr_add_str(path, "mpy"); + stat = mp_import_stat(vstr_str(path)); + if (stat == MP_IMPORT_STAT_FILE) { + return stat; + } return MP_IMPORT_STAT_NO_EXIST; } @@ -138,6 +145,18 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { do_load_from_lexer(module_obj, lex, vstr_str(file)); } +#if MICROPY_MODULE_EXTERN +STATIC void do_load_extern(mp_obj_t module_obj, vstr_t *file) { + #if MICROPY_PY___FILE__ + mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(vstr_str(file)))); + #endif + + // load the extern module in its context + mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj); + mp_extern_load(vstr_str(file), mod_globals); +} +#endif + mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) { #if DEBUG_PRINT DEBUG_printf("__import__:\n"); @@ -334,7 +353,14 @@ mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) { vstr_cut_tail_bytes(&path, sizeof("/__init__.py") - 1); // cut off /__init__.py } } else { // MP_IMPORT_STAT_FILE - do_load(module_obj, &path); + #if MICROPY_MODULE_EXTERN + if (path.buf[path.len - 3] == 'm') { + do_load_extern(module_obj, &path); + } else + #endif + { + do_load(module_obj, &path); + } // TODO: We cannot just break here, at the very least, we must execute // trailer code below. But otherwise if there're remaining components, // that would be (??) object path within module, not modules path within FS. diff --git a/py/modmicropython.c b/py/modmicropython.c index 15d8f40e5682..2663de28bc6e 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -107,9 +107,6 @@ STATIC const mp_map_elem_t mp_module_micropython_globals_table[] = { #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0) { MP_OBJ_NEW_QSTR(MP_QSTR_alloc_emergency_exception_buf), (mp_obj_t)&mp_alloc_emergency_exception_buf_obj }, #endif -#if MICROPY_PY_MICROPYTHON_LOAD - { MP_OBJ_NEW_QSTR(MP_QSTR_load), (mp_obj_t)&mp_extern_load_obj }, -#endif }; STATIC MP_DEFINE_CONST_DICT(mp_module_micropython_globals, mp_module_micropython_globals_table); diff --git a/py/mpconfig.h b/py/mpconfig.h index b82f672f94ec..81036259e4e8 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -333,6 +333,11 @@ typedef double mp_float_t; #define MICROPY_MODULE_FROZEN (0) #endif +// Whether to support loading external native modules +#ifndef MICROPY_MODULE_EXTERN +#define MICROPY_MODULE_EXTERN (0) +#endif + // Whether you can override builtins in the builtins module #ifndef MICROPY_CAN_OVERRIDE_BUILTINS #define MICROPY_CAN_OVERRIDE_BUILTINS (0) @@ -396,11 +401,6 @@ typedef double mp_float_t; #define MICROPY_PY_MICROPYTHON_MEM_INFO (0) #endif -// Whether to support loading external native images -#ifndef MICROPY_PY_MICROPYTHON_LOAD -#define MICROPY_PY_MICROPYTHON_LOAD (0) -#endif - // Whether to provide "array" module. Note that large chunk of the // underlying code is shared with "bytearray" builtin type, so to // get real savings, it should be disabled too. diff --git a/py/mpextern.c b/py/mpextern.c index ce8479922975..d3e203e1911a 100644 --- a/py/mpextern.c +++ b/py/mpextern.c @@ -1,6 +1,6 @@ #include "mpextern.h" -#if MICROPY_PY_MICROPYTHON_LOAD +#if MICROPY_MODULE_EXTERN typedef mp_obj_t (*mp_fun_ext_0_t)(const mp_ext_table_t *et); typedef mp_obj_t (*mp_fun_ext_1_t)(const mp_ext_table_t *et, mp_obj_t); @@ -91,8 +91,8 @@ STATIC const mp_ext_table_t mp_ext_table = { .mp_binary_op = mp_binary_op, }; -STATIC mp_obj_t mp_extern_load(mp_obj_t ext_name) { - const byte *buf = mp_extern_load_binary(mp_obj_str_get_str(ext_name)); +void mp_extern_load(const char *ext_name, mp_obj_dict_t *globals) { + const byte *buf = mp_extern_load_binary(ext_name); if (buf == NULL) { nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_OSError, "could not load MPY binary")); @@ -106,25 +106,23 @@ STATIC mp_obj_t mp_extern_load(mp_obj_t ext_name) { nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "MPY binary has wrong version")); } - // create new globals dict - mp_obj_dict_t *gl = mp_obj_new_dict(0); + if (buf[6] != MP_EXT_ARCH_CURRENT) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "MPY binary has wrong arch")); + } // push context mp_obj_dict_t *old_locals = mp_locals_get(); mp_obj_dict_t *old_globals = mp_globals_get(); - mp_locals_set(gl); - mp_globals_set(gl); + mp_locals_set(globals); + mp_globals_set(globals); // call extern init mp_obj_t (*f)(const mp_ext_table_t*) = (mp_obj_t(*)(const mp_ext_table_t*))MICROPY_MAKE_POINTER_CALLABLE(buf + 8); - mp_obj_t ret = f(&mp_ext_table); + f(&mp_ext_table); // pop context mp_globals_set(old_globals); mp_locals_set(old_locals); - - return ret; } -MP_DEFINE_CONST_FUN_OBJ_1(mp_extern_load_obj, mp_extern_load); -#endif // MICROPY_PY_MICROPYTHON_LOAD +#endif // MICROPY_MODULE_EXTERN diff --git a/py/mpextern.h b/py/mpextern.h index 5ad03ff932d0..c3bc27f56540 100644 --- a/py/mpextern.h +++ b/py/mpextern.h @@ -32,13 +32,20 @@ #define MP_EXT_VERSION_MAJOR (0) #define MP_EXT_VERSION_MINOR (0) #define MP_EXT_VERSION_SUBMINOR (1) +#define MP_EXT_ARCH_X86 (1) +#define MP_EXT_ARCH_X64 (2) +#define MP_EXT_ARCH_ARM (3) +#define MP_EXT_ARCH_THUMB2 (4) + +// TODO auto-detect current arch +#define MP_EXT_ARCH_CURRENT (MP_EXT_ARCH_X86) #define MP_EXT_HEADER \ __attribute__((section(".mpyheader"))) \ const byte header[8] = { \ 'M', 'P', 'Y', \ MP_EXT_VERSION_MAJOR, MP_EXT_VERSION_MINOR, MP_EXT_VERSION_SUBMINOR, \ - 0, 0, \ + MP_EXT_ARCH_CURRENT, 0, \ }; #define MP_EXT_INIT \ @@ -55,12 +62,12 @@ typedef struct _mp_ext_table_t { mp_obj_t (*mp_binary_op)(mp_uint_t op, mp_obj_t lhs, mp_obj_t rhs); } mp_ext_table_t; -MP_DECLARE_CONST_FUN_OBJ(mp_extern_load_obj); +void mp_extern_load(const char *ext_name, mp_obj_dict_t *globals); // to be implemented per-port const byte *mp_extern_load_binary(const char *ext_name); // entry point for the extern binary -mp_obj_t init(const mp_ext_table_t *et); +void init(const mp_ext_table_t *et); #endif // __MICROPY_INCLUDED_PY_MPEXTERN_H__ diff --git a/py/qstrdefs.h b/py/qstrdefs.h index eaf382378f23..23c25072d943 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -378,10 +378,6 @@ Q(mem_info) Q(qstr_info) #endif -#if MICROPY_PY_MICROPYTHON_LOAD -Q(load) -#endif - #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0) Q(alloc_emergency_exception_buf) #endif diff --git a/stmhal/mpconfigport.h b/stmhal/mpconfigport.h index 74c0668fda69..a4438767c628 100644 --- a/stmhal/mpconfigport.h +++ b/stmhal/mpconfigport.h @@ -53,13 +53,13 @@ #define MICROPY_LFN_CODE_PAGE (437) /* 1=SFN/ANSI 437=LFN/U.S.(OEM) */ #define MICROPY_STREAMS_NON_BLOCK (1) #define MICROPY_MODULE_WEAK_LINKS (1) +#define MICROPY_MODULE_EXTERN (1) #define MICROPY_CAN_OVERRIDE_BUILTINS (1) #define MICROPY_PY_BUILTINS_STR_UNICODE (1) #define MICROPY_PY_BUILTINS_MEMORYVIEW (1) #define MICROPY_PY_BUILTINS_FROZENSET (1) #define MICROPY_PY_BUILTINS_EXECFILE (1) #define MICROPY_PY_MICROPYTHON_MEM_INFO (1) -#define MICROPY_PY_MICROPYTHON_LOAD (1) #define MICROPY_PY_SYS_EXIT (1) #define MICROPY_PY_SYS_STDFILES (1) #define MICROPY_PY_CMATH (1) diff --git a/unix/main.c b/unix/main.c index 2c1ccd58d60f..b731d29d2889 100644 --- a/unix/main.c +++ b/unix/main.c @@ -538,10 +538,12 @@ const byte *mp_extern_load_binary(const char *ext_name) { } off_t len = lseek(fd, 0, SEEK_END); lseek(fd, 0, SEEK_SET); - byte *buf = m_new(byte, len); + byte *buf; + mp_uint_t actual_size; + MP_PLAT_ALLOC_EXEC(len, (void**)&buf, &actual_size); ssize_t n = read(fd, buf, len); if (n != len) { - m_del(byte, buf, len); + MP_PLAT_FREE_EXEC(buf, actual_size); return NULL; } close(fd); diff --git a/unix/mpconfigport.h b/unix/mpconfigport.h index 5ebc74596052..c28d33b2259f 100644 --- a/unix/mpconfigport.h +++ b/unix/mpconfigport.h @@ -53,13 +53,13 @@ #define MICROPY_STREAMS_NON_BLOCK (1) #define MICROPY_OPT_COMPUTED_GOTO (1) #define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (1) +#define MICROPY_MODULE_EXTERN (1) #define MICROPY_CAN_OVERRIDE_BUILTINS (1) #define MICROPY_PY_BUILTINS_STR_UNICODE (1) #define MICROPY_PY_BUILTINS_MEMORYVIEW (1) #define MICROPY_PY_BUILTINS_FROZENSET (1) #define MICROPY_PY_BUILTINS_COMPILE (1) #define MICROPY_PY_MICROPYTHON_MEM_INFO (1) -#define MICROPY_PY_MICROPYTHON_LOAD (1) #define MICROPY_PY_SYS_EXIT (1) #define MICROPY_PY_SYS_PLATFORM "linux" #define MICROPY_PY_SYS_MAXSIZE (1)