py: Integrate extern native loading into import system.

Can now "import" native module as you would a normal .py module: >>> import modx >>> modx.data >>> modx.add1(1) where modx.mpy (compiled from extmod/modx) is in your path.
micropython · Jan 24, 2015 · 1f28413 · 1f28413 · danicampora · Feb 4, 2015
1 parent db95338
commit 1f28413
Show file tree

Hide file tree

Showing 12 changed files with 64 additions and 40 deletions.
diff --git a/extmod/modx/modx.c b/extmod/modx/modx.c
@@ -7,7 +7,9 @@ STATIC mp_obj_t modx_add1(const mp_ext_table_t *et, mp_obj_t x) {
 MP_EXT_HEADER
 
 MP_EXT_INIT
-mp_obj_t init(const mp_ext_table_t *et) {
-    mp_obj_t list[6] = {MP_OBJ_NEW_SMALL_INT(1), MP_OBJ_NEW_SMALL_INT(2), MP_OBJ_NEW_SMALL_INT(3), et->mp_const_true_, MP_OBJ_NEW_QSTR(et->qstr_from_str("modx")), et->mp_obj_new_fun_extern(false, 1, 1, modx_add1)};
-    return et->mp_obj_new_list(6, list);
+void init(const mp_ext_table_t *et) {
+    mp_obj_t f_add1 = et->mp_obj_new_fun_extern(false, 1, 1, modx_add1);
+    mp_obj_t list[6] = {MP_OBJ_NEW_SMALL_INT(1), MP_OBJ_NEW_SMALL_INT(2), MP_OBJ_NEW_SMALL_INT(3), et->mp_const_true_, MP_OBJ_NEW_QSTR(et->qstr_from_str("modx")), f_add1};
+    et->mp_store_global(et->qstr_from_str("data"), et->mp_obj_new_list(6, list));
+    et->mp_store_global(et->qstr_from_str("add1"), f_add1);
 }
diff --git a/extmod/modx/modx.py b/extmod/modx/modx.py
diff --git a/extmod/modx/mpconfigport.h b/extmod/modx/mpconfigport.h
@@ -42,12 +42,12 @@
 #define MICROPY_OPT_COMPUTED_GOTO   (1)
 #define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (1)
 #define MICROPY_CAN_OVERRIDE_BUILTINS (1)
+#define MICROPY_MODULE_EXTERN       (1)
 #define MICROPY_PY_BUILTINS_STR_UNICODE (1)
 #define MICROPY_PY_BUILTINS_MEMORYVIEW (1)
 #define MICROPY_PY_BUILTINS_FROZENSET (1)
 #define MICROPY_PY_BUILTINS_COMPILE (1)
 #define MICROPY_PY_MICROPYTHON_MEM_INFO (1)
-#define MICROPY_PY_MICROPYTHON_LOAD (1)
 #define MICROPY_PY_SYS_EXIT         (1)
 #define MICROPY_PY_SYS_PLATFORM     "linux"
 #define MICROPY_PY_SYS_MAXSIZE      (1)

diff --git a/py/builtinimport.c b/py/builtinimport.c
@@ -35,6 +35,7 @@
 #include "py/runtime.h"
 #include "py/builtin.h"
 #include "py/frozenmod.h"
+#include "py/mpextern.h"
 
 #if 0 // print debugging info
 #define DEBUG_PRINT (1)
@@ -71,6 +72,12 @@ STATIC mp_import_stat_t stat_dir_or_file(vstr_t *path) {
     if (stat == MP_IMPORT_STAT_FILE) {
         return stat;
     }
+    vstr_cut_tail_bytes(path, 2);
+    vstr_add_str(path, "mpy");
+    stat = mp_import_stat(vstr_str(path));
+    if (stat == MP_IMPORT_STAT_FILE) {
+        return stat;
+    }
     return MP_IMPORT_STAT_NO_EXIST;
 }
 
@@ -138,6 +145,18 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
     do_load_from_lexer(module_obj, lex, vstr_str(file));
 }
 
+#if MICROPY_MODULE_EXTERN
+STATIC void do_load_extern(mp_obj_t module_obj, vstr_t *file) {
+    #if MICROPY_PY___FILE__
+    mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(vstr_str(file))));
+    #endif
+
+    // load the extern module in its context
+    mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj);
+    mp_extern_load(vstr_str(file), mod_globals);
+}
+#endif
+
 mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) {
 #if DEBUG_PRINT
     DEBUG_printf("__import__:\n");
@@ -334,7 +353,14 @@ mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) {
                         vstr_cut_tail_bytes(&path, sizeof("/__init__.py") - 1); // cut off /__init__.py
                     }
                 } else { // MP_IMPORT_STAT_FILE
-                    do_load(module_obj, &path);
+                    #if MICROPY_MODULE_EXTERN
+                    if (path.buf[path.len - 3] == 'm') {
+                        do_load_extern(module_obj, &path);
+                    } else
+                    #endif
+                    {
+                        do_load(module_obj, &path);
+                    }
                     // TODO: We cannot just break here, at the very least, we must execute
                     // trailer code below. But otherwise if there're remaining components,
                     // that would be (??) object path within module, not modules path within FS.

diff --git a/py/modmicropython.c b/py/modmicropython.c
@@ -107,9 +107,6 @@ STATIC const mp_map_elem_t mp_module_micropython_globals_table[] = {
 #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0)
     { MP_OBJ_NEW_QSTR(MP_QSTR_alloc_emergency_exception_buf), (mp_obj_t)&mp_alloc_emergency_exception_buf_obj },
 #endif
-#if MICROPY_PY_MICROPYTHON_LOAD
-    { MP_OBJ_NEW_QSTR(MP_QSTR_load), (mp_obj_t)&mp_extern_load_obj },
-#endif
 };
 
 STATIC MP_DEFINE_CONST_DICT(mp_module_micropython_globals, mp_module_micropython_globals_table);

diff --git a/py/mpconfig.h b/py/mpconfig.h
@@ -333,6 +333,11 @@ typedef double mp_float_t;
 #define MICROPY_MODULE_FROZEN (0)
 #endif
 
+// Whether to support loading external native modules
+#ifndef MICROPY_MODULE_EXTERN
+#define MICROPY_MODULE_EXTERN (0)
+#endif
+
 // Whether you can override builtins in the builtins module
 #ifndef MICROPY_CAN_OVERRIDE_BUILTINS
 #define MICROPY_CAN_OVERRIDE_BUILTINS (0)
@@ -396,11 +401,6 @@ typedef double mp_float_t;
 #define MICROPY_PY_MICROPYTHON_MEM_INFO (0)
 #endif
 
-// Whether to support loading external native images
-#ifndef MICROPY_PY_MICROPYTHON_LOAD
-#define MICROPY_PY_MICROPYTHON_LOAD (0)
-#endif
-
 // Whether to provide "array" module. Note that large chunk of the
 // underlying code is shared with "bytearray" builtin type, so to
 // get real savings, it should be disabled too.

diff --git a/py/mpextern.c b/py/mpextern.c
@@ -1,6 +1,6 @@
 #include "mpextern.h"
 
-#if MICROPY_PY_MICROPYTHON_LOAD
+#if MICROPY_MODULE_EXTERN
 
 typedef mp_obj_t (*mp_fun_ext_0_t)(const mp_ext_table_t *et);
 typedef mp_obj_t (*mp_fun_ext_1_t)(const mp_ext_table_t *et, mp_obj_t);
@@ -91,8 +91,8 @@ STATIC const mp_ext_table_t mp_ext_table = {
     .mp_binary_op = mp_binary_op,
 };
 
-STATIC mp_obj_t mp_extern_load(mp_obj_t ext_name) {
-    const byte *buf = mp_extern_load_binary(mp_obj_str_get_str(ext_name));
+void mp_extern_load(const char *ext_name, mp_obj_dict_t *globals) {
+    const byte *buf = mp_extern_load_binary(ext_name);
 
     if (buf == NULL) {
         nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_OSError, "could not load MPY binary"));
@@ -106,25 +106,23 @@ STATIC mp_obj_t mp_extern_load(mp_obj_t ext_name) {
         nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "MPY binary has wrong version"));
     }
 
-    // create new globals dict
-    mp_obj_dict_t *gl = mp_obj_new_dict(0);
+    if (buf[6] != MP_EXT_ARCH_CURRENT) {
+        nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "MPY binary has wrong arch"));
+    }
 
     // push context
     mp_obj_dict_t *old_locals = mp_locals_get();
     mp_obj_dict_t *old_globals = mp_globals_get();
-    mp_locals_set(gl);
-    mp_globals_set(gl);
+    mp_locals_set(globals);
+    mp_globals_set(globals);
 
     // call extern init
     mp_obj_t (*f)(const mp_ext_table_t*) = (mp_obj_t(*)(const mp_ext_table_t*))MICROPY_MAKE_POINTER_CALLABLE(buf + 8);
-    mp_obj_t ret = f(&mp_ext_table);
+    f(&mp_ext_table);
 
     // pop context
     mp_globals_set(old_globals);
     mp_locals_set(old_locals);
-
-    return ret;
 }
-MP_DEFINE_CONST_FUN_OBJ_1(mp_extern_load_obj, mp_extern_load);
 
-#endif // MICROPY_PY_MICROPYTHON_LOAD
+#endif // MICROPY_MODULE_EXTERN
diff --git a/py/mpextern.h b/py/mpextern.h
@@ -32,13 +32,20 @@
 #define MP_EXT_VERSION_MAJOR (0)
 #define MP_EXT_VERSION_MINOR (0)
 #define MP_EXT_VERSION_SUBMINOR (1)
+#define MP_EXT_ARCH_X86    (1)
+#define MP_EXT_ARCH_X64    (2)
+#define MP_EXT_ARCH_ARM    (3)
+#define MP_EXT_ARCH_THUMB2 (4)
+
+// TODO auto-detect current arch
+#define MP_EXT_ARCH_CURRENT (MP_EXT_ARCH_X86)
 
 #define MP_EXT_HEADER \
     __attribute__((section(".mpyheader"))) \
     const byte header[8] = { \
         'M', 'P', 'Y', \
         MP_EXT_VERSION_MAJOR, MP_EXT_VERSION_MINOR, MP_EXT_VERSION_SUBMINOR, \
-        0, 0, \
+        MP_EXT_ARCH_CURRENT, 0, \
     };
 
 #define MP_EXT_INIT \
@@ -55,12 +62,12 @@ typedef struct _mp_ext_table_t {
     mp_obj_t (*mp_binary_op)(mp_uint_t op, mp_obj_t lhs, mp_obj_t rhs);
 } mp_ext_table_t;
 
-MP_DECLARE_CONST_FUN_OBJ(mp_extern_load_obj);
+void mp_extern_load(const char *ext_name, mp_obj_dict_t *globals);
 
 // to be implemented per-port
 const byte *mp_extern_load_binary(const char *ext_name);
 
 // entry point for the extern binary
-mp_obj_t init(const mp_ext_table_t *et);
+void init(const mp_ext_table_t *et);
 
 #endif // __MICROPY_INCLUDED_PY_MPEXTERN_H__
diff --git a/py/qstrdefs.h b/py/qstrdefs.h
@@ -378,10 +378,6 @@ Q(mem_info)
 Q(qstr_info)
 #endif
 
-#if MICROPY_PY_MICROPYTHON_LOAD
-Q(load)
-#endif
-
 #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0)
 Q(alloc_emergency_exception_buf)
 #endif

diff --git a/stmhal/mpconfigport.h b/stmhal/mpconfigport.h
@@ -53,13 +53,13 @@
 #define MICROPY_LFN_CODE_PAGE       (437) /* 1=SFN/ANSI 437=LFN/U.S.(OEM) */
 #define MICROPY_STREAMS_NON_BLOCK   (1)
 #define MICROPY_MODULE_WEAK_LINKS   (1)
+#define MICROPY_MODULE_EXTERN       (1)
 #define MICROPY_CAN_OVERRIDE_BUILTINS (1)
 #define MICROPY_PY_BUILTINS_STR_UNICODE (1)
 #define MICROPY_PY_BUILTINS_MEMORYVIEW (1)
 #define MICROPY_PY_BUILTINS_FROZENSET (1)
 #define MICROPY_PY_BUILTINS_EXECFILE (1)
 #define MICROPY_PY_MICROPYTHON_MEM_INFO (1)
-#define MICROPY_PY_MICROPYTHON_LOAD (1)
 #define MICROPY_PY_SYS_EXIT         (1)
 #define MICROPY_PY_SYS_STDFILES     (1)
 #define MICROPY_PY_CMATH            (1)

diff --git a/unix/main.c b/unix/main.c
@@ -538,10 +538,12 @@ const byte *mp_extern_load_binary(const char *ext_name) {
     }
     off_t len = lseek(fd, 0, SEEK_END);
     lseek(fd, 0, SEEK_SET);
-    byte *buf = m_new(byte, len);
+    byte *buf;
+    mp_uint_t actual_size;
+    MP_PLAT_ALLOC_EXEC(len, (void**)&buf, &actual_size);
     ssize_t n = read(fd, buf, len);
     if (n != len) {
-        m_del(byte, buf, len);
+        MP_PLAT_FREE_EXEC(buf, actual_size);
         return NULL;
     }
     close(fd);

diff --git a/unix/mpconfigport.h b/unix/mpconfigport.h
@@ -53,13 +53,13 @@
 #define MICROPY_STREAMS_NON_BLOCK   (1)
 #define MICROPY_OPT_COMPUTED_GOTO   (1)
 #define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (1)
+#define MICROPY_MODULE_EXTERN       (1)
 #define MICROPY_CAN_OVERRIDE_BUILTINS (1)
 #define MICROPY_PY_BUILTINS_STR_UNICODE (1)
 #define MICROPY_PY_BUILTINS_MEMORYVIEW (1)
 #define MICROPY_PY_BUILTINS_FROZENSET (1)
 #define MICROPY_PY_BUILTINS_COMPILE (1)
 #define MICROPY_PY_MICROPYTHON_MEM_INFO (1)
-#define MICROPY_PY_MICROPYTHON_LOAD (1)
 #define MICROPY_PY_SYS_EXIT         (1)
 #define MICROPY_PY_SYS_PLATFORM     "linux"
 #define MICROPY_PY_SYS_MAXSIZE      (1)