tools/mpy_ld.py: Add new mpy_ld.py tool and associated build files.

This commit adds a new tool called mpy_ld.py which is essentially a linker
that builds .mpy files directly from .o files.  A new header file
(dynruntime.h) and makefile fragment (dynruntime.mk) are also included
which allow building .mpy files from C source code.  Such .mpy files can
then be dynamically imported as though they were a normal Python module,
even though they are implemented in C.

Converting .o files directly (rather than pre-linked .elf files) allows the
resulting .mpy to be more efficient because it has more control over the
relocations; for example it can skip PLT indirection.  Doing it this way
also allows supporting more architectures, such as Xtensa which has
specific needs for position-independent code and the GOT.

The tool supports targets of x86, x86-64, ARM Thumb and Xtensa (windowed
and non-windowed).  BSS, text and rodata sections are supported, with
relocations to all internal sections and symbols, as well as relocations to
some external symbols (defined by dynruntime.h), and linking of qstrs.

py/dynruntime.h

@@ -0,0 +1,193 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_PY_DYNRUNTIME_H
+#define MICROPY_INCLUDED_PY_DYNRUNTIME_H
+
+// This header file contains definitions to dynamically implement the static
+// MicroPython runtime API defined in py/obj.h and py/runtime.h.
+
+#include "py/nativeglue.h"
+#include "py/objstr.h"
+
+#undef MP_ROM_QSTR
+#undef MP_OBJ_QSTR_VALUE
+#undef MP_OBJ_NEW_QSTR
+#undef mp_const_none
+#undef mp_const_false
+#undef mp_const_true
+#undef mp_const_empty_tuple
+#undef nlr_raise
+
+/******************************************************************************/
+// Memory allocation
+
+#define m_malloc(n)                     (m_malloc_dyn((n)))
+#define m_free(ptr)                     (m_free_dyn((ptr)))
+#define m_realloc(ptr, new_num_bytes)   (m_realloc_dyn((ptr), (new_num_bytes)))
+
+static inline void *m_malloc_dyn(size_t n) {
+    // TODO won't raise on OOM
+    return mp_fun_table.realloc_(NULL, n, false);
+}
+
+static inline void m_free_dyn(void *ptr) {
+    mp_fun_table.realloc_(ptr, 0, false);
+}
+
+static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) {
+    // TODO won't raise on OOM
+    return mp_fun_table.realloc_(ptr, new_num_bytes, true);
+}
+
+/******************************************************************************/
+// Printing
+
+#define mp_plat_print               (*mp_fun_table.plat_print)
+#define mp_printf(p, ...)           (mp_fun_table.printf_((p), __VA_ARGS__))
+#define mp_vprintf(p, fmt, args)    (mp_fun_table.vprintf_((p), (fmt), (args)))
+
+/******************************************************************************/
+// Types and objects
+
+#define MP_OBJ_NEW_QSTR(x) MP_OBJ_NEW_QSTR_ ## x
+
+#define mp_type_type                        (*mp_fun_table.type_type)
+#define mp_type_str                         (*mp_fun_table.type_str)
+#define mp_type_list                        (*mp_fun_table.type_list)
+#define mp_type_EOFError                    (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_EOFError)))
+#define mp_type_IndexError                  (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_IndexError)))
+#define mp_type_KeyError                    (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_KeyError)))
+#define mp_type_NotImplementedError         (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_NotImplementedError)))
+#define mp_type_RuntimeError                (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_RuntimeError)))
+#define mp_type_TypeError                   (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_TypeError)))
+#define mp_type_ValueError                  (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_ValueError)))
+
+#define mp_stream_read_obj                  (*mp_fun_table.stream_read_obj)
+#define mp_stream_readinto_obj              (*mp_fun_table.stream_readinto_obj)
+#define mp_stream_unbuffered_readline_obj   (*mp_fun_table.stream_unbuffered_readline_obj)
+#define mp_stream_write_obj                 (*mp_fun_table.stream_write_obj)
+
+#define mp_const_none                       ((mp_obj_t)mp_fun_table.const_none)
+#define mp_const_false                      ((mp_obj_t)mp_fun_table.const_false)
+#define mp_const_true                       ((mp_obj_t)mp_fun_table.const_true)
+#define mp_const_empty_tuple                (mp_fun_table.new_tuple(0, NULL))
+
+#define mp_obj_new_bool(b)                  ((b) ? (mp_obj_t)mp_fun_table.const_true : (mp_obj_t)mp_fun_table.const_false)
+#define mp_obj_new_int(i)                   (mp_fun_table.native_to_obj(i, MP_NATIVE_TYPE_INT))
+#define mp_obj_new_str(data, len)           (mp_fun_table.obj_new_str((data), (len)))
+#define mp_obj_new_str_of_type(t, d, l)     (mp_obj_new_str_of_type_dyn((t), (d), (l)))
+#define mp_obj_new_bytes(data, len)         (mp_fun_table.obj_new_bytes((data), (len)))
+#define mp_obj_new_bytearray_by_ref(n, i)   (mp_fun_table.obj_new_bytearray_by_ref((n), (i)))
+#define mp_obj_new_tuple(n, items)          (mp_fun_table.new_tuple((n), (items)))
+#define mp_obj_new_list(n, items)           (mp_fun_table.new_list((n), (items)))
+
+#define mp_obj_get_type(o)                  (mp_fun_table.obj_get_type((o)))
+#define mp_obj_get_int(o)                   (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_INT))
+#define mp_obj_str_get_str(s)               ((void*)mp_fun_table.native_from_obj(s, MP_NATIVE_TYPE_PTR))
+#define mp_obj_str_get_data(o, len)         (mp_obj_str_get_data_dyn((o), (len)))
+#define mp_get_buffer_raise(o, bufinfo, fl) (mp_fun_table.get_buffer_raise((o), (bufinfo), (fl)))
+#define mp_get_stream_raise(s, flags)       (mp_fun_table.get_stream_raise((s), (flags)))
+
+#define mp_obj_list_append(list, item)      (mp_fun_table.list_append((list), (item)))
+
+static inline mp_obj_t mp_obj_new_str_of_type_dyn(const mp_obj_type_t *type, const byte* data, size_t len) {
+    if (type == &mp_type_str) {
+        return mp_obj_new_str((const char*)data, len);
+    } else {
+        return mp_obj_new_bytes(data, len);
+    }
+}
+
+static inline void *mp_obj_str_get_data_dyn(mp_obj_t o, size_t *l) {
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(o, &bufinfo, MP_BUFFER_READ);
+    *l = bufinfo.len;
+    return bufinfo.buf;
+}
+
+/******************************************************************************/
+// General runtime functions
+
+#define mp_load_name(qst)           (mp_fun_table.load_name(qst))
+#define mp_load_global(qst)         (mp_fun_table.load_global(qst))
+#define mp_store_global(qst, obj)   (mp_fun_table.store_global((qst), (obj)))
+#define mp_unary_op(op, obj)        (mp_fun_table.unary_op((op), (obj)))
+#define mp_binary_op(op, lhs, rhs)  (mp_fun_table.binary_op((op), (lhs), (rhs)))
+
+#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \
+    (mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args)))
+
+#define mp_call_function_n_kw(fun, n_args, n_kw, args) \
+    (mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args))
+
+#define mp_arg_check_num(n_args, n_kw, n_args_min, n_args_max, takes_kw) \
+    (mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw))))
+
+#define MP_DYNRUNTIME_INIT_ENTRY \
+    mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \
+    mp_raw_code_t rc; \
+    rc.kind = MP_CODE_NATIVE_VIPER; \
+    rc.scope_flags = 0; \
+    rc.const_table = (void*)self->const_table; \
+    (void)rc;
+
+#define MP_DYNRUNTIME_INIT_EXIT \
+    mp_fun_table.swap_globals(old_globals); \
+    return mp_const_none;
+
+#define MP_DYNRUNTIME_MAKE_FUNCTION(f) \
+    (mp_make_function_from_raw_code((rc.fun_data = (f), &rc), MP_OBJ_NULL, MP_OBJ_NULL))
+
+/******************************************************************************/
+// Exceptions
+
+#define mp_obj_new_exception(o)                 ((mp_obj_t)(o)) // Assumes returned object will be raised, will create instance then
+#define mp_obj_new_exception_arg1(e_type, arg)  (mp_obj_new_exception_arg1_dyn((e_type), (arg)))
+
+#define nlr_raise(o)                            (mp_raise_dyn(o))
+#define mp_raise_msg(type, msg)                 (mp_fun_table.raise_msg((type), (msg)))
+#define mp_raise_OSError(er)                    (mp_raise_OSError_dyn(er))
+#define mp_raise_NotImplementedError(msg)       (mp_raise_msg(&mp_type_NotImplementedError, (msg)))
+#define mp_raise_TypeError(msg)                 (mp_raise_msg(&mp_type_TypeError, (msg)))
+#define mp_raise_ValueError(msg)                (mp_raise_msg(&mp_type_ValueError, (msg)))
+
+static inline mp_obj_t mp_obj_new_exception_arg1_dyn(const mp_obj_type_t *exc_type, mp_obj_t arg) {
+    mp_obj_t args[1] = { arg };
+    return mp_call_function_n_kw(MP_OBJ_FROM_PTR(exc_type), 1, 0, &args[0]);
+}
+
+static NORETURN inline void mp_raise_dyn(mp_obj_t o) {
+    mp_fun_table.raise(o);
+    for (;;) {
+    }
+}
+
+static inline void mp_raise_OSError_dyn(int er) {
+    mp_obj_t args[1] = { MP_OBJ_NEW_SMALL_INT(er) };
+    nlr_raise(mp_call_function_n_kw(mp_load_global(MP_QSTR_OSError), 1, 0, &args[0]));
+}
+
+#endif // MICROPY_INCLUDED_PY_DYNRUNTIME_H

py/dynruntime.mk

@@ -0,0 +1,134 @@
+# Makefile fragment for generating native .mpy files from C source
+# MPY_DIR must be set to the top of the MicroPython source tree
+
+BUILD ?= build
+
+ECHO = @echo
+RM = /bin/rm
+MKDIR = /bin/mkdir
+PYTHON = python3
+MPY_CROSS = $(MPY_DIR)/mpy-cross/mpy-cross
+MPY_TOOL = $(PYTHON) $(MPY_DIR)/tools/mpy-tool.py
+MPY_LD = $(PYTHON) $(MPY_DIR)/tools/mpy_ld.py
+
+Q = @
+ifeq ("$(origin V)", "command line")
+ifeq ($(V),1)
+Q =
+MPY_LD += '-vvv'
+endif
+endif
+
+ARCH_UPPER = $(shell echo $(ARCH) | tr '[:lower:]' '[:upper:]')
+CONFIG_H = $(BUILD)/$(MOD).config.h
+
+CFLAGS += -I. -I$(MPY_DIR)
+CFLAGS += -std=c99
+CFLAGS += -Os
+CFLAGS += -Wall -Werror -DNDEBUG
+CFLAGS += -DNO_QSTR
+CFLAGS += -DMP_CONFIGFILE='<$(CONFIG_H)>'
+CFLAGS += -fpic -fno-common
+CFLAGS += -U _FORTIFY_SOURCE # prevent use of __*_chk libc functions
+#CFLAGS += -fdata-sections -ffunction-sections
+
+MPY_CROSS_FLAGS += -march=$(ARCH)
+
+SRC_O += $(addprefix $(BUILD)/, $(patsubst %.c,%.o,$(filter %.c,$(SRC))))
+SRC_MPY += $(addprefix $(BUILD)/, $(patsubst %.py,%.mpy,$(filter %.py,$(SRC))))
+
+################################################################################
+# Architecture configuration
+
+ifeq ($(ARCH),x86)
+
+# x86
+CROSS =
+CFLAGS += -m32 -fno-stack-protector
+MPY_CROSS_FLAGS += -mcache-lookup-bc
+
+else ifeq ($(ARCH),x64)
+
+# x64
+CROSS =
+CFLAGS += -fno-stack-protector
+MPY_CROSS_FLAGS += -mcache-lookup-bc
+
+else ifeq ($(ARCH),armv7m)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m3
+
+else ifeq ($(ARCH),armv7emsp)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m4
+CFLAGS += -mfpu=fpv4-sp-d16 -mfloat-abi=hard
+
+else ifeq ($(ARCH),armv7emdp)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m7
+CFLAGS += -mfpu=fpv5-d16 -mfloat-abi=hard
+
+else ifeq ($(ARCH),xtensa)
+
+# xtensa
+CROSS = xtensa-lx106-elf-
+CFLAGS += -mforce-l32
+
+else ifeq ($(ARCH),xtensawin)
+
+# xtensawin
+CROSS = xtensa-esp32-elf-
+CFLAGS +=
+
+else
+$(error architecture '$(ARCH)' not supported)
+endif
+
+CFLAGS += $(CFLAGS_EXTRA)
+
+################################################################################
+# Build rules
+
+.PHONY: all clean
+
+all: $(MOD).mpy
+
+clean:
+	$(RM) -rf $(BUILD) $(CLEAN_EXTRA)
+
+# Create build destination directories first
+BUILD_DIRS = $(sort $(dir $(CONFIG_H) $(SRC_O) $(SRC_MPY)))
+$(CONFIG_H) $(SRC_O) $(SRC_MPY): | $(BUILD_DIRS)
+$(BUILD_DIRS):
+	$(Q)$(MKDIR) -p $@
+
+# Preprocess all source files to generate $(CONFIG_H)
+$(CONFIG_H): $(SRC)
+	$(ECHO) "GEN $@"
+	$(Q)$(MPY_LD) --arch $(ARCH) --preprocess -o $@ $^
+
+# Build .o from .c source files
+$(BUILD)/%.o: %.c $(CONFIG_H) Makefile
+	$(ECHO) "CC $<"
+	$(Q)$(CROSS)gcc $(CFLAGS) -o $@ -c $<
+
+# Build .mpy from .py source files
+$(BUILD)/%.mpy: %.py
+	$(ECHO) "MPY $<"
+	$(Q)$(MPY_CROSS) $(MPY_CROSS_FLAGS) -o $@ $<
+
+# Build native .mpy from object files
+$(BUILD)/$(MOD).native.mpy: $(SRC_O)
+	$(ECHO) "LINK $<"
+	$(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) -o $@ $^
+
+# Build final .mpy from all intermediate .mpy files
+$(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY)
+	$(ECHO) "GEN $@"
+	$(Q)$(MPY_TOOL) --merge -o $@ $^