Skip to content

Commit

Permalink
tools/mpy_ld.py: Add new mpy_ld.py tool and associated build files.
Browse files Browse the repository at this point in the history
This commit adds a new tool called mpy_ld.py which is essentially a linker
that builds .mpy files directly from .o files.  A new header file
(dynruntime.h) and makefile fragment (dynruntime.mk) are also included
which allow building .mpy files from C source code.  Such .mpy files can
then be dynamically imported as though they were a normal Python module,
even though they are implemented in C.

Converting .o files directly (rather than pre-linked .elf files) allows the
resulting .mpy to be more efficient because it has more control over the
relocations; for example it can skip PLT indirection.  Doing it this way
also allows supporting more architectures, such as Xtensa which has
specific needs for position-independent code and the GOT.

The tool supports targets of x86, x86-64, ARM Thumb and Xtensa (windowed
and non-windowed).  BSS, text and rodata sections are supported, with
relocations to all internal sections and symbols, as well as relocations to
some external symbols (defined by dynruntime.h), and linking of qstrs.
  • Loading branch information
dpgeorge committed Dec 12, 2019
1 parent 2787984 commit aad79ad
Show file tree
Hide file tree
Showing 3 changed files with 1,305 additions and 0 deletions.
193 changes: 193 additions & 0 deletions py/dynruntime.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2019 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef MICROPY_INCLUDED_PY_DYNRUNTIME_H
#define MICROPY_INCLUDED_PY_DYNRUNTIME_H

// This header file contains definitions to dynamically implement the static
// MicroPython runtime API defined in py/obj.h and py/runtime.h.

#include "py/nativeglue.h"
#include "py/objstr.h"

#undef MP_ROM_QSTR
#undef MP_OBJ_QSTR_VALUE
#undef MP_OBJ_NEW_QSTR
#undef mp_const_none
#undef mp_const_false
#undef mp_const_true
#undef mp_const_empty_tuple
#undef nlr_raise

/******************************************************************************/
// Memory allocation

#define m_malloc(n) (m_malloc_dyn((n)))
#define m_free(ptr) (m_free_dyn((ptr)))
#define m_realloc(ptr, new_num_bytes) (m_realloc_dyn((ptr), (new_num_bytes)))

static inline void *m_malloc_dyn(size_t n) {
// TODO won't raise on OOM
return mp_fun_table.realloc_(NULL, n, false);
}

static inline void m_free_dyn(void *ptr) {
mp_fun_table.realloc_(ptr, 0, false);
}

static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) {
// TODO won't raise on OOM
return mp_fun_table.realloc_(ptr, new_num_bytes, true);
}

/******************************************************************************/
// Printing

#define mp_plat_print (*mp_fun_table.plat_print)
#define mp_printf(p, ...) (mp_fun_table.printf_((p), __VA_ARGS__))
#define mp_vprintf(p, fmt, args) (mp_fun_table.vprintf_((p), (fmt), (args)))

/******************************************************************************/
// Types and objects

#define MP_OBJ_NEW_QSTR(x) MP_OBJ_NEW_QSTR_ ## x

#define mp_type_type (*mp_fun_table.type_type)
#define mp_type_str (*mp_fun_table.type_str)
#define mp_type_list (*mp_fun_table.type_list)
#define mp_type_EOFError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_EOFError)))
#define mp_type_IndexError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_IndexError)))
#define mp_type_KeyError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_KeyError)))
#define mp_type_NotImplementedError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_NotImplementedError)))
#define mp_type_RuntimeError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_RuntimeError)))
#define mp_type_TypeError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_TypeError)))
#define mp_type_ValueError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_ValueError)))

#define mp_stream_read_obj (*mp_fun_table.stream_read_obj)
#define mp_stream_readinto_obj (*mp_fun_table.stream_readinto_obj)
#define mp_stream_unbuffered_readline_obj (*mp_fun_table.stream_unbuffered_readline_obj)
#define mp_stream_write_obj (*mp_fun_table.stream_write_obj)

#define mp_const_none ((mp_obj_t)mp_fun_table.const_none)
#define mp_const_false ((mp_obj_t)mp_fun_table.const_false)
#define mp_const_true ((mp_obj_t)mp_fun_table.const_true)
#define mp_const_empty_tuple (mp_fun_table.new_tuple(0, NULL))

#define mp_obj_new_bool(b) ((b) ? (mp_obj_t)mp_fun_table.const_true : (mp_obj_t)mp_fun_table.const_false)
#define mp_obj_new_int(i) (mp_fun_table.native_to_obj(i, MP_NATIVE_TYPE_INT))
#define mp_obj_new_str(data, len) (mp_fun_table.obj_new_str((data), (len)))
#define mp_obj_new_str_of_type(t, d, l) (mp_obj_new_str_of_type_dyn((t), (d), (l)))
#define mp_obj_new_bytes(data, len) (mp_fun_table.obj_new_bytes((data), (len)))
#define mp_obj_new_bytearray_by_ref(n, i) (mp_fun_table.obj_new_bytearray_by_ref((n), (i)))
#define mp_obj_new_tuple(n, items) (mp_fun_table.new_tuple((n), (items)))
#define mp_obj_new_list(n, items) (mp_fun_table.new_list((n), (items)))

#define mp_obj_get_type(o) (mp_fun_table.obj_get_type((o)))
#define mp_obj_get_int(o) (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_INT))
#define mp_obj_str_get_str(s) ((void*)mp_fun_table.native_from_obj(s, MP_NATIVE_TYPE_PTR))
#define mp_obj_str_get_data(o, len) (mp_obj_str_get_data_dyn((o), (len)))
#define mp_get_buffer_raise(o, bufinfo, fl) (mp_fun_table.get_buffer_raise((o), (bufinfo), (fl)))
#define mp_get_stream_raise(s, flags) (mp_fun_table.get_stream_raise((s), (flags)))

#define mp_obj_list_append(list, item) (mp_fun_table.list_append((list), (item)))

static inline mp_obj_t mp_obj_new_str_of_type_dyn(const mp_obj_type_t *type, const byte* data, size_t len) {
if (type == &mp_type_str) {
return mp_obj_new_str((const char*)data, len);
} else {
return mp_obj_new_bytes(data, len);
}
}

static inline void *mp_obj_str_get_data_dyn(mp_obj_t o, size_t *l) {
mp_buffer_info_t bufinfo;
mp_get_buffer_raise(o, &bufinfo, MP_BUFFER_READ);
*l = bufinfo.len;
return bufinfo.buf;
}

/******************************************************************************/
// General runtime functions

#define mp_load_name(qst) (mp_fun_table.load_name(qst))
#define mp_load_global(qst) (mp_fun_table.load_global(qst))
#define mp_store_global(qst, obj) (mp_fun_table.store_global((qst), (obj)))
#define mp_unary_op(op, obj) (mp_fun_table.unary_op((op), (obj)))
#define mp_binary_op(op, lhs, rhs) (mp_fun_table.binary_op((op), (lhs), (rhs)))

#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \
(mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args)))

#define mp_call_function_n_kw(fun, n_args, n_kw, args) \
(mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args))

#define mp_arg_check_num(n_args, n_kw, n_args_min, n_args_max, takes_kw) \
(mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw))))

#define MP_DYNRUNTIME_INIT_ENTRY \
mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \
mp_raw_code_t rc; \
rc.kind = MP_CODE_NATIVE_VIPER; \
rc.scope_flags = 0; \
rc.const_table = (void*)self->const_table; \
(void)rc;

#define MP_DYNRUNTIME_INIT_EXIT \
mp_fun_table.swap_globals(old_globals); \
return mp_const_none;

#define MP_DYNRUNTIME_MAKE_FUNCTION(f) \
(mp_make_function_from_raw_code((rc.fun_data = (f), &rc), MP_OBJ_NULL, MP_OBJ_NULL))

/******************************************************************************/
// Exceptions

#define mp_obj_new_exception(o) ((mp_obj_t)(o)) // Assumes returned object will be raised, will create instance then
#define mp_obj_new_exception_arg1(e_type, arg) (mp_obj_new_exception_arg1_dyn((e_type), (arg)))

#define nlr_raise(o) (mp_raise_dyn(o))
#define mp_raise_msg(type, msg) (mp_fun_table.raise_msg((type), (msg)))
#define mp_raise_OSError(er) (mp_raise_OSError_dyn(er))
#define mp_raise_NotImplementedError(msg) (mp_raise_msg(&mp_type_NotImplementedError, (msg)))
#define mp_raise_TypeError(msg) (mp_raise_msg(&mp_type_TypeError, (msg)))
#define mp_raise_ValueError(msg) (mp_raise_msg(&mp_type_ValueError, (msg)))

static inline mp_obj_t mp_obj_new_exception_arg1_dyn(const mp_obj_type_t *exc_type, mp_obj_t arg) {
mp_obj_t args[1] = { arg };
return mp_call_function_n_kw(MP_OBJ_FROM_PTR(exc_type), 1, 0, &args[0]);
}

static NORETURN inline void mp_raise_dyn(mp_obj_t o) {
mp_fun_table.raise(o);
for (;;) {
}
}

static inline void mp_raise_OSError_dyn(int er) {
mp_obj_t args[1] = { MP_OBJ_NEW_SMALL_INT(er) };
nlr_raise(mp_call_function_n_kw(mp_load_global(MP_QSTR_OSError), 1, 0, &args[0]));
}

#endif // MICROPY_INCLUDED_PY_DYNRUNTIME_H
134 changes: 134 additions & 0 deletions py/dynruntime.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
# Makefile fragment for generating native .mpy files from C source
# MPY_DIR must be set to the top of the MicroPython source tree

BUILD ?= build

ECHO = @echo
RM = /bin/rm
MKDIR = /bin/mkdir
PYTHON = python3
MPY_CROSS = $(MPY_DIR)/mpy-cross/mpy-cross
MPY_TOOL = $(PYTHON) $(MPY_DIR)/tools/mpy-tool.py
MPY_LD = $(PYTHON) $(MPY_DIR)/tools/mpy_ld.py

Q = @
ifeq ("$(origin V)", "command line")
ifeq ($(V),1)
Q =
MPY_LD += '-vvv'
endif
endif

ARCH_UPPER = $(shell echo $(ARCH) | tr '[:lower:]' '[:upper:]')
CONFIG_H = $(BUILD)/$(MOD).config.h

CFLAGS += -I. -I$(MPY_DIR)
CFLAGS += -std=c99
CFLAGS += -Os
CFLAGS += -Wall -Werror -DNDEBUG
CFLAGS += -DNO_QSTR
CFLAGS += -DMP_CONFIGFILE='<$(CONFIG_H)>'
CFLAGS += -fpic -fno-common
CFLAGS += -U _FORTIFY_SOURCE # prevent use of __*_chk libc functions
#CFLAGS += -fdata-sections -ffunction-sections

MPY_CROSS_FLAGS += -march=$(ARCH)

SRC_O += $(addprefix $(BUILD)/, $(patsubst %.c,%.o,$(filter %.c,$(SRC))))
SRC_MPY += $(addprefix $(BUILD)/, $(patsubst %.py,%.mpy,$(filter %.py,$(SRC))))

################################################################################
# Architecture configuration

ifeq ($(ARCH),x86)

# x86
CROSS =
CFLAGS += -m32 -fno-stack-protector
MPY_CROSS_FLAGS += -mcache-lookup-bc

else ifeq ($(ARCH),x64)

# x64
CROSS =
CFLAGS += -fno-stack-protector
MPY_CROSS_FLAGS += -mcache-lookup-bc

else ifeq ($(ARCH),armv7m)

# thumb
CROSS = arm-none-eabi-
CFLAGS += -mthumb -mcpu=cortex-m3

else ifeq ($(ARCH),armv7emsp)

# thumb
CROSS = arm-none-eabi-
CFLAGS += -mthumb -mcpu=cortex-m4
CFLAGS += -mfpu=fpv4-sp-d16 -mfloat-abi=hard

else ifeq ($(ARCH),armv7emdp)

# thumb
CROSS = arm-none-eabi-
CFLAGS += -mthumb -mcpu=cortex-m7
CFLAGS += -mfpu=fpv5-d16 -mfloat-abi=hard

else ifeq ($(ARCH),xtensa)

# xtensa
CROSS = xtensa-lx106-elf-
CFLAGS += -mforce-l32

else ifeq ($(ARCH),xtensawin)

# xtensawin
CROSS = xtensa-esp32-elf-
CFLAGS +=

else
$(error architecture '$(ARCH)' not supported)
endif

CFLAGS += $(CFLAGS_EXTRA)

################################################################################
# Build rules

.PHONY: all clean

all: $(MOD).mpy

clean:
$(RM) -rf $(BUILD) $(CLEAN_EXTRA)

# Create build destination directories first
BUILD_DIRS = $(sort $(dir $(CONFIG_H) $(SRC_O) $(SRC_MPY)))
$(CONFIG_H) $(SRC_O) $(SRC_MPY): | $(BUILD_DIRS)
$(BUILD_DIRS):
$(Q)$(MKDIR) -p $@

# Preprocess all source files to generate $(CONFIG_H)
$(CONFIG_H): $(SRC)
$(ECHO) "GEN $@"
$(Q)$(MPY_LD) --arch $(ARCH) --preprocess -o $@ $^

# Build .o from .c source files
$(BUILD)/%.o: %.c $(CONFIG_H) Makefile
$(ECHO) "CC $<"
$(Q)$(CROSS)gcc $(CFLAGS) -o $@ -c $<

# Build .mpy from .py source files
$(BUILD)/%.mpy: %.py
$(ECHO) "MPY $<"
$(Q)$(MPY_CROSS) $(MPY_CROSS_FLAGS) -o $@ $<

# Build native .mpy from object files
$(BUILD)/$(MOD).native.mpy: $(SRC_O)
$(ECHO) "LINK $<"
$(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) -o $@ $^

# Build final .mpy from all intermediate .mpy files
$(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY)
$(ECHO) "GEN $@"
$(Q)$(MPY_TOOL) --merge -o $@ $^

0 comments on commit aad79ad

Please sign in to comment.