Add threading implementation for the ESP32

esp32/Makefile

@@ -128,6 +128,7 @@ SRC_C = \
 	modesp.c \
 	espneopixel.c \
 	machine_hw_spi.c \
+	mpthreadport.c \
 	$(SRC_MOD)
 
 STM_SRC_C = $(addprefix stmhal/,\

esp32/gccollect.c

@@ -6,6 +6,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2014 Damien P. George
+ * Copyright (c) 2017 Pycom Limited
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -28,29 +29,38 @@
 
 #include <stdio.h>
 
+#include "py/mpconfig.h"
+#include "py/mpstate.h"
 #include "py/gc.h"
+#include "py/mpthread.h"
 #include "gccollect.h"
+#include "soc/cpu.h"
+#include "xtensa/hal.h"
 
-// As we do not have control over the application entry point, there is no way
-// to figure out the real stack base on runtime, so it needs to be hardcoded
-#define STACK_END   0x40000000
 
-mp_uint_t gc_helper_get_regs_and_sp(mp_uint_t *regs) {
-    // TODO properly
-    return (mp_uint_t)regs;
+static void gc_collect_inner(int level) {
+    if (level < XCHAL_NUM_AREGS / 8) {
+        gc_collect_inner(level + 1);
+        if (level != 0) {
+            return;
+        }
+    }
+
+    if (level == XCHAL_NUM_AREGS / 8) {
+        // get the sp
+        volatile uint32_t sp = (uint32_t)get_sp();
+        gc_collect_root((void**)sp, ((mp_uint_t)MP_STATE_THREAD(stack_top) - sp) / sizeof(uint32_t));
+        return;
+    }
+
+    // trace root pointers from any threads
+    #if MICROPY_PY_THREAD
+    mp_thread_gc_others();
+    #endif
 }
 
 void gc_collect(void) {
-    // start the GC
     gc_collect_start();
-
-    // get the registers and the sp
-    mp_uint_t regs[8];
-    mp_uint_t sp = gc_helper_get_regs_and_sp(regs);
-
-    // trace the stack, including the registers (since they live on the stack in this function)
-    gc_collect_root((void**)sp, (STACK_END - sp) / sizeof(uint32_t));
-
-    // end the GC
+    gc_collect_inner(0);
     gc_collect_end();
 }

esp32/main.c

@@ -34,6 +34,7 @@
 #include "esp_system.h"
 #include "nvs_flash.h"
 #include "esp_task.h"
+#include "soc/cpu.h"
 
 #include "py/stackctrl.h"
 #include "py/nlr.h"
@@ -46,22 +47,30 @@
 #include "lib/utils/pyexec.h"
 #include "uart.h"
 #include "modmachine.h"
+#include "mpthreadport.h"
+
 
 // MicroPython runs as a task under FreeRTOS
 #define MP_TASK_PRIORITY        (ESP_TASK_PRIO_MIN + 1)
 #define MP_TASK_STACK_SIZE      (16 * 1024)
 #define MP_TASK_STACK_LEN       (MP_TASK_STACK_SIZE / sizeof(StackType_t))
 #define MP_TASK_HEAP_SIZE       (96 * 1024)
 
-//STATIC StaticTask_t mp_task_tcb;
-//STATIC StackType_t mp_task_stack[MP_TASK_STACK_LEN] __attribute__((aligned (8)));
+STATIC StaticTask_t mp_task_tcb;
+STATIC StackType_t mp_task_stack[MP_TASK_STACK_LEN] __attribute__((aligned (8)));
 STATIC uint8_t mp_task_heap[MP_TASK_HEAP_SIZE];
 
 void mp_task(void *pvParameter) {
+    volatile uint32_t sp = (uint32_t)get_sp();
+    #if MICROPY_PY_THREAD
+    mp_thread_init(&mp_task_stack[0], MP_TASK_STACK_LEN);
+    #endif
     uart_init();
+
 soft_reset:
-    mp_stack_set_top((void*)&pvParameter);
-    mp_stack_set_limit(MP_TASK_STACK_SIZE - 512);
+    // initialise the stack pointer for the main thread
+    mp_stack_set_top((void *)sp);
+    mp_stack_set_limit(MP_TASK_STACK_SIZE - 1024);
     gc_init(mp_task_heap, mp_task_heap + sizeof(mp_task_heap));
     mp_init();
     mp_obj_list_init(mp_sys_path, 0);
@@ -92,6 +101,10 @@ void mp_task(void *pvParameter) {
         }
     }
 
+    #if MICROPY_PY_THREAD
+    mp_thread_deinit();
+    #endif
+
     mp_hal_stdout_tx_str("PYB: soft reboot\r\n");
 
     // deinitialise peripherals
@@ -104,9 +117,8 @@ void mp_task(void *pvParameter) {
 
 void app_main(void) {
     nvs_flash_init();
-    // TODO use xTaskCreateStatic (needs custom FreeRTOSConfig.h)
-    xTaskCreatePinnedToCore(mp_task, "mp_task", MP_TASK_STACK_LEN, NULL, MP_TASK_PRIORITY, NULL, 0);
-    //xTaskCreateStatic(mp_task, "mp_task", MP_TASK_STACK_LEN, NULL, MP_TASK_PRIORITY, &mp_task_stack[0], &mp_task_tcb);
+    xTaskCreateStaticPinnedToCore(mp_task, "mp_task", MP_TASK_STACK_LEN, NULL, MP_TASK_PRIORITY,
+                                  &mp_task_stack[0], &mp_task_tcb, 0);
 }
 
 void nlr_jump_fail(void *val) {

esp32/modnetwork.c

@@ -241,7 +241,9 @@ STATIC mp_obj_t esp_connect(size_t n_args, const mp_obj_t *args) {
         memcpy(wifi_sta_config.sta.password, p, MIN(len, sizeof(wifi_sta_config.sta.password)));
         ESP_EXCEPTIONS( esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_sta_config) );
     }
+    MP_THREAD_GIL_EXIT();
     ESP_EXCEPTIONS( esp_wifi_connect() );
+    MP_THREAD_GIL_ENTER();
     wifi_sta_connected = true;
 
     return mp_const_none;
@@ -274,7 +276,9 @@ STATIC mp_obj_t esp_scan(mp_obj_t self_in) {
     mp_obj_t list = mp_obj_new_list(0, NULL);
     wifi_scan_config_t config = { 0 };
     // XXX how do we scan hidden APs (and if we can scan them, are they really hidden?)
+    MP_THREAD_GIL_EXIT();
     esp_err_t status = esp_wifi_scan_start(&config, 1);
+    MP_THREAD_GIL_ENTER();
     if (status == 0) {
         uint16_t count = 0;
         ESP_EXCEPTIONS( esp_wifi_scan_get_ap_num(&count) );

esp32/modsocket.c

@@ -112,7 +112,11 @@ static int _socket_getaddrinfo2(const mp_obj_t host, const mp_obj_t portx, struc
     const char *host_str = mp_obj_str_get_data(host, &host_len);
     const char *port_str = mp_obj_str_get_data(port, &port_len);
 
-    return lwip_getaddrinfo(host_str, port_str, &hints, resp);
+    MP_THREAD_GIL_EXIT();
+    int res = lwip_getaddrinfo(host_str, port_str, &hints, resp);
+    MP_THREAD_GIL_ENTER();
+
+    return res;
 }
 
 int _socket_getaddrinfo(const mp_obj_t addrtuple, struct addrinfo **resp) {
@@ -151,7 +155,9 @@ STATIC mp_obj_t socket_accept(const mp_obj_t arg0) {
 
     int new_fd = -1;
     for (int i=0; i<=self->retries; i++) {
+        MP_THREAD_GIL_EXIT();
         new_fd = lwip_accept_r(self->fd, &addr, &addr_len);
+        MP_THREAD_GIL_ENTER();
         if (new_fd >= 0) break;
         if (errno != EAGAIN) exception_from_errno(errno);
         check_for_exceptions();
@@ -182,7 +188,9 @@ STATIC mp_obj_t socket_connect(const mp_obj_t arg0, const mp_obj_t arg1) {
     socket_obj_t *self = MP_OBJ_TO_PTR(arg0);
     struct addrinfo *res;
     _socket_getaddrinfo(arg1, &res);
+    MP_THREAD_GIL_EXIT();
     int r = lwip_connect_r(self->fd, res->ai_addr, res->ai_addrlen);
+    MP_THREAD_GIL_ENTER();
     lwip_freeaddrinfo(res);
     if (r != 0) {
         exception_from_errno(errno);
@@ -255,7 +263,9 @@ mp_obj_t _socket_recvfrom(mp_obj_t self_in, mp_obj_t len_in,
 
     // XXX Would be nicer to use RTC to handle timeouts
     for (int i=0; i<=sock->retries; i++) {
+        MP_THREAD_GIL_EXIT();
         int r = lwip_recvfrom_r(sock->fd, vstr.buf, len, 0, from, from_len);
+        MP_THREAD_GIL_ENTER();
         if (r >= 0) { vstr.len = r; return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr); }
         if (errno != EWOULDBLOCK) exception_from_errno(errno);
         check_for_exceptions();
@@ -286,7 +296,9 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(socket_recvfrom_obj, socket_recvfrom);
 int _socket_send(socket_obj_t *sock, const char *data, size_t datalen) {
     int sentlen = 0;
     for (int i=0; i<sock->retries && sentlen < datalen; i++) {
+        MP_THREAD_GIL_EXIT();
         int r = lwip_write_r(sock->fd, data+sentlen, datalen-sentlen);
+        MP_THREAD_GIL_ENTER();
         if (r < 0 && errno != EWOULDBLOCK) exception_from_errno(errno);
         if (r > 0) sentlen += r;
         check_for_exceptions();
@@ -331,7 +343,9 @@ STATIC mp_obj_t socket_sendto(mp_obj_t self_in, mp_obj_t data_in, mp_obj_t addr_
 
     // send the data
     for (int i=0; i<self->retries; i++) {
+        MP_THREAD_GIL_EXIT();
         int ret = lwip_sendto_r(self->fd, bufinfo.buf, bufinfo.len, 0, (struct sockaddr*)&to, sizeof(to));
+        MP_THREAD_GIL_ENTER();
         if (ret > 0) return mp_obj_new_int_from_uint(ret);
         if (ret == -1 && errno != EWOULDBLOCK) {
             exception_from_errno(errno);
@@ -364,7 +378,9 @@ STATIC mp_uint_t socket_stream_read(mp_obj_t self_in, void *buf, mp_uint_t size,
 
     // XXX Would be nicer to use RTC to handle timeouts
     for (int i=0; i<=sock->retries; i++) {
+        MP_THREAD_GIL_EXIT();
         int x = lwip_recvfrom_r(sock->fd, buf, size, 0, NULL, NULL);
+        MP_THREAD_GIL_ENTER();
         if (x >= 0) return x;
         if (x < 0 && errno != EWOULDBLOCK) { *errcode = errno; return MP_STREAM_ERROR; }
         check_for_exceptions();
@@ -375,7 +391,9 @@ STATIC mp_uint_t socket_stream_read(mp_obj_t self_in, void *buf, mp_uint_t size,
 STATIC mp_uint_t socket_stream_write(mp_obj_t self_in, const void *buf, mp_uint_t size, int *errcode) {
     socket_obj_t *sock = self_in;
     for (int i=0; i<sock->retries; i++) {
+        MP_THREAD_GIL_EXIT();
         int r = lwip_write_r(sock->fd, buf, size);
+        MP_THREAD_GIL_ENTER();
         if (r > 0) return r;
         if (r < 0 && errno != EWOULDBLOCK) { *errcode = errno; return MP_STREAM_ERROR; }
         check_for_exceptions();

esp32/mpconfigport.h

@@ -108,6 +108,9 @@
 #define MICROPY_PY_UERRNO                   (1)
 #define MICROPY_PY_USELECT                  (1)
 #define MICROPY_PY_UTIME_MP_HAL             (1)
+#define MICROPY_PY_THREAD                   (1)
+#define MICROPY_PY_THREAD_GIL               (1)
+#define MICROPY_PY_THREAD_GIL_VM_DIVISOR    (32)
 
 // extended modules
 #define MICROPY_PY_UCTYPES                  (1)
@@ -202,12 +205,23 @@ extern const struct _mp_obj_module_t mp_module_network;
 #define MICROPY_MAKE_POINTER_CALLABLE(p) ((void*)((mp_uint_t)(p)))
 #define MP_PLAT_PRINT_STRN(str, len) mp_hal_stdout_tx_strn_cooked(str, len)
 #define MP_SSIZE_MAX (0x7fffffff)
+
+#if MICROPY_PY_THREAD
+#define MICROPY_EVENT_POLL_HOOK \
+    do { \
+        extern void mp_handle_pending(void); \
+        mp_handle_pending(); \
+        MP_THREAD_GIL_EXIT(); \
+        MP_THREAD_GIL_ENTER(); \
+    } while (0);
+#else
 #define MICROPY_EVENT_POLL_HOOK \
     do { \
         extern void mp_handle_pending(void); \
         mp_handle_pending(); \
         asm("waiti 0"); \
     } while (0);
+#endif
 
 #define UINT_FMT "%u"
 #define INT_FMT "%d"

esp32/mphalport.c

@@ -59,24 +59,30 @@ void mp_hal_stdout_tx_char(char c) {
 }
 
 void mp_hal_stdout_tx_str(const char *str) {
+    MP_THREAD_GIL_EXIT();
     while (*str) {
         mp_hal_stdout_tx_char(*str++);
     }
+    MP_THREAD_GIL_ENTER();
 }
 
 void mp_hal_stdout_tx_strn(const char *str, uint32_t len) {
+    MP_THREAD_GIL_EXIT();
     while (len--) {
         mp_hal_stdout_tx_char(*str++);
     }
+    MP_THREAD_GIL_ENTER();
 }
 
 void mp_hal_stdout_tx_strn_cooked(const char *str, uint32_t len) {
+    MP_THREAD_GIL_EXIT();
     while (len--) {
         if (*str == '\n') {
             mp_hal_stdout_tx_char('\r');
         }
         mp_hal_stdout_tx_char(*str++);
     }
+    MP_THREAD_GIL_ENTER();
 }
 
 uint32_t mp_hal_ticks_ms(void) {