Merge 6d145b8 into 680ce45

docs/esp32/quickref.rst

@@ -537,3 +537,59 @@ the corresponding functions, or you can use the command-line client
 
 See the MicroPython forum for other community-supported alternatives
 to transfer files to an ESP32 board.
+
+Controlling the Python heap size
+--------------------------------
+
+By default MicroPython allocates the largest contiguous chunk of memory to the python heap.
+On a "simple" esp32 this comes out to around 100KB and on an esp32 with external SPIRAM this
+ends up being the full SPIRAM, typically 4MB. This default allocation may not be desirable
+and can be reduced for two use-cases by setting one of two variables in the ``micropython`` NVS
+namespace, see :ref:`esp32.NVS <esp32.NVS>` for details about accessing NVS (Non-Volatile
+Storage).
+
+Because MicroPython allocates the heap as one of the very first actions it is not possible to run
+python code to set the heap size. This is the reason that NVS variables are used and it also means
+that a hard reset is necessary after setting the variables before they take effect.
+A typical use is for ``main.py`` to check heap sizes and, if they're not appropriate,
+to set an NVS variable and perform a hard reset.
+
+The first use-case for this feature is to guarantee that ESP-IDF has some minimum amount of memory
+to work with. For example, by default without SPIRAM there is around 90KB left for ESP-IDF
+right at boot time. This is not enough for two TLS connections and not enough for one
+TLS connection and BLE either. (While 90KB might seem like a lot, it disappears quickly once
+Wifi is started and sockets are connected.)
+
+To give esp-idf a bit more memory, use something like::
+
+    import machine
+    from esp32 import NVS, idf_heap_info, HEAP_DATA
+
+    idf_free = sum([h[2] for h in idf_heap_info(HEAP_DATA)])
+    print("IDF heap free:", idf_free)
+
+    nvs = NVS("micropython")
+    nvs.set_i32("min_idf_heap", 120000)
+    nvs.commit()
+    machine.reset()
+
+Setting the ``min_idf_heap`` NVS variable to 120000 tells MicroPython to reduce its heap allocation
+from the default such that at least 120000 bytes are left for esp-idf.
+
+A second use case is to reduce GC times when using SPIRAM. A GC collection has to read sequentially
+though all of RAM during its sweep phase. When using a SPIRAM with the default 4MB allocation this
+takes about 90ms (assuming 240Mhz cpu and 80Mhz QIO SPIRAM), which is very impactful in a not so
+good way. Often applications only need a few hundred KB and this can be accomplished by setting the
+``max_mp_heap`` NVS variable to the desired size in bytes.
+
+A similar use-case is with an SPIRAM where it is desired to leave memory to a native module, for
+example to allocate a camera framebuffer. The size of the MP heap can be limited to at most 300KB
+using something like::
+
+    import machine
+    from esp32 import NVS
+
+    nvs = NVS("micropython")
+    nvs.set_i32("max_mp_heap", 300*1024)
+    nvs.commit()
+    machine.reset()

docs/library/esp32.rst

@@ -270,6 +270,8 @@ Constants
 
    Selects the wake level for pins.
 
+.. _esp32.NVS:
+
 Non-Volatile Storage
 --------------------
 

ports/esp32/boards/sdkconfig.spiram

@@ -3,4 +3,5 @@
 CONFIG_ESP32_SPIRAM_SUPPORT=y
 CONFIG_SPIRAM_CACHE_WORKAROUND=y
 CONFIG_SPIRAM_IGNORE_NOTFOUND=y
-CONFIG_SPIRAM_USE_MEMMAP=y
+CONFIG_SPIRAM_USE_MEMMAP=n
+CONFIG_SPIRAM_USE_CAPS_ALLOC=y

ports/esp32/esp32_nvs.c

@@ -33,6 +33,7 @@
 #include "nvs_flash.h"
 #include "nvs.h"
 
+
 // This file implements the NVS (Non-Volatile Storage) class in the esp32 module.
 // It provides simple access to the NVS feature provided by ESP-IDF.
 

ports/esp32/main.c

@@ -75,30 +75,43 @@ void mp_task(void *pvParameter) {
     uart_init();
     machine_init();
 
-    // TODO: CONFIG_SPIRAM_SUPPORT is for 3.3 compatibility, remove after move to 4.0.
-    #if CONFIG_ESP32_SPIRAM_SUPPORT || CONFIG_SPIRAM_SUPPORT
-    // Try to use the entire external SPIRAM directly for the heap
-    size_t mp_task_heap_size;
-    void *mp_task_heap = (void *)0x3f800000;
-    switch (esp_spiram_get_chip_size()) {
-        case ESP_SPIRAM_SIZE_16MBITS:
-            mp_task_heap_size = 2 * 1024 * 1024;
-            break;
-        case ESP_SPIRAM_SIZE_32MBITS:
-        case ESP_SPIRAM_SIZE_64MBITS:
-            mp_task_heap_size = 4 * 1024 * 1024;
-            break;
-        default:
-            // No SPIRAM, fallback to normal allocation
-            mp_task_heap_size = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT);
-            mp_task_heap = malloc(mp_task_heap_size);
-            break;
+    // Allocate MicroPython heap. By default we grab the largest contiguous chunk (GC requires
+    // having a contiguous heap). But this can be customized by setting two NVS variables.
+    // There's nothing special here about SPIRAM because the SDK config should set
+    // CONFIG_SPIRAM_USE_CAPS_ALLOC=y which makes any external SPIRAM automatically
+    // show up here under MALLOC_CAP_8BIT memory.
+#define MIN_HEAP_SIZE (20 * 1024)  // simple safety to avoid ending up with a non-functional heap
+    size_t avail_heap_size = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT); // in bytes
+    size_t total_free = heap_caps_get_free_size(MALLOC_CAP_8BIT);
+    size_t mp_task_heap_size = avail_heap_size; // proposed MP heap size
+    nvs_handle_t mp_nvs;
+    if (nvs_open("micropython", NVS_READONLY, &mp_nvs) == ESP_OK) {
+        // implement minimum heap left for esp-idf
+        int32_t min_idf_heap_size = 0; // minimum we leave to esp-idf, in bytes
+        if (nvs_get_i32(mp_nvs, "min_idf_heap", &min_idf_heap_size) == ESP_OK) {
+            if (total_free - mp_task_heap_size < min_idf_heap_size) {
+                // we can't take the largest contig chunk, need to leave more to esp-idf
+                mp_task_heap_size = total_free - min_idf_heap_size;
+                if (mp_task_heap_size < MIN_HEAP_SIZE) {
+                    mp_task_heap_size = MIN_HEAP_SIZE;
+                }
+            }
+        }
+        // implement maximum MP heap size
+        int32_t max_mp_heap_size = mp_task_heap_size; // max we alllocate to MicroPython, in bytes
+        if (nvs_get_i32(mp_nvs, "max_mp_heap", &max_mp_heap_size) == ESP_OK) {
+            if (max_mp_heap_size > MIN_HEAP_SIZE && mp_task_heap_size > max_mp_heap_size) {
+                // we're about to create too large a heap, be more modest
+                mp_task_heap_size = max_mp_heap_size;
+            }
+        }
+        nvs_close(mp_nvs);
+    }
+    void *mp_task_heap = heap_caps_malloc(mp_task_heap_size, MALLOC_CAP_8BIT);
+    if (avail_heap_size != mp_task_heap_size) {
+        printf("Heap limited: avail=%d, actual=%d, left to idf=%d\n",
+            avail_heap_size, mp_task_heap_size, total_free - mp_task_heap_size);
     }
-    #else
-    // Allocate the uPy heap using malloc and get the largest available region
-    size_t mp_task_heap_size = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT);
-    void *mp_task_heap = malloc(mp_task_heap_size);
-    #endif
 
 soft_reset:
     // initialise the stack pointer for the main thread

ports/esp32/modesp32.c

@@ -190,6 +190,7 @@ STATIC const mp_rom_map_elem_t esp32_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_Partition), MP_ROM_PTR(&esp32_partition_type) },
     { MP_ROM_QSTR(MP_QSTR_RMT), MP_ROM_PTR(&esp32_rmt_type) },
     { MP_ROM_QSTR(MP_QSTR_ULP), MP_ROM_PTR(&esp32_ulp_type) },
+    { MP_ROM_QSTR(MP_QSTR_NVS), MP_ROM_PTR(&esp32_nvs_type) },
 
     { MP_ROM_QSTR(MP_QSTR_WAKEUP_ALL_LOW), MP_ROM_FALSE },
     { MP_ROM_QSTR(MP_QSTR_WAKEUP_ANY_HIGH), MP_ROM_TRUE },

ports/esp32/modesp32.h

@@ -30,5 +30,6 @@ extern const mp_obj_type_t esp32_nvs_type;
 extern const mp_obj_type_t esp32_partition_type;
 extern const mp_obj_type_t esp32_rmt_type;
 extern const mp_obj_type_t esp32_ulp_type;
+extern const mp_obj_type_t esp32_nvs_type;
 
 #endif // MICROPY_INCLUDED_ESP32_MODESP32_H

tests/esp32/limit_heap.py

@@ -0,0 +1,44 @@
+# Test MicroPython heap limits on the esp32.
+# This test requires resetting the device and thus does not simply run in run-tests.
+# You can run this test manually using pyboard and hitting ctrl-c after each reset and rerunning
+# the script. It will cycle through the various settings.
+import gc
+import machine
+from esp32 import NVS, idf_heap_info, HEAP_DATA
+
+nvs = NVS("micropython")
+try:
+    min_idf = nvs.get_i32("min_idf_heap")
+except OSError:
+    min_idf = 0
+try:
+    max_mp = nvs.get_i32("max_mp_heap")
+except OSError:
+    max_mp = None
+
+mp_total = gc.mem_alloc() + gc.mem_free()
+print("MP heap:", mp_total)
+idf_free = sum([h[2] for h in idf_heap_info(HEAP_DATA)])
+print("IDF heap free:", idf_free)
+
+if min_idf == 0:
+    nvs.set_i32("min_idf_heap", 100000)
+    nvs.commit()
+    print("IDF MIN heap changed to 100000")
+    machine.reset()
+elif max_mp is None:
+    nvs.set_i32("max_mp_heap", 50000)
+    nvs.commit()
+    print("MAX heap changed to 50000")
+    machine.reset()
+else:
+    try:
+        nvs.erase_key("min_idf_heap")
+    except OSError:
+        pass
+    try:
+        nvs.erase_key("max_mp_heap")
+    except OSError:
+        pass
+    print("Everything reset to default")
+    machine.reset()