Add Storage Extension Support

locale/circuitpython.pot

@@ -112,6 +112,10 @@ msgstr ""
 msgid "%q init failed"
 msgstr ""
 
+#: shared-bindings/dualbank/__init__.c
+msgid "%q is %q"
+msgstr ""
+
 #: py/argcheck.c
 msgid "%q length must be %d"
 msgstr ""
@@ -152,14 +156,6 @@ msgstr ""
 msgid "%q must be >= %d"
 msgstr ""
 
-#: py/argcheck.c
-msgid "%q must be >= 0"
-msgstr ""
-
-#: shared-bindings/vectorio/Circle.c shared-bindings/vectorio/Rectangle.c
-msgid "%q must be >= 1"
-msgstr ""
-
 #: shared-bindings/analogbufio/BufferedIn.c
 #: shared-bindings/audiocore/RawSample.c
 msgid "%q must be a bytearray or array of type 'h', 'H', 'b', or 'B'"
@@ -214,7 +210,7 @@ msgstr ""
 msgid "%q, %q, and %q must all be the same length"
 msgstr ""
 
-#: py/objint.c
+#: py/objint.c shared-bindings/storage/__init__.c
 msgid "%q=%q"
 msgstr ""
 
@@ -910,8 +906,7 @@ msgstr ""
 msgid "Error: Failure to bind"
 msgstr ""
 
-#: ports/raspberrypi/bindings/rp2pio/StateMachine.c py/enum.c
-#: shared-bindings/_bleio/__init__.c shared-bindings/aesio/aes.c
+#: py/enum.c shared-bindings/_bleio/__init__.c shared-bindings/aesio/aes.c
 #: shared-bindings/alarm/__init__.c shared-bindings/busio/SPI.c
 #: shared-bindings/microcontroller/Pin.c
 #: shared-bindings/neopixel_write/__init__.c
@@ -1569,10 +1564,12 @@ msgid "Only 8 or 16 bit mono with "
 msgstr ""
 
 #: ports/espressif/common-hal/wifi/__init__.c
+#: ports/raspberrypi/common-hal/wifi/__init__.c
 msgid "Only IPv4 addresses supported"
 msgstr ""
 
 #: ports/espressif/common-hal/socketpool/Socket.c
+#: ports/raspberrypi/common-hal/socketpool/Socket.c
 msgid "Only IPv4 sockets supported"
 msgstr ""
 
@@ -1642,6 +1639,7 @@ msgid "Out of memory"
 msgstr ""
 
 #: ports/espressif/common-hal/socketpool/Socket.c
+#: ports/raspberrypi/common-hal/socketpool/Socket.c
 msgid "Out of sockets"
 msgstr ""
 
@@ -1696,7 +1694,6 @@ msgid "Pin interrupt already in use"
 msgstr ""
 
 #: shared-bindings/adafruit_bus_device/spi_device/SPIDevice.c
-#: shared-bindings/digitalio/DigitalInOut.c
 msgid "Pin is input only"
 msgstr ""
 
@@ -1916,6 +1913,7 @@ msgid "Slices not supported"
 msgstr ""
 
 #: ports/espressif/common-hal/socketpool/SocketPool.c
+#: ports/raspberrypi/common-hal/socketpool/SocketPool.c
 msgid "SocketPool can only be used with wifi.radio"
 msgstr ""
 
@@ -2341,10 +2339,6 @@ msgstr ""
 msgid "a bytes-like object is required"
 msgstr ""
 
-#: shared-bindings/i2ctarget/I2CTarget.c
-msgid "address out of bounds"
-msgstr ""
-
 #: shared-bindings/i2ctarget/I2CTarget.c
 msgid "addresses is empty"
 msgstr ""
@@ -2814,10 +2808,6 @@ msgstr ""
 msgid "destination buffer must be an array of type 'H' for bit_depth = 16"
 msgstr ""
 
-#: shared-bindings/audiobusio/PDMIn.c
-msgid "destination_length must be an int >= 0"
-msgstr ""
-
 #: py/objdict.c
 msgid "dict update sequence has wrong length"
 msgstr ""

ports/espressif/common-hal/dualbank/__init__.c

@@ -32,6 +32,8 @@
 #include "esp_log.h"
 #include "esp_ota_ops.h"
 
+#include "supervisor/flash.h"
+
 static const esp_partition_t *update_partition = NULL;
 static esp_ota_handle_t update_handle = 0;
 

ports/espressif/supervisor/internal_flash.c

@@ -24,6 +24,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #include "supervisor/internal_flash.h"
 
 #include <stdint.h>
@@ -32,47 +33,114 @@
 
 #include "extmod/vfs.h"
 #include "extmod/vfs_fat.h"
+
 #include "py/mphal.h"
 #include "py/obj.h"
 #include "py/runtime.h"
-#include "lib/oofatfs/ff.h"
 
-#include "components/spi_flash/include/esp_partition.h"
+#include "esp_ota_ops.h"
+#include "esp_partition.h"
 
+#include "supervisor/filesystem.h"
 #include "supervisor/flash.h"
 #include "supervisor/usb.h"
 
-STATIC const esp_partition_t *_partition;
+#define OP_READ     0
+#define OP_WRITE    1
 
 // TODO: Split the caching out of supervisor/shared/external_flash so we can use it.
 #define SECTOR_SIZE 4096
 STATIC uint8_t _cache[SECTOR_SIZE];
 STATIC uint32_t _cache_lba = 0xffffffff;
 
+#if CIRCUITPY_STORAGE_EXTEND
+#if FF_MAX_SS == FF_MIN_SS
+#define SECSIZE(fs) (FF_MIN_SS)
+#else
+#define SECSIZE(fs) ((fs)->ssize)
+#endif // FF_MAX_SS == FF_MIN_SS
+STATIC DWORD fatfs_bytes(void) {
+    FATFS *fatfs = filesystem_circuitpy();
+    return (fatfs->csize * SECSIZE(fatfs)) * (fatfs->n_fatent - 2);
+}
+STATIC bool storage_extended = true;
+STATIC const esp_partition_t *_partition[2];
+#else
+STATIC const esp_partition_t *_partition[1];
+#endif // CIRCUITPY_STORAGE_EXTEND
+
 void supervisor_flash_init(void) {
-    _partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA,
+    if (_partition[0] != NULL) {
+        return;
+    }
+    _partition[0] = esp_partition_find_first(ESP_PARTITION_TYPE_DATA,
         ESP_PARTITION_SUBTYPE_DATA_FAT,
         NULL);
+    #if CIRCUITPY_STORAGE_EXTEND
+    _partition[1] = esp_ota_get_next_update_partition(NULL);
+    #endif
 }
 
 uint32_t supervisor_flash_get_block_size(void) {
     return FILESYSTEM_BLOCK_SIZE;
 }
 
 uint32_t supervisor_flash_get_block_count(void) {
-    return _partition->size / FILESYSTEM_BLOCK_SIZE;
+    #if CIRCUITPY_STORAGE_EXTEND
+    return ((storage_extended) ? (_partition[0]->size + _partition[1]->size) : _partition[0]->size) / FILESYSTEM_BLOCK_SIZE;
+    #else
+    return _partition[0]->size / FILESYSTEM_BLOCK_SIZE;
+    #endif
 }
 
 void port_internal_flash_flush(void) {
+}
 
+STATIC void single_partition_rw(const esp_partition_t *partition, uint8_t *data,
+    const uint32_t offset, const uint32_t size_total, const bool op) {
+    if (op == OP_READ) {
+        esp_partition_read(partition, offset, data, size_total);
+    } else {
+        esp_partition_erase_range(partition, offset, size_total);
+        esp_partition_write(partition, offset, _cache, size_total);
+    }
+}
+
+#if CIRCUITPY_STORAGE_EXTEND
+STATIC void multi_partition_rw(uint8_t *data,
+    const uint32_t offset, const uint32_t size_total, const bool op) {
+    if (offset > _partition[0]->size) {
+        // only r/w partition 1
+        single_partition_rw(_partition[1], data, (offset - _partition[0]->size), size_total, op);
+    } else if ((offset + size_total) > _partition[0]->size) {
+        // first r/w partition 0, then partition 1
+        uint32_t size_0 = _partition[0]->size - offset;
+        uint32_t size_1 = size_total - size_0;
+        if (op == OP_READ) {
+            esp_partition_read(_partition[0], offset, data, size_0);
+            esp_partition_read(_partition[1], 0, (data + size_0), size_1);
+        } else {
+            esp_partition_erase_range(_partition[0], offset, size_0);
+            esp_partition_write(_partition[0], offset, _cache, size_0);
+            esp_partition_erase_range(_partition[1], 0, size_1);
+            esp_partition_write(_partition[1], 0, (_cache + size_0), size_1);
+        }
+    } else {
+        // only r/w partition 0
+        single_partition_rw(_partition[0], data, offset, size_total, op);
+    }
 }
+#endif
 
 mp_uint_t supervisor_flash_read_blocks(uint8_t *dest, uint32_t block, uint32_t num_blocks) {
-    esp_partition_read(_partition,
-        block * FILESYSTEM_BLOCK_SIZE,
-        dest,
-        num_blocks * FILESYSTEM_BLOCK_SIZE);
-    return 0;
+    const uint32_t offset = block * FILESYSTEM_BLOCK_SIZE;
+    const uint32_t read_total = num_blocks * FILESYSTEM_BLOCK_SIZE;
+    #if CIRCUITPY_STORAGE_EXTEND
+    multi_partition_rw(dest, offset, read_total, OP_READ);
+    #else
+    single_partition_rw(_partition[0], dest, offset, read_total, OP_READ);
+    #endif
+    return 0; // success
 }
 
 mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32_t num_blocks) {
@@ -82,12 +150,8 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32
         uint32_t block_address = lba + block;
         uint32_t sector_offset = block_address / blocks_per_sector * SECTOR_SIZE;
         uint8_t block_offset = block_address % blocks_per_sector;
-
         if (_cache_lba != block_address) {
-            esp_partition_read(_partition,
-                sector_offset,
-                _cache,
-                SECTOR_SIZE);
+            supervisor_flash_read_blocks(_cache, sector_offset / FILESYSTEM_BLOCK_SIZE, blocks_per_sector);
             _cache_lba = sector_offset;
         }
         for (uint8_t b = block_offset; b < blocks_per_sector; b++) {
@@ -100,15 +164,34 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32
                 FILESYSTEM_BLOCK_SIZE);
             block++;
         }
-        esp_partition_erase_range(_partition, sector_offset, SECTOR_SIZE);
-        esp_partition_write(_partition,
-            sector_offset,
-            _cache,
-            SECTOR_SIZE);
+        #if CIRCUITPY_STORAGE_EXTEND
+        multi_partition_rw(_cache, sector_offset, SECTOR_SIZE, OP_WRITE);
+        #else
+        single_partition_rw(_partition[0], _cache, sector_offset, SECTOR_SIZE, OP_READ);
+        #endif
     }
-
     return 0; // success
 }
 
 void supervisor_flash_release_cache(void) {
 }
+
+void supervisor_flash_set_extended(bool extended) {
+    #if CIRCUITPY_STORAGE_EXTEND
+    storage_extended = extended;
+    #endif
+}
+
+bool supervisor_flash_get_extended(void) {
+    #if CIRCUITPY_STORAGE_EXTEND
+    return storage_extended;
+    #else
+    return false;
+    #endif
+}
+
+void supervisor_flash_update_extended(void) {
+    #if CIRCUITPY_STORAGE_EXTEND
+    storage_extended = (_partition[0]->size < fatfs_bytes());
+    #endif
+}

py/circuitpy_mpconfig.mk

@@ -384,6 +384,9 @@ CFLAGS += -DCIRCUITPY_STATUS_BAR=$(CIRCUITPY_STATUS_BAR)
 CIRCUITPY_STORAGE ?= 1
 CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE)
 
+CIRCUITPY_STORAGE_EXTEND ?= $(CIRCUITPY_DUALBANK)
+CFLAGS += -DCIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND)
+
 CIRCUITPY_STRUCT ?= 1
 CFLAGS += -DCIRCUITPY_STRUCT=$(CIRCUITPY_STRUCT)
 

shared-bindings/dualbank/__init__.c

@@ -26,6 +26,10 @@
 
 #include "shared-bindings/dualbank/__init__.h"
 
+#if CIRCUITPY_STORAGE_EXTEND
+#include "supervisor/flash.h"
+#endif
+
 //| """DUALBANK Module
 //|
 //| The `dualbank` module adds ability to update and switch
@@ -55,6 +59,14 @@
 //| """
 //| ...
 
+#if CIRCUITPY_STORAGE_EXTEND
+STATIC void raise_error_if_storage_extended(void) {
+    if (supervisor_flash_get_extended()) {
+        mp_raise_msg_varg(&mp_type_RuntimeError, translate("%q is %q"), MP_QSTR_storage, MP_QSTR_extended);
+    }
+}
+#endif
+
 //| def flash(buffer: ReadableBuffer, offset: int = 0) -> None:
 //|     """Writes one of two app partitions at the given offset.
 //|
@@ -70,6 +82,10 @@ STATIC mp_obj_t dualbank_flash(size_t n_args, const mp_obj_t *pos_args, mp_map_t
         { MP_QSTR_offset, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = 0} },
     };
 
+    #if CIRCUITPY_STORAGE_EXTEND
+    raise_error_if_storage_extended();
+    #endif
+
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
 
@@ -94,6 +110,9 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_KW(dualbank_flash_obj, 0, dualbank_flash);
 //|     ...
 //|
 STATIC mp_obj_t dualbank_switch(void) {
+    #if CIRCUITPY_STORAGE_EXTEND
+    raise_error_if_storage_extended();
+    #endif
     common_hal_dualbank_switch();
     return mp_const_none;
 }