FlashStorage & EEPROM libraries

This implements a FlashStorage library, and an EEPROM emulation layer on top of
it. Trade-offs and design decisions are documented in FlashStorage's README.

Signed-off-by: Gergely Nagy <algernon@keyboard.io>

libraries/EEPROM/library.properties

@@ -0,0 +1,8 @@
+name=EEPROM
+version=0.1.0
+author=Keyboard.io, Inc.
+maintainer=Keyboard.io, Inc.
+sentence=EEPROM emulation layer for GD32.
+paragraph=
+category=Data Storage
+architectures=gd32

libraries/EEPROM/src/EEPROM.h

@@ -0,0 +1,38 @@
+/* -*- mode: c++ -*-
+ * Copyright (c) 2021  Keyboard.io, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice, this
+ *     list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *  3. Neither the name of the copyright holder nor the names of its contributors
+ *     may be used to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "FlashStorage.h"
+#include "FlashAsEEPROM.h"
+
+#ifndef EEPROM_EMULATION_SIZE
+#define EEPROM_EMULATION_SIZE 10240
+#endif
+
+static EEPROMClass<EEPROM_EMULATION_SIZE> EEPROM;

libraries/FlashStorage/README.md

@@ -0,0 +1,25 @@
+# FlashStorage
+
+This is a library that implements a way to read and write data from and to
+flash, from within the running program.
+
+## Design decisions
+
+There are different ways to do that, with different trade-offs. The goal during
+the design of this library was simplicity, and to fulfill the particular goals
+of the Kaleidoscope firmware.
+
+As such, the storage in this implementation is always at the end of Flash. This
+allows us to reserve space for it externally, and prevent ever overwriting it,
+so when we flash new firmware, the storage area is not destroyed. As a
+consequence, we can only have one storage area, but its size can be changed at
+compile time. On the other hand, all of this allows the code to be fairly simple
+and straightforward.
+
+## FlashAsEEPROM
+
+For practical reasons, the library includes a `FlashAsEEPROM.h` header, a class
+that implements an Arduino-esque EEPROM emulation layer on top of
+`FlashStorage`. The header does not export an `EEPROM` instance object, that's
+what the `EEPROM` library does. A separate library, for practical and
+convenience reasons.

libraries/FlashStorage/library.properties

@@ -0,0 +1,8 @@
+name=FlashStorage
+version=0.1.0
+author=Keyboard.io, Inc.
+maintainer=Keyboard.io, Inc.
+sentence=The FlashStorage library aims to provide a convenient way to store and retrieve user's data using the non-volatile flash memory of microcontrollers.
+paragraph=
+category=Data Storage
+architectures=gd32

libraries/FlashStorage/src/FlashAsEEPROM.h

@@ -0,0 +1,91 @@
+/* -*- mode: c++ -*-
+ * Copyright (c) 2021  Keyboard.io, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice, this
+ *     list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *  3. Neither the name of the copyright holder nor the names of its contributors
+ *     may be used to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "FlashStorage.h"
+
+template <uint32_t _size>
+class EEPROMClass {
+ private:
+  FlashStorage<_size> storage_;
+
+ public:
+  template<typename T>
+  T& get(uint32_t offset, T& t) {
+    storage_.read(offset, (uint8_t *) &t, sizeof(T));
+    return t;
+  }
+
+  template<typename T>
+  const T& put(uint32_t offset, const T& t) {
+    storage_.writeBulk((const uint8_t *) &t, sizeof(T), offset);
+  }
+  const uint32_t put(uint32_t offset, const uint32_t &u) {
+    storage_.writeSmall((const uint8_t *)&u, sizeof(uint32_t), offset);
+  }
+  const uint16_t put(uint32_t offset, const uint16_t &u) {
+    storage_.writeSmall((const uint8_t *)&u, sizeof(uint16_t), offset);
+  }
+  const uint8_t put(uint32_t offset, const uint8_t &u) {
+    storage_.writeSmall((const uint8_t *)&u, sizeof(uint8_t), offset);
+  }
+
+  uint8_t read(uint32_t offset) {
+    uint8_t val;
+    storage_.read(offset, &val, sizeof(val));
+    return val;
+  }
+
+  void write(uint32_t offset, uint8_t val) {
+    storage_.writeSmall((uint8_t *)&val, sizeof(val), offset);
+  }
+  void write(uint32_t offset, uint16_t val) {
+    storage_.writeSmall((uint8_t *)&val, sizeof(val), offset);
+  }
+  void write(uint32_t offset, uint32_t val) {
+    storage_.writeSmall((uint8_t *)&val, sizeof(val), offset);
+  }
+  void update(uint32_t offset, uint8_t val) {
+    write(offset, val);
+  }
+  void update(uint32_t offset, uint16_t val) {
+    write(offset, val);
+  }
+  void update(uint32_t offset, uint32_t val) {
+    write(offset, val);
+  }
+
+  void begin() {
+    storage_.begin();
+  }
+
+  uint32_t length() {
+    return _size;
+  }
+};

libraries/FlashStorage/src/FlashStorage.h

@@ -0,0 +1,150 @@
+/* -*- mode: c++ -*-
+ * Copyright (c) 2020  GigaDevice Semiconductor Inc.
+ *               2021  Keyboard.io, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice, this
+ *     list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *  3. Neither the name of the copyright holder nor the names of its contributors
+ *     may be used to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+template <uint32_t _storage_size>
+class FlashStorage {
+ private:
+  static constexpr uint32_t fmc_base_address = 0x08000000;
+  static constexpr uint32_t fmc_end_address =
+      fmc_base_address + ARDUINO_UPLOAD_MAXIMUM_SIZE;
+  static constexpr uint32_t data_area_start = fmc_end_address - _storage_size;
+  static constexpr uint32_t bank0_end = fmc_base_address + 512 * 1024 - 1;
+
+  uint16_t pageSizeForAddress(uint32_t addr) {
+    if (addr > bank0_end)
+      return 4096;
+    else
+      return 2048;
+  }
+
+  uint32_t addr(uint32_t offset) {
+    return data_area_start + offset;
+  }
+
+  uint32_t alignAddress(uint32_t addr) {
+    const uint16_t page_size = pageSizeForAddress(addr);
+    return (addr / page_size) * page_size;
+  }
+
+  uint32_t updatePage(uint8_t alignedAddress, uint32_t offset,
+                      const uint8_t *data, uint32_t data_size, uint32_t *data_pos) {
+    uint8_t page_data[4096];
+    uint16_t page_size = pageSizeForAddress(alignedAddress);
+    uint32_t pos = *data_pos;
+
+    // Copy the page data to a temporary, updateable place
+    read(alignedAddress, (uint8_t *)page_data, page_size);
+
+    // Update the page data from our updated data set
+    do {
+      page_data[offset++] = data[pos++];
+    } while (offset < page_size && pos < data_size);
+
+    // Write it all back to the same address
+    fmc_page_erase(alignedAddress);
+
+    uint32_t word_count = page_size / 4;
+    uint32_t i = 0;
+    uint32_t *ptrs = (uint32_t *)page_data;
+
+    do {
+      fmc_word_program(alignedAddress, *ptrs++);
+      alignedAddress += 4U;
+    } while (++i < word_count);
+
+    *data_pos = pos;
+    return alignedAddress;
+  }
+
+ public:
+  void begin() {
+    fmc_unlock();
+  }
+
+  const uint32_t length() {
+    return _storage_size;
+  }
+
+  void read(uint32_t offset, uint8_t *data, uint32_t data_size) {
+    uint32_t address = addr(offset);
+
+    // If we're out of bounds, bail out.
+    if (address < data_area_start || address > fmc_end_address)
+      return;
+
+    uint8_t *src = (uint8_t *)(address);
+    for (auto i = 0; i < data_size; i++) {
+      data[i] = src[i];
+    }
+  }
+
+  void writeSmall(const uint8_t *data, uint8_t size, uint32_t offset) {
+    uint32_t address = addr(offset);
+
+    // If we're out of bounds, bail out.
+    if (address < data_area_start || address > fmc_end_address)
+      return;
+
+    // Do we span bank boundaries?
+    if (address < bank0_end && address + 4 >= bank0_end) {
+      // Instead of figuring out where to split the data and do two small
+      // writes, fall back to writeBulk(), to simplify the logic. A bit of a
+      // waste, as we do two big writes, but this case should be rare enough to
+      // not matter.
+      return writeBulk(data, size, offset);
+    }
+
+    // Within a single bank.
+    uint8_t cell_data[4];
+    read(offset, (uint8_t *)cell_data, sizeof(cell_data));
+    for (uint8_t i = 0; i < size; i++) {
+      cell_data[i] = data[i];
+    }
+    fmc_word_reprogram(address, *((uint32_t *)cell_data));
+  }
+
+  void writeBulk(const uint8_t *data, uint32_t size, uint32_t offset = 0) {
+    uint32_t address = addr(offset);
+
+    // If we're out of bounds, bail out.
+    if (address < data_area_start || address > fmc_end_address)
+      return;
+
+    uint32_t alignedAddr = alignAddress(address);
+    uint32_t page_offset = address - alignedAddr;
+    uint32_t pos = 0;
+
+    do {
+      alignedAddr = updatePage(alignedAddr, page_offset, data, size, &pos);
+      page_offset = 0;
+    } while (pos < size);
+  }
+};

platform.txt

@@ -4,7 +4,7 @@ name=GD32 ARM Boards
 version=1.0.0
 
 #compile variables
-##compile Include 
+##compile Include
 compiler.gd.extra_include= "-I{build.source.path}" "-I{build.core.path}/api/deprecated-avr-comp" "-I{build.core.path}/api/deprecated" "-I{build.core.path}/gd32" "-I{build.core.path}/gd32/Source" "-I{build.system.path}/{build.series}_firmware/{build.series}_standard_peripheral/Source" "-I{build.system.path}/{build.series}_firmware/{build.series}_standard_peripheral/Include"  "-I{build.system.path}/{build.series}_firmware/CMSIS" "-I{build.system.path}/{build.series}_firmware/CMSIS/GD/{build.series}/Include" "-I{build.system.path}/{build.series}_firmware/CMSIS/GD/{build.series}/Source/GCC" "-I{build.system.path}/{build.series}_firmware/CMSIS/GD/{build.series}/Source"
 
 ## compile warning
@@ -25,7 +25,7 @@ compiler.elf2hex.cmd=arm-none-eabi-objcopy
 compiler.size.cmd=arm-none-eabi-size
 ##compile flags
 compiler.libraries.ldflags=
-compiler.extra_flags=-mcpu={build.mcu} {build.flags.fp} -mthumb 
+compiler.extra_flags=-mcpu={build.mcu} {build.flags.fp} -mthumb
 compiler.S.flags={compiler.extra_flags} -c -x assembler-with-cpp {compiler.gd.extra_include}
 compiler.c.flags={compiler.extra_flags} -c {build.flags.optimize} {compiler.warning_flags} -std=gnu11 -ffunction-sections -fdata-sections -g3 -T -nostdlib --param max-inline-insns-single=500 -MMD {compiler.gd.extra_include}
 compiler.cpp.flags={compiler.extra_flags} -c {build.flags.optimize} {compiler.warning_flags} -std={compiler.cpp.std} -ffunction-sections -fdata-sections -g3 -T -nostdlib -fno-threadsafe-statics --param max-inline-insns-single=500 -fno-rtti -fno-exceptions -fno-use-cxa-atexit -MMD {compiler.gd.extra_include}
@@ -55,7 +55,7 @@ build.bootloader_flags=-DVECT_TAB_OFFSET={build.flash_offset}
 build.ldscript=ldscript.ld
 
 ## Build information's flag
-build.info.flags=-D{build.series} -D{build.product_line} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} -DBOARD_NAME="{build.board}"
+build.info.flags=-D{build.series} -D{build.product_line} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} -DBOARD_NAME="{build.board}" -DARDUINO_UPLOAD_MAXIMUM_SIZE={upload.maximum_size}
 
 ## Defaults config
 build.xSerial=
@@ -141,4 +141,3 @@ tools.dfu-util.cmd=dfu-util
 tools.dfu-util.upload.params.verbose=-d
 tools.dfu-util.upload.params.quiet=
 tools.dfu-util.upload.pattern="{path}/{cmd}" --device {upload.vid}:{upload.pid} -D "{build.path}/{build.project_name}.bin" -R {upload.options}
-

tools/platformio/platformio-build.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Extended and rewritten by Maximilian Gerhardt <maximilian.gerhardt@rub.de>
-# for GD32 core. 
+# for GD32 core.
 
 """
 Arduino
@@ -218,6 +218,7 @@ def get_arduino_board_id(board_config, mcu):
         "ARDUINO_ARCH_GD32",
         "ARDUINO_%s" % board_id,
         ("BOARD_NAME", '\\"%s\\"' % board_id),
+        ("ARDUINO_UPLOAD_MAXIMUM_SIZE", board_config.get("upload.maximum_size")),
     ],
     CPPPATH=[
         join(FRAMEWORK_DIR, "cores", "arduino", "api", "deprecated"),