Merge branch 'feature/partition_api_new_component_2' into 'master'

Storage: Partition APIs moved to the new component 'esp_partition'

Closes IDF-1234

See merge request espressif/esp-idf!20747

.gitlab/CODEOWNERS

@@ -92,6 +92,7 @@
 /components/esp_lcd/                  @esp-idf-codeowners/peripherals
 /components/esp_local_ctrl/           @esp-idf-codeowners/app-utilities
 /components/esp_netif/                @esp-idf-codeowners/network
+/components/esp_partition/            @esp-idf-codeowners/storage
 /components/esp_phy/                  @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
 /components/esp_pm/                   @esp-idf-codeowners/power-management @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi
 /components/esp_psram/                @esp-idf-codeowners/peripherals @esp-idf-codeowners/system

.gitlab/ci/host-test.yml

@@ -441,7 +441,7 @@ test_system_cxx:
 test_partition_api_host:
   extends: .host_test_template
   script:
-    - cd ${IDF_PATH}/components/spi_flash/host_test/partition_api_test
+    - cd ${IDF_PATH}/components/esp_partition/host_test/partition_api_test
     - idf.py build
     - timeout 5 ./build/partition_api_test.elf | tee test.txt
     - grep " 0 Failures" test.txt

components/app_update/CMakeLists.txt

@@ -1,7 +1,7 @@
 idf_component_register(SRCS "esp_ota_ops.c" "esp_ota_app_desc.c"
                     INCLUDE_DIRS "include"
-                    REQUIRES spi_flash partition_table bootloader_support esp_app_format
-                    PRIV_REQUIRES esptool_py efuse)
+                    REQUIRES partition_table bootloader_support esp_app_format esp_partition
+                    PRIV_REQUIRES esptool_py efuse spi_flash)
 
 if(NOT BOOTLOADER_BUILD)
     partition_table_get_partition_info(otadata_offset "--partition-type data --partition-subtype ota" "offset")

components/app_update/esp_ota_ops.c

@@ -14,7 +14,6 @@
 
 #include "esp_err.h"
 #include "esp_partition.h"
-#include "spi_flash_mmap.h"
 #include "esp_image_format.h"
 #include "esp_secure_boot.h"
 #include "esp_flash_encrypt.h"
@@ -84,16 +83,16 @@ static const esp_partition_t *read_otadata(esp_ota_select_entry_t *two_otadata)
         return NULL;
     }
 
-    spi_flash_mmap_handle_t ota_data_map;
+    esp_partition_mmap_handle_t ota_data_map;
     const void *result = NULL;
-    esp_err_t err = esp_partition_mmap(otadata_partition, 0, otadata_partition->size, SPI_FLASH_MMAP_DATA, &result, &ota_data_map);
+    esp_err_t err = esp_partition_mmap(otadata_partition, 0, otadata_partition->size, ESP_PARTITION_MMAP_DATA, &result, &ota_data_map);
     if (err != ESP_OK) {
         ESP_LOGE(TAG, "mmap otadata filed. Err=0x%8x", err);
         return NULL;
     } else {
         memcpy(&two_otadata[0], result, sizeof(esp_ota_select_entry_t));
         memcpy(&two_otadata[1], result + SPI_FLASH_SEC_SIZE, sizeof(esp_ota_select_entry_t));
-        spi_flash_munmap(ota_data_map);
+        esp_partition_munmap(ota_data_map);
     }
     return otadata_partition;
 }

components/app_update/test_apps/main/CMakeLists.txt

@@ -1,5 +1,6 @@
 idf_component_register(SRC_DIRS "."
                        PRIV_INCLUDE_DIRS "."
-                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver
-                       WHOLE_ARCHIVE)
+                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver spi_flash
+                      WHOLE_ARCHIVE)
+
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/driver/test/CMakeLists.txt

@@ -1,7 +1,7 @@
 idf_component_register(SRC_DIRS . param_test
                        PRIV_INCLUDE_DIRS include param_test/include
                        PRIV_REQUIRES cmock test_utils driver nvs_flash
-                                     esp_timer esp_adc esp_event esp_wifi)
+                                     esp_timer esp_adc esp_event esp_wifi spi_flash)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
 
 # A local copy of idf-extra-components esp_serial_slave_link, for stabilities of the SDIO test

components/driver/test/test_spi_bus_lock.c

@@ -8,6 +8,7 @@
 #include "driver/spi_master.h"
 #include "driver/gpio.h"
 #include "esp_flash_spi_init.h"
+#include "spi_flash_mmap.h"
 
 #include "test/test_common_spi.h"
 #include "unity.h"

components/esp_hw_support/test/CMakeLists.txt

@@ -1,6 +1,6 @@
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "${include_dirs}"
-                    PRIV_REQUIRES cmock test_utils esp_hw_support driver efuse esp_timer esp_psram)
+                    PRIV_REQUIRES cmock test_utils esp_hw_support driver efuse esp_timer esp_psram spi_flash)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
 
 target_link_libraries(${COMPONENT_LIB} INTERFACE "-u ld_include_test_dport_xt_highint5")

components/esp_partition/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_partition/host_test/partition_api_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/esp_partition/CMakeLists.txt

@@ -0,0 +1,30 @@
+set(srcs "partition.c")
+set(priv_reqs esp_system bootloader_support spi_flash app_update partition_table)
+set(reqs)
+set(include_dirs "include")
+
+idf_build_get_property(target IDF_TARGET)
+if(${target} STREQUAL "linux")
+    list(APPEND srcs "partition_linux.c")
+    set(priv_reqs partition_table)
+
+    # Steal some include directories from bootloader_support and hal components:
+    idf_component_get_property(hal_dir hal COMPONENT_DIR)
+    idf_component_get_property(bootloader_support_dir bootloader_support COMPONENT_DIR)
+    list(APPEND include_dirs include ${hal_dir}/include ${bootloader_support_dir}/include)
+else()
+    list(APPEND srcs "partition_target.c")
+endif()
+
+idf_component_register(SRCS "${srcs}"
+    INCLUDE_DIRS ${include_dirs}
+    REQUIRES ${reqs}
+    PRIV_REQUIRES ${priv_reqs})
+
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    # These flags are GCC specific
+    set_property(SOURCE ${cache_srcs} APPEND_STRING PROPERTY COMPILE_FLAGS
+        " -fno-inline-small-functions -fno-inline-functions-called-once")
+endif()
+
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/esp_partition/host_test/partition_api_test/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "partition_api_test.c"
+                       REQUIRES esp_partition unity)

components/spi_flash/host_test/partition_api_test/main/partition_api_test.c → components/esp_partition/host_test/partition_api_test/main/partition_api_test.c

@@ -98,7 +98,7 @@ TEST(partition_api, test_partition_ops)
     memset(buffout, 0, sizeof(buffout));
     size_t sector_off = 0; //erase works per whole sector - offset must be aligned to 4kB boundaries
 
-    err = esp_partition_erase_range(partition_data, sector_off, SPI_FLASH_SEC_SIZE);
+    err = esp_partition_erase_range(partition_data, sector_off, partition_data->erase_size);
     assert(esp_partition_read(partition_data, off, (void *)buffout, bufsize) == ESP_OK);
     TEST_ESP_OK(err);
     TEST_ASSERT_EQUAL(0, memcmp(buffout, buferase, bufsize));

components/spi_flash/include/esp_partition.h → components/esp_partition/include/esp_partition.h

@@ -11,8 +11,6 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include "esp_err.h"
-#include "esp_flash.h"
-#include "spi_flash_mmap.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -23,6 +21,22 @@ extern "C" {
  * @brief Partition APIs
  */
 
+/** @cond */
+typedef struct esp_flash_t esp_flash_t;
+/** @endcond */
+
+/**
+ * @brief Enumeration which specifies memory space requested in an mmap call
+ */
+typedef enum {
+    ESP_PARTITION_MMAP_DATA,    /**< map to data memory (Vaddr0), allows byte-aligned access, 4 MB total */
+    ESP_PARTITION_MMAP_INST,    /**< map to instruction memory (Vaddr1-3), allows only 4-byte-aligned access, 11 MB total */
+} esp_partition_mmap_memory_t;
+
+/**
+ * @brief Opaque handle for memory region obtained from esp_partition_mmap.
+ */
+typedef uint32_t esp_partition_mmap_handle_t;
 
 /**
  * @brief Partition type
@@ -114,6 +128,7 @@ typedef struct {
     esp_partition_subtype_t subtype;    /*!< partition subtype */
     uint32_t address;                   /*!< starting address of the partition in flash */
     uint32_t size;                      /*!< size of the partition, in bytes */
+    uint32_t erase_size;                /*!< size the erase operation should be aligned to */
     char label[17];                     /*!< partition label, zero-terminated ASCII string */
     bool encrypted;                     /*!< flag is set to true if partition is encrypted */
 } esp_partition_t;
@@ -318,9 +333,9 @@ esp_err_t esp_partition_write_raw(const esp_partition_t* partition,
  *                  esp_partition_find_first or esp_partition_get.
  *                  Must be non-NULL.
  * @param offset Offset from the beginning of partition where erase operation
- *               should start. Must be aligned to 4 kilobytes.
+ *               should start. Must be aligned to partition->erase_size.
  * @param size Size of the range which should be erased, in bytes.
- *                   Must be divisible by 4 kilobytes.
+ *                   Must be divisible by partition->erase_size.
  *
  * @return ESP_OK, if the range was erased successfully;
  *         ESP_ERR_INVALID_ARG, if iterator or dst are NULL;
@@ -342,7 +357,7 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
  * requested offset (not necessarily to the beginning of mmap-ed region).
  *
  * To release mapped memory, pass handle returned via out_handle argument to
- * spi_flash_munmap function.
+ * esp_partition_munmap function.
  *
  * @param partition Pointer to partition structure obtained using
  *                  esp_partition_find_first or esp_partition_get.
@@ -351,13 +366,25 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
  * @param size Size of the area to be mapped.
  * @param memory  Memory space where the region should be mapped
  * @param out_ptr  Output, pointer to the mapped memory region
- * @param out_handle  Output, handle which should be used for spi_flash_munmap call
+ * @param out_handle  Output, handle which should be used for esp_partition_munmap call
  *
  * @return ESP_OK, if successful
  */
 esp_err_t esp_partition_mmap(const esp_partition_t* partition, size_t offset, size_t size,
-                             spi_flash_mmap_memory_t memory,
-                             const void** out_ptr, spi_flash_mmap_handle_t* out_handle);
+                             esp_partition_mmap_memory_t memory,
+                             const void** out_ptr, esp_partition_mmap_handle_t* out_handle);
+
+/**
+ * @brief Release region previously obtained using esp_partition_mmap
+ *
+ * @note Calling this function will not necessarily unmap memory region.
+ *       Region will only be unmapped when there are no other handles which
+ *       reference this region. In case of partially overlapping regions
+ *       it is possible that memory will be unmapped partially.
+ *
+ * @param handle  Handle obtained from spi_flash_mmap
+ */
+void esp_partition_munmap(esp_partition_mmap_handle_t handle);
 
 /**
  * @brief Get SHA-256 digest for required partition.

components/spi_flash/include/esp_private/partition_linux.h → components/esp_partition/include/esp_private/partition_linux.h

@@ -19,6 +19,9 @@ extern "C" {
  * @brief Private API functions used for Linux-target emulation of the Partition APIs (host-side testing)
  */
 
+/** @brief emulated sector size for the partition API on Linux */
+#define ESP_PARTITION_EMULATED_SECTOR_SIZE 0x1000
+
 /**
  * @brief Partition type to string conversion routine
  *

components/spi_flash/partition.c → components/esp_partition/partition.c

@@ -12,10 +12,10 @@
 #include "sdkconfig.h"
 #include "esp_flash_partitions.h"
 #include "esp_attr.h"
-#include "esp_flash.h"
 #include "esp_partition.h"
 
 #if !CONFIG_IDF_TARGET_LINUX
+#include "esp_flash.h"
 #include "esp_flash_encrypt.h"
 #endif
 
@@ -90,9 +90,11 @@ static esp_err_t load_partitions(void)
 
 #if CONFIG_IDF_TARGET_LINUX
     esp_err_t err = esp_partition_file_mmap(&p_start);
+    size_t mapped_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
 #else
     esp_err_t err = spi_flash_mmap(partition_align_pg_size,
                                    SPI_FLASH_SEC_SIZE, SPI_FLASH_MMAP_DATA, (const void **)&p_start, &handle);
+    size_t mapped_size = SPI_FLASH_SEC_SIZE;
 #endif
 
     if (err != ESP_OK) {
@@ -101,7 +103,7 @@ static esp_err_t load_partitions(void)
 
     // calculate partition address within mmap-ed region
     p_start += partition_pad;
-    p_end = p_start + SPI_FLASH_SEC_SIZE;
+    p_end = p_start + mapped_size;
 
     for (const uint8_t *p_entry = p_start; p_entry < p_end; p_entry += sizeof(esp_partition_info_t)) {
         esp_partition_info_t entry;
@@ -136,6 +138,11 @@ static esp_err_t load_partitions(void)
 #endif
         item->info.address = entry.pos.offset;
         item->info.size = entry.pos.size;
+#if CONFIG_IDF_TARGET_LINUX
+        item->info.erase_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
+#else
+        item->info.erase_size = SPI_FLASH_SEC_SIZE;
+#endif
         item->info.type = entry.type;
         item->info.subtype = entry.subtype;
         item->info.encrypted = entry.flags & PART_FLAG_ENCRYPTED;
@@ -363,9 +370,14 @@ esp_err_t esp_partition_register_external(esp_flash_t *flash_chip, size_t offset
         *out_partition = NULL;
     }
 
+#if CONFIG_IDF_TARGET_LINUX
+    return ESP_ERR_NOT_SUPPORTED;
+
+#else
     if (offset + size > flash_chip->size) {
         return ESP_ERR_INVALID_SIZE;
     }
+#endif // CONFIG_IDF_TARGET_LINUX
 
     esp_err_t err = ensure_partitions_loaded();
     if (err != ESP_OK) {

components/spi_flash/partition_linux.c → components/esp_partition/partition_linux.c

@@ -238,10 +238,10 @@ esp_err_t esp_partition_erase_range(const esp_partition_t *partition, size_t off
 {
     assert(partition != NULL);
 
-    if (offset > partition->size || offset % SPI_FLASH_SEC_SIZE != 0) {
+    if (offset > partition->size || offset % partition->erase_size != 0) {
         return ESP_ERR_INVALID_ARG;
     }
-    if (offset + size > partition->size || size % SPI_FLASH_SEC_SIZE != 0) {
+    if (offset + size > partition->size || size % partition->erase_size != 0) {
         return ESP_ERR_INVALID_SIZE;
     }
 

components/spi_flash/partition_target.c → components/esp_partition/partition_target.c

@@ -143,8 +143,8 @@ esp_err_t esp_partition_erase_range(const esp_partition_t *partition,
  * mmaped pointers, and a single handle for all these regions.
  */
 esp_err_t esp_partition_mmap(const esp_partition_t *partition, size_t offset, size_t size,
-                             spi_flash_mmap_memory_t memory,
-                             const void **out_ptr, spi_flash_mmap_handle_t *out_handle)
+                             esp_partition_mmap_memory_t memory,
+                             const void **out_ptr, esp_partition_mmap_handle_t *out_handle)
 {
     assert(partition != NULL);
     if (offset > partition->size) {
@@ -160,14 +160,19 @@ esp_err_t esp_partition_mmap(const esp_partition_t *partition, size_t offset, si
     // offset within mmu page size block
     size_t region_offset = phys_addr & (CONFIG_MMU_PAGE_SIZE - 1);
     size_t mmap_addr = phys_addr & ~(CONFIG_MMU_PAGE_SIZE - 1);
-    esp_err_t rc = spi_flash_mmap(mmap_addr, size + region_offset, memory, out_ptr, out_handle);
+    esp_err_t rc = spi_flash_mmap(mmap_addr, size + region_offset, (spi_flash_mmap_memory_t) memory, out_ptr, (spi_flash_mmap_handle_t*) out_handle);
     // adjust returned pointer to point to the correct offset
     if (rc == ESP_OK) {
         *out_ptr = (void *) (((ptrdiff_t) * out_ptr) + region_offset);
     }
     return rc;
 }
 
+void esp_partition_munmap(esp_partition_mmap_handle_t handle)
+{
+    spi_flash_munmap((spi_flash_mmap_handle_t) handle);
+}
+
 esp_err_t esp_partition_get_sha256(const esp_partition_t *partition, uint8_t *sha_256)
 {
     return bootloader_common_get_sha256_of_partition(partition->address, partition->size, partition->type, sha_256);

components/esp_partition/test/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(SRC_DIRS "."
+                       PRIV_INCLUDE_DIRS "."
+                       PRIV_REQUIRES test_utils esp_partition esp_system app_update bootloader_support spi_flash)
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/partition_table/test/test_partition.c → components/esp_partition/test/test_partition.c

@@ -49,7 +49,7 @@ TEST_CASE("Can write, read, mmap partition", "[partition][ignore]")
     const esp_partition_t *p = get_test_data_partition();
     printf("Using partition %s at 0x%x, size 0x%x\n", p->label, p->address, p->size);
     TEST_ASSERT_NOT_NULL(p);
-    const size_t max_size = 2 * SPI_FLASH_SEC_SIZE;
+    const size_t max_size = 2 * p->erase_size;
     uint8_t *data = (uint8_t *) malloc(max_size);
     TEST_ASSERT_NOT_NULL(data);
 
@@ -85,10 +85,10 @@ TEST_CASE("Can write, read, mmap partition", "[partition][ignore]")
     free(data);
 
     const uint32_t *mmap_data;
-    spi_flash_mmap_handle_t mmap_handle;
+    esp_partition_mmap_handle_t mmap_handle;
     size_t begin = 3000;
     size_t size = 64000; //chosen so size is smaller than 64K but the mmap straddles 2 MMU blocks
-    TEST_ASSERT_EQUAL(ESP_OK, esp_partition_mmap(p, begin, size, SPI_FLASH_MMAP_DATA,
+    TEST_ASSERT_EQUAL(ESP_OK, esp_partition_mmap(p, begin, size, ESP_PARTITION_MMAP_DATA,
                       (const void **)&mmap_data, &mmap_handle));
     srand(0);
     for (size_t offset = 0; offset < p->size; offset += block_size) {
@@ -107,5 +107,5 @@ TEST_CASE("Can write, read, mmap partition", "[partition][ignore]")
         }
     }
 
-    spi_flash_munmap(mmap_handle);
+    esp_partition_munmap(mmap_handle);
 }

components/spi_flash/test/test_partition_ext.c → components/esp_partition/test/test_partition_ext.c

@@ -1,4 +1,10 @@
+/*
+ * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
 #include "esp_flash.h"
+#include "spi_flash_mmap.h"
 #include "esp_partition.h"
 #include "unity.h"
 

components/spi_flash/test/test_partitions.c → components/esp_partition/test/test_partitions.c

@@ -18,6 +18,8 @@
 #include <esp_log.h>
 #include <esp_partition.h>
 #include <esp_attr.h>
+#include "esp_flash.h"
+#include "spi_flash_mmap.h"
 
 TEST_CASE("Test erase partition", "[spi_flash][esp_flash]")
 {