subsys/fs/littlefs: allow customization of file system configuration

There's desire to be able to customize parameters on a per-filesystem
basis, which means we need a way to override the Kconfig defaults which
are global.  This also means the littlefs data structure cannot own the
cache and lookahead buffers.

Switch to using a macro to define the littlefs data structure.  The
default version uses the Kconfig constants.  A custom one takes
arguments providing the most likely partition-specific parameters.
Finally the user is free to bypass the helper macros and set any
parameters desired, though validation is limited and only present when
CONFIG_DEBUG is enabled.

Extend the test suite with a performance module, which confirms that
these settings have an impact proportional to the log of changes to the
cache or IO sizes.

Signed-off-by: Peter A. Bigot <pab@pabigot.com>

include/fs/littlefs.h

@@ -19,19 +19,86 @@ extern "C" {
 
 /** @brief Filesystem info structure for LittleFS mount */
 struct fs_littlefs {
+	/* Defaulted in driver, customizable before mount. */
+	struct lfs_config cfg;
+
+	/* Must be cfg.cache_size */
+	u8_t *read_buffer;
+
+	/* Must be cfg.cache_size */
+	u8_t *prog_buffer;
+
+	/* Mustbe cfg.lookahead_size/4 elements, and
+	 * cfg.lookahead_size must be a multiple of 8.
+	 */
+	u32_t *lookahead_buffer[CONFIG_FS_LITTLEFS_LOOKAHEAD_SIZE / sizeof(u32_t)];
+
 	/* These structures are filled automatically at mount. */
 	struct lfs lfs;
-	struct lfs_config cfg;
 	const struct flash_area *area;
 	struct k_mutex mutex;
-
-	/* Static buffers */
-	u8_t read_buffer[CONFIG_FS_LITTLEFS_CACHE_SIZE];
-	u8_t prog_buffer[CONFIG_FS_LITTLEFS_CACHE_SIZE];
-	/* Multiple of 8 bytes, but 4-byte aligned (littlefs #239) */
-	u32_t lookahead_buffer[CONFIG_FS_LITTLEFS_LOOKAHEAD_SIZE / sizeof(u32_t)];
 };
 
+/** @brief Define a littlefs configuration with customized size
+ * characteristics.
+ *
+ * This defines static arrays required for caches, and initializes the
+ * littlefs configuration structure to use the provided values instead
+ * of the global Kconfig defaults.  A pointer to the named object must
+ * be stored in the ``.fs_data`` field of a :c:type:`struct fs_mount`
+ * object.
+ *
+ * To define an instance for the Kconfig defaults, use
+ * :cpp:func:`FS_LITTLEFS_DECLARE_DEFAULT_CONFIG`.
+ *
+ * To completely control file system configuration the application can
+ * directly define and initialize a :c:type:`struct fs_littlefs`
+ * object.  The application is responsible for ensuring the configured
+ * values are consistent with littlefs requirements.
+ *
+ * @note If you use a non-default configuration for cache size, you
+ * must also select :option:`CONFIG_FS_LITTLEFS_FC_MEM_POOL` to relax
+ * the size constraints on per-file cache allocations.
+ *
+ * @param name the name for the structure.  The defined object has
+ * file scope.
+ * @param read_sz see :option:`CONFIG_FS_LITTLEFS_READ_SIZE`
+ * @param prog_sz see :option:`CONFIG_FS_LITTLEFS_PROG_SIZE`
+ * @param cache_sz see :option:`CONFIG_FS_LITTLEFS_CACHE_SIZE`
+ * @param lookahead_sz see :option:`CONFIG_FS_LITTLEFS_LOOKAHEAD_SIZE`
+ */
+#define FS_LITTLEFS_DECLARE_CUSTOM_CONFIG(name, read_sz, prog_sz, cache_sz, lookahead_sz) \
+	static u8_t name ## _read_buffer[cache_sz];					  \
+	static u8_t name ## _prog_buffer[cache_sz];					  \
+	static u32_t name ## _lookahead_buffer[(lookahead_sz) / sizeof(u32_t)];		  \
+	static struct fs_littlefs name = {						  \
+		.cfg = {								  \
+			.read_size = (read_sz),						  \
+			.prog_size = (prog_sz),						  \
+			.cache_size = (cache_sz),					  \
+			.lookahead_size = (lookahead_sz),				  \
+			.read_buffer = name ## _read_buffer,				  \
+			.prog_buffer = name ## _prog_buffer,				  \
+			.lookahead_buffer = name ## _lookahead_buffer,			  \
+		},									  \
+	}
+
+/** @brief Define a littlefs configuration with default characteristics.
+ *
+ * This defines static arrays and initializes the littlefs
+ * configuration structure to use the default size configuration
+ * provided by Kconfig.
+ *
+ * @param name the name for the structure.  The defined object has
+ * file scope.
+ */
+#define FS_LITTLEFS_DECLARE_DEFAULT_CONFIG(name)			 \
+	FS_LITTLEFS_DECLARE_CUSTOM_CONFIG(name,				 \
+					  CONFIG_FS_LITTLEFS_READ_SIZE,	 \
+					  CONFIG_FS_LITTLEFS_PROG_SIZE,	 \
+					  CONFIG_FS_LITTLEFS_CACHE_SIZE, \
+					  CONFIG_FS_LITTLEFS_LOOKAHEAD_SIZE)
+
 #ifdef __cplusplus
 }
 #endif

samples/subsys/fs/littlefs/prj.conf

@@ -10,8 +10,10 @@
 # fs_dirent structures are big.
 CONFIG_MAIN_STACK_SIZE=2048
 
+# Let __ASSERT do its job
+CONFIG_DEBUG=y
+
 CONFIG_LOG=y
-#CONFIG_FS_LOG_LEVEL_DBG=y
 
 CONFIG_FLASH=y
 CONFIG_FLASH_MAP=y

samples/subsys/fs/littlefs/src/main.c

@@ -17,10 +17,10 @@
 /* Matches LFS_NAME_MAX */
 #define MAX_PATH_LEN 255
 
-static struct fs_littlefs lfs_storage;
+FS_LITTLEFS_DECLARE_DEFAULT_CONFIG(storage);
 static struct fs_mount_t lfs_storage_mnt = {
 	.type = FS_LITTLEFS,
-	.fs_data = &lfs_storage,
+	.fs_data = &storage,
 	.storage_dev = (void *)DT_FLASH_AREA_STORAGE_ID,
 	.mnt_point = "/lfs",
 };

subsys/fs/Kconfig

@@ -128,65 +128,9 @@ config NFFS_FILESYSTEM_MAX_BLOCK_SIZE
 
 endmenu
 
-menu "LittleFS Settings"
-	visible if FILE_SYSTEM_LITTLEFS
-
-config FS_LITTLEFS_NUM_FILES
-	int "Maximum number of opened files"
-	default 4
-	help
-	  This is a global maximum across all mounted littlefs filesystems.
-
-config FS_LITTLEFS_NUM_DIRS
-	int "Maximum number of opened directories"
-	default 4
-	help
-	  This is a global maximum across all mounted littlefs filesystems.
-
-config FS_LITTLEFS_READ_SIZE
-	int "Minimum size of a block read"
-	default 16
-	help
-	  All read operations will be a multiple of this value.
-
-config FS_LITTLEFS_PROG_SIZE
-	int "Minimum size of a block program"
-	default 16
-	help
-	  All program operations will be a multiple of this value.
-
-config FS_LITTLEFS_CACHE_SIZE
-	int "Size of block caches in bytes"
-	default 64
-	help
-	  Each cache buffers a portion of a block in RAM.  The littlefs
-	  needs a read cache, a program cache, and one additional cache
-	  per file. Larger caches can improve performance by storing
-	  more data and reducing the number of disk accesses. Must be a
-	  multiple of the read and program sizes of the underlying flash
-	  device, and a factor of the block size.
-
-config FS_LITTLEFS_LOOKAHEAD_SIZE
-	int "Size of lookahead buffer in bytes"
-	default 32
-	help
-	  A larger lookahead buffer increases the number of blocks found
-	  during an allocation pass. The lookahead buffer is stored as a
-	  compact bitmap, so each byte of RAM can track 8 blocks. Must
-	  be a multiple of 8.
-
-config FS_LITTLEFS_BLOCK_CYCLES
-	int "Number of erase cycles before moving data to another block"
-	default 512
-	help
-	  For dynamic wear leveling, the number of erase cycles before data
-	  is moved to another block.  Set to a non-positive value to
-	  disable leveling.
-
-endmenu
-
 endif # FILE_SYSTEM
 
+source "subsys/fs/Kconfig.littlefs"
 source "subsys/fs/fcb/Kconfig"
 source "subsys/fs/nvs/Kconfig"
 

subsys/fs/Kconfig.littlefs

@@ -0,0 +1,95 @@
+#
+# Copyright (c) 2019 Bolt Innovation Management, LLC
+# Copyright (c) 2019 Peter Bigot Consulting, LLC
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+menu "LittleFS Settings"
+	visible if FILE_SYSTEM_LITTLEFS
+
+config FS_LITTLEFS_NUM_FILES
+	int "Maximum number of opened files"
+	default 4
+	help
+	  This is a global maximum across all mounted littlefs filesystems.
+
+config FS_LITTLEFS_NUM_DIRS
+	int "Maximum number of opened directories"
+	default 4
+	help
+	  This is a global maximum across all mounted littlefs filesystems.
+
+config FS_LITTLEFS_READ_SIZE
+	int "Minimum size of a block read"
+	default 16
+	help
+	  All read operations will be a multiple of this value.
+
+config FS_LITTLEFS_PROG_SIZE
+	int "Minimum size of a block program"
+	default 16
+	help
+	  All program operations will be a multiple of this value.
+
+config FS_LITTLEFS_CACHE_SIZE
+	int "Size of block caches in bytes"
+	default 64
+	help
+	  Each cache buffers a portion of a block in RAM.  The littlefs
+	  needs a read cache, a program cache, and one additional cache
+	  per file. Larger caches can improve performance by storing
+	  more data and reducing the number of disk accesses. Must be a
+	  multiple of the read and program sizes of the underlying flash
+	  device, and a factor of the block size.
+
+config FS_LITTLEFS_LOOKAHEAD_SIZE
+	int "Size of lookahead buffer in bytes"
+	default 32
+	help
+	  A larger lookahead buffer increases the number of blocks found
+	  during an allocation pass. The lookahead buffer is stored as a
+	  compact bitmap, so each byte of RAM can track 8 blocks. Must
+	  be a multiple of 8.
+
+config FS_LITTLEFS_BLOCK_CYCLES
+	int "Number of erase cycles before moving data to another block"
+	default 512
+	help
+	  For dynamic wear leveling, the number of erase cycles before data
+	  is moved to another block.  Set to a non-positive value to
+	  disable leveling.
+
+menuconfig FS_LITTLEFS_FC_MEM_POOL
+	bool "Enable flexible file cache sizes for littlefs"
+	help
+	  littlefs requires a per-file buffer to cache data.  For
+	  applications that use the default configuration parameters a
+	  memory slab is reserved to support up to
+	  FS_LITTLE_FS_NUM_FILES blocks of FS_LITTLEFS_CACHE_SIZE bytes.
+
+	  When applications customize littlefs configurations and
+	  support different cache sizes for different partitions this
+	  preallocation is inadequate.
+
+	  Select this feature to enable a memory pool allocator for
+	  littlefs file caches.
+
+if FS_LITTLEFS_FC_MEM_POOL
+
+config FS_LITTLEFS_FC_MEM_POOL_MIN_SIZE
+	int "Minimum block size for littlefs file cache memory pool"
+	default 16
+
+config FS_LITTLEFS_FC_MEM_POOL_MAX_SIZE
+	int "Maximum block size for littlefs file cache memory pool"
+	default 1024
+
+config FS_LITTLEFS_FC_MEM_POOL_NUM_BLOCKS
+	int "Number of maximum sized blocks in littlefs file cache memory pool"
+	default 2
+
+endif # FS_LITTLEFS_FC_MEM_POOL
+
+endmenu
+