Skip to content

Commit

Permalink
Merge branch 'bugfix/fatfs_cannot_work_after_multiple_operations' int…
Browse files Browse the repository at this point in the history
…o 'master'

fix(ESPAT-1655): Fixed that AT+FS might return wrong data after multiple write

See merge request application/esp-at!1537
  • Loading branch information
xcguang committed Mar 20, 2024
2 parents 4a6ee01 + ebdb5b5 commit 195fa22
Show file tree
Hide file tree
Showing 23 changed files with 1,395 additions and 1 deletion.
2 changes: 1 addition & 1 deletion components/customized_partitions/generation_tools/fatfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def main():
fat_type = 16
continue

fatfs_param = '--sector_size {} {} --sectors_per_cluster {} --fat_type {}'.format(
fatfs_param = '--sector_size {} {} --sectors_per_cluster {} --fat_type {} --wl_mode "safe"'.format(
sector_size, long_name_support, sectors_per_cluster, fat_type)

if sys.platform == 'win32':
Expand Down
123 changes: 123 additions & 0 deletions module_config/module_esp32-d2wd/patch/fatfs_generation.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
diff --git a/components/fatfs/fatfs_utils/utils.py b/components/fatfs/fatfs_utils/utils.py
index e7d2e25a7f..ddc68c9225 100644
--- a/components/fatfs/fatfs_utils/utils.py
+++ b/components/fatfs/fatfs_utils/utils.py
@@ -1,14 +1,17 @@
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
-
import argparse
import binascii
import os
import uuid
from datetime import datetime
-from typing import List, Optional, Tuple
+from typing import List
+from typing import Optional
+from typing import Tuple

-from construct import BitsInteger, BitStruct, Int16ul
+from construct import BitsInteger
+from construct import BitStruct
+from construct import Int16ul

FAT12_MAX_CLUSTERS: int = 4085
FAT16_MAX_CLUSTERS: int = 65525
@@ -200,6 +203,11 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
calculation using cluster size and partition size.
""")
+ parser.add_argument('--wl_mode',
+ default=None,
+ type=str,
+ choices=['safe', 'perf'],
+ help='Wear levelling mode to use. Safe or performance. Only for sector size of 512')

args = parser.parse_args()
if args.fat_type == 0:
@@ -207,6 +215,9 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
args.partition_size = int(str(args.partition_size), 0)
if not os.path.isdir(args.input_directory):
raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
+ if args.wl_mode is not None:
+ if args.sector_size != 512:
+ raise ValueError('Wear levelling mode can be set only for sector size 512')
return args


diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py
index b6c5dd4fe7..8881938f03 100755
--- a/components/fatfs/wl_fatfsgen.py
+++ b/components/fatfs/wl_fatfsgen.py
@@ -1,13 +1,19 @@
#!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
+from typing import List
+from typing import Optional

-from typing import List, Optional
-
-from construct import Const, Int32ul, Struct
+from construct import Const
+from construct import Int32ul
+from construct import Struct
from fatfs_utils.exceptions import WLNotInitialized
-from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, generate_4bytes_random,
- get_args_for_partition_generator)
+from fatfs_utils.utils import crc32
+from fatfs_utils.utils import FATDefaults
+from fatfs_utils.utils import FULL_BYTE
+from fatfs_utils.utils import generate_4bytes_random
+from fatfs_utils.utils import get_args_for_partition_generator
+from fatfs_utils.utils import UINT32_MAX
from fatfsgen import FATFS


@@ -55,6 +61,7 @@ class WLFATFS:
WL_STATE_HEADER_SIZE = 64
WL_STATE_COPY_COUNT = 2 # always 2 copies for power failure safety
WL_SECTOR_SIZE = 0x1000
+ WL_SAFE_MODE_DUMP_SECTORS = 2

WL_STATE_T_DATA = Struct(
'pos' / Int32ul,
@@ -99,7 +106,8 @@ class WLFATFS:
temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE,
device_id: int = None,
root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
- media_type: int = FATDefaults.MEDIA_TYPE) -> None:
+ media_type: int = FATDefaults.MEDIA_TYPE,
+ wl_mode: Optional[str] = None) -> None:
self._initialized = False
self._version = version
self._temp_buff_size = temp_buff_size
@@ -107,6 +115,7 @@ class WLFATFS:
self.partition_size = size
self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE
self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
+ self.wl_mode = wl_mode

# determine the number of required sectors (roundup to sector size)
self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
@@ -116,6 +125,9 @@ class WLFATFS:

wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
+ if self.wl_mode is not None and self.wl_mode == 'safe':
+ wl_sectors += WLFATFS.WL_SAFE_MODE_DUMP_SECTORS
+
self.plain_fat_sectors = self.total_sectors - wl_sectors
self.plain_fatfs = FATFS(
explicit_fat_type=explicit_fat_type,
@@ -226,7 +238,8 @@ if __name__ == '__main__':
root_entry_count=args.root_entry_count,
explicit_fat_type=args.fat_type,
long_names_enabled=args.long_name_support,
- use_default_datetime=args.use_default_datetime)
+ use_default_datetime=args.use_default_datetime,
+ wl_mode=args.wl_mode)

wl_fatfs.wl_generate(args.input_directory)
wl_fatfs.init_wl()
4 changes: 4 additions & 0 deletions module_config/module_esp32-d2wd/patch/patch_list.ini
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,7 @@
[esp_tls_parse_ecc_key.patch]
path = esp-idf
note = "Fixed an issue where esp-tls is unable to parse ECC keys"

[fatfs_generation.patch]
path = esp-idf
note = "Fixed an issue where data read after writing fatfs partition more than 63 times might be incorrect"
123 changes: 123 additions & 0 deletions module_config/module_esp32-sdio/patch/fatfs_generation.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
diff --git a/components/fatfs/fatfs_utils/utils.py b/components/fatfs/fatfs_utils/utils.py
index e7d2e25a7f..ddc68c9225 100644
--- a/components/fatfs/fatfs_utils/utils.py
+++ b/components/fatfs/fatfs_utils/utils.py
@@ -1,14 +1,17 @@
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
-
import argparse
import binascii
import os
import uuid
from datetime import datetime
-from typing import List, Optional, Tuple
+from typing import List
+from typing import Optional
+from typing import Tuple

-from construct import BitsInteger, BitStruct, Int16ul
+from construct import BitsInteger
+from construct import BitStruct
+from construct import Int16ul

FAT12_MAX_CLUSTERS: int = 4085
FAT16_MAX_CLUSTERS: int = 65525
@@ -200,6 +203,11 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
calculation using cluster size and partition size.
""")
+ parser.add_argument('--wl_mode',
+ default=None,
+ type=str,
+ choices=['safe', 'perf'],
+ help='Wear levelling mode to use. Safe or performance. Only for sector size of 512')

args = parser.parse_args()
if args.fat_type == 0:
@@ -207,6 +215,9 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
args.partition_size = int(str(args.partition_size), 0)
if not os.path.isdir(args.input_directory):
raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
+ if args.wl_mode is not None:
+ if args.sector_size != 512:
+ raise ValueError('Wear levelling mode can be set only for sector size 512')
return args


diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py
index b6c5dd4fe7..8881938f03 100755
--- a/components/fatfs/wl_fatfsgen.py
+++ b/components/fatfs/wl_fatfsgen.py
@@ -1,13 +1,19 @@
#!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
+from typing import List
+from typing import Optional

-from typing import List, Optional
-
-from construct import Const, Int32ul, Struct
+from construct import Const
+from construct import Int32ul
+from construct import Struct
from fatfs_utils.exceptions import WLNotInitialized
-from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, generate_4bytes_random,
- get_args_for_partition_generator)
+from fatfs_utils.utils import crc32
+from fatfs_utils.utils import FATDefaults
+from fatfs_utils.utils import FULL_BYTE
+from fatfs_utils.utils import generate_4bytes_random
+from fatfs_utils.utils import get_args_for_partition_generator
+from fatfs_utils.utils import UINT32_MAX
from fatfsgen import FATFS


@@ -55,6 +61,7 @@ class WLFATFS:
WL_STATE_HEADER_SIZE = 64
WL_STATE_COPY_COUNT = 2 # always 2 copies for power failure safety
WL_SECTOR_SIZE = 0x1000
+ WL_SAFE_MODE_DUMP_SECTORS = 2

WL_STATE_T_DATA = Struct(
'pos' / Int32ul,
@@ -99,7 +106,8 @@ class WLFATFS:
temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE,
device_id: int = None,
root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
- media_type: int = FATDefaults.MEDIA_TYPE) -> None:
+ media_type: int = FATDefaults.MEDIA_TYPE,
+ wl_mode: Optional[str] = None) -> None:
self._initialized = False
self._version = version
self._temp_buff_size = temp_buff_size
@@ -107,6 +115,7 @@ class WLFATFS:
self.partition_size = size
self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE
self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
+ self.wl_mode = wl_mode

# determine the number of required sectors (roundup to sector size)
self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
@@ -116,6 +125,9 @@ class WLFATFS:

wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
+ if self.wl_mode is not None and self.wl_mode == 'safe':
+ wl_sectors += WLFATFS.WL_SAFE_MODE_DUMP_SECTORS
+
self.plain_fat_sectors = self.total_sectors - wl_sectors
self.plain_fatfs = FATFS(
explicit_fat_type=explicit_fat_type,
@@ -226,7 +238,8 @@ if __name__ == '__main__':
root_entry_count=args.root_entry_count,
explicit_fat_type=args.fat_type,
long_names_enabled=args.long_name_support,
- use_default_datetime=args.use_default_datetime)
+ use_default_datetime=args.use_default_datetime,
+ wl_mode=args.wl_mode)

wl_fatfs.wl_generate(args.input_directory)
wl_fatfs.init_wl()
4 changes: 4 additions & 0 deletions module_config/module_esp32-sdio/patch/patch_list.ini
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,7 @@
[esp_tls_parse_ecc_key.patch]
path = esp-idf
note = "Fixed an issue where esp-tls is unable to parse ECC keys"

[fatfs_generation.patch]
path = esp-idf
note = "Fixed an issue where data read after writing fatfs partition more than 63 times might be incorrect"
123 changes: 123 additions & 0 deletions module_config/module_esp32_default/patch/fatfs_generation.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
diff --git a/components/fatfs/fatfs_utils/utils.py b/components/fatfs/fatfs_utils/utils.py
index e7d2e25a7f..ddc68c9225 100644
--- a/components/fatfs/fatfs_utils/utils.py
+++ b/components/fatfs/fatfs_utils/utils.py
@@ -1,14 +1,17 @@
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
-
import argparse
import binascii
import os
import uuid
from datetime import datetime
-from typing import List, Optional, Tuple
+from typing import List
+from typing import Optional
+from typing import Tuple

-from construct import BitsInteger, BitStruct, Int16ul
+from construct import BitsInteger
+from construct import BitStruct
+from construct import Int16ul

FAT12_MAX_CLUSTERS: int = 4085
FAT16_MAX_CLUSTERS: int = 65525
@@ -200,6 +203,11 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
calculation using cluster size and partition size.
""")
+ parser.add_argument('--wl_mode',
+ default=None,
+ type=str,
+ choices=['safe', 'perf'],
+ help='Wear levelling mode to use. Safe or performance. Only for sector size of 512')

args = parser.parse_args()
if args.fat_type == 0:
@@ -207,6 +215,9 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
args.partition_size = int(str(args.partition_size), 0)
if not os.path.isdir(args.input_directory):
raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
+ if args.wl_mode is not None:
+ if args.sector_size != 512:
+ raise ValueError('Wear levelling mode can be set only for sector size 512')
return args


diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py
index b6c5dd4fe7..8881938f03 100755
--- a/components/fatfs/wl_fatfsgen.py
+++ b/components/fatfs/wl_fatfsgen.py
@@ -1,13 +1,19 @@
#!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
+from typing import List
+from typing import Optional

-from typing import List, Optional
-
-from construct import Const, Int32ul, Struct
+from construct import Const
+from construct import Int32ul
+from construct import Struct
from fatfs_utils.exceptions import WLNotInitialized
-from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, generate_4bytes_random,
- get_args_for_partition_generator)
+from fatfs_utils.utils import crc32
+from fatfs_utils.utils import FATDefaults
+from fatfs_utils.utils import FULL_BYTE
+from fatfs_utils.utils import generate_4bytes_random
+from fatfs_utils.utils import get_args_for_partition_generator
+from fatfs_utils.utils import UINT32_MAX
from fatfsgen import FATFS


@@ -55,6 +61,7 @@ class WLFATFS:
WL_STATE_HEADER_SIZE = 64
WL_STATE_COPY_COUNT = 2 # always 2 copies for power failure safety
WL_SECTOR_SIZE = 0x1000
+ WL_SAFE_MODE_DUMP_SECTORS = 2

WL_STATE_T_DATA = Struct(
'pos' / Int32ul,
@@ -99,7 +106,8 @@ class WLFATFS:
temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE,
device_id: int = None,
root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
- media_type: int = FATDefaults.MEDIA_TYPE) -> None:
+ media_type: int = FATDefaults.MEDIA_TYPE,
+ wl_mode: Optional[str] = None) -> None:
self._initialized = False
self._version = version
self._temp_buff_size = temp_buff_size
@@ -107,6 +115,7 @@ class WLFATFS:
self.partition_size = size
self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE
self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
+ self.wl_mode = wl_mode

# determine the number of required sectors (roundup to sector size)
self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
@@ -116,6 +125,9 @@ class WLFATFS:

wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
+ if self.wl_mode is not None and self.wl_mode == 'safe':
+ wl_sectors += WLFATFS.WL_SAFE_MODE_DUMP_SECTORS
+
self.plain_fat_sectors = self.total_sectors - wl_sectors
self.plain_fatfs = FATFS(
explicit_fat_type=explicit_fat_type,
@@ -226,7 +238,8 @@ if __name__ == '__main__':
root_entry_count=args.root_entry_count,
explicit_fat_type=args.fat_type,
long_names_enabled=args.long_name_support,
- use_default_datetime=args.use_default_datetime)
+ use_default_datetime=args.use_default_datetime,
+ wl_mode=args.wl_mode)

wl_fatfs.wl_generate(args.input_directory)
wl_fatfs.init_wl()
Loading

0 comments on commit 195fa22

Please sign in to comment.