-
Notifications
You must be signed in to change notification settings - Fork 7k
/
utils.py
284 lines (221 loc) · 10.4 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import binascii
import os
import uuid
from datetime import datetime
from typing import List, Optional, Tuple
from construct import BitsInteger, BitStruct, Int16ul
FAT12_MAX_CLUSTERS: int = 4085
FAT16_MAX_CLUSTERS: int = 65525
RESERVED_CLUSTERS_COUNT: int = 2
PAD_CHAR: int = 0x20
FAT12: int = 12
FAT16: int = 16
FAT32: int = 32
FULL_BYTE: bytes = b'\xff'
EMPTY_BYTE: bytes = b'\x00'
# redundant
BYTES_PER_DIRECTORY_ENTRY: int = 32
UINT32_MAX: int = (1 << 32) - 1
MAX_NAME_SIZE: int = 8
MAX_EXT_SIZE: int = 3
DATETIME = Tuple[int, int, int]
FATFS_INCEPTION_YEAR: int = 1980
FATFS_INCEPTION: datetime = datetime(FATFS_INCEPTION_YEAR, 1, 1, 0, 0, 0, 0)
FATFS_MAX_HOURS = 24
FATFS_MAX_MINUTES = 60
FATFS_MAX_SECONDS = 60
FATFS_MAX_DAYS = 31
FATFS_MAX_MONTHS = 12
FATFS_MAX_YEARS = 127
FATFS_SECONDS_GRANULARITY: int = 2
# long names are encoded to two bytes in utf-16
LONG_NAMES_ENCODING: str = 'utf-16'
SHORT_NAMES_ENCODING: str = 'utf-8'
ALLOWED_SECTOR_SIZES: List[int] = [4096]
ALLOWED_SECTORS_PER_CLUSTER: List[int] = [1, 2, 4, 8, 16, 32, 64, 128]
def crc32(input_values: List[int], crc: int) -> int:
"""
Name Polynomial Reversed? Init-value XOR-out
crc32 0x104C11DB7 True 4294967295 (UINT32_MAX) 0xFFFFFFFF
"""
return binascii.crc32(bytearray(input_values), crc)
def number_of_clusters(number_of_sectors: int, sectors_per_cluster: int) -> int:
return number_of_sectors // sectors_per_cluster
def get_non_data_sectors_cnt(reserved_sectors_cnt: int, sectors_per_fat_cnt: int, root_dir_sectors_cnt: int) -> int:
return reserved_sectors_cnt + sectors_per_fat_cnt + root_dir_sectors_cnt
def get_fatfs_type(clusters_count: int) -> int:
if clusters_count < FAT12_MAX_CLUSTERS:
return FAT12
if clusters_count <= FAT16_MAX_CLUSTERS:
return FAT16
return FAT32
def get_fat_sectors_count(clusters_count: int, sector_size: int) -> int:
fatfs_type_ = get_fatfs_type(clusters_count)
if fatfs_type_ == FAT32:
raise NotImplementedError('FAT32 is not supported!')
# number of byte halves
cluster_s: int = fatfs_type_ // 4
fat_size_bytes: int = (
clusters_count * 2 + cluster_s) if fatfs_type_ == FAT16 else (clusters_count * 3 + 1) // 2 + cluster_s
return (fat_size_bytes + sector_size - 1) // sector_size
def required_clusters_count(cluster_size: int, content: bytes) -> int:
# compute number of required clusters for file text
return (len(content) + cluster_size - 1) // cluster_size
def generate_4bytes_random() -> int:
return uuid.uuid4().int & 0xFFFFFFFF
def pad_string(content: str, size: Optional[int] = None, pad: int = PAD_CHAR) -> str:
# cut string if longer and fill with pad character if shorter than size
return content.ljust(size or len(content), chr(pad))[:size]
def right_strip_string(content: str, pad: int = PAD_CHAR) -> str:
return content.rstrip(chr(pad))
def build_lfn_short_entry_name(name: str, extension: str, order: int) -> str:
return '{}{}'.format(pad_string(content=name[:MAX_NAME_SIZE - 2] + '~' + chr(order), size=MAX_NAME_SIZE),
pad_string(extension[:MAX_EXT_SIZE], size=MAX_EXT_SIZE))
def lfn_checksum(short_entry_name: str) -> int:
"""
Function defined by FAT specification. Computes checksum out of name in the short file name entry.
"""
checksum_result = 0
for i in range(MAX_NAME_SIZE + MAX_EXT_SIZE):
# operation is a right rotation on 8 bits (Python equivalent for unsigned char in C)
checksum_result = (0x80 if checksum_result & 1 else 0x00) + (checksum_result >> 1) + ord(short_entry_name[i])
checksum_result &= 0xff
return checksum_result
def convert_to_utf16_and_pad(content: str,
expected_size: int,
pad: bytes = FULL_BYTE,
terminator: bytes = b'\x00\x00') -> bytes:
# we need to get rid of the Byte order mark 0xfeff or 0xfffe, fatfs does not use it
bom_utf16: bytes = b'\xfe\xff'
encoded_content_utf16: bytes = content.encode(LONG_NAMES_ENCODING)[len(bom_utf16):]
terminated_encoded_content_utf16: bytes = (encoded_content_utf16 + terminator) if (2 * expected_size > len(
encoded_content_utf16) > 0) else encoded_content_utf16
return terminated_encoded_content_utf16.ljust(2 * expected_size, pad)
def split_to_name_and_extension(full_name: str) -> Tuple[str, str]:
name, extension = os.path.splitext(full_name)
return name, extension.replace('.', '')
def is_valid_fatfs_name(string: str) -> bool:
return string == string.upper()
def split_by_half_byte_12_bit_little_endian(value: int) -> Tuple[int, int, int]:
value_as_bytes: bytes = Int16ul.build(value)
return value_as_bytes[0] & 0x0f, value_as_bytes[0] >> 4, value_as_bytes[1] & 0x0f
def merge_by_half_byte_12_bit_little_endian(v1: int, v2: int, v3: int) -> int:
return v1 | v2 << 4 | v3 << 8
def build_byte(first_half: int, second_half: int) -> int:
return (first_half << 4) | second_half
def split_content_into_sectors(content: bytes, sector_size: int) -> List[bytes]:
result = []
clusters_cnt: int = required_clusters_count(cluster_size=sector_size, content=content)
for i in range(clusters_cnt):
result.append(content[sector_size * i:(i + 1) * sector_size])
return result
def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
parser: argparse.ArgumentParser = argparse.ArgumentParser(description=desc)
parser.add_argument('input_directory',
help='Path to the directory that will be encoded into fatfs image')
parser.add_argument('--output_file',
default='fatfs_image.img',
help='Filename of the generated fatfs image')
parser.add_argument('--partition_size',
default=FATDefaults.SIZE,
help='Size of the partition in bytes')
parser.add_argument('--sector_size',
default=FATDefaults.SECTOR_SIZE,
type=int,
choices=ALLOWED_SECTOR_SIZES,
help='Size of the partition in bytes')
parser.add_argument('--sectors_per_cluster',
default=1,
type=int,
choices=ALLOWED_SECTORS_PER_CLUSTER,
help='Number of sectors per cluster')
parser.add_argument('--root_entry_count',
default=FATDefaults.ROOT_ENTRIES_COUNT,
help='Number of entries in the root directory')
parser.add_argument('--long_name_support',
action='store_true',
help='Set flag to enable long names support.')
parser.add_argument('--use_default_datetime',
action='store_true',
help='For test purposes. If the flag is set the files are created with '
'the default timestamp that is the 1st of January 1980')
parser.add_argument('--fat_type',
default=0,
type=int,
choices=[FAT12, FAT16, 0],
help="""
Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
calculation using cluster size and partition size.
""")
args = parser.parse_args()
if args.fat_type == 0:
args.fat_type = None
args.partition_size = int(str(args.partition_size), 0)
if not os.path.isdir(args.input_directory):
raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
return args
def read_filesystem(path: str) -> bytearray:
with open(path, 'rb') as fs_file:
return bytearray(fs_file.read())
DATE_ENTRY = BitStruct(
'year' / BitsInteger(7),
'month' / BitsInteger(4),
'day' / BitsInteger(5))
TIME_ENTRY = BitStruct(
'hour' / BitsInteger(5),
'minute' / BitsInteger(6),
'second' / BitsInteger(5),
)
def build_date_entry(year: int, mon: int, mday: int) -> int:
"""
:param year: denotes year starting from 1980 (0 ~ 1980, 1 ~ 1981, etc), valid values are 1980 + 0..127 inclusive
thus theoretically 1980 - 2107
:param mon: denotes number of month of year in common order (1 ~ January, 2 ~ February, etc.),
valid values: 1..12 inclusive
:param mday: denotes number of day in month, valid values are 1..31 inclusive
:returns: 16 bit integer number (7 bits for year, 4 bits for month and 5 bits for day of the month)
"""
assert year in range(FATFS_INCEPTION_YEAR, FATFS_INCEPTION_YEAR + FATFS_MAX_YEARS)
assert mon in range(1, FATFS_MAX_MONTHS + 1)
assert mday in range(1, FATFS_MAX_DAYS + 1)
return int.from_bytes(DATE_ENTRY.build(dict(year=year - FATFS_INCEPTION_YEAR, month=mon, day=mday)), 'big')
def build_time_entry(hour: int, minute: int, sec: int) -> int:
"""
:param hour: denotes number of hour, valid values are 0..23 inclusive
:param minute: denotes minutes, valid range 0..59 inclusive
:param sec: denotes seconds with granularity 2 seconds (e.g. 1 ~ 2, 29 ~ 58), valid range 0..29 inclusive
:returns: 16 bit integer number (5 bits for hour, 6 bits for minute and 5 bits for second)
"""
assert hour in range(FATFS_MAX_HOURS)
assert minute in range(FATFS_MAX_MINUTES)
assert sec in range(FATFS_MAX_SECONDS)
return int.from_bytes(TIME_ENTRY.build(
dict(hour=hour, minute=minute, second=sec // FATFS_SECONDS_GRANULARITY)),
byteorder='big'
)
class FATDefaults:
# FATFS defaults
SIZE: int = 1024 * 1024
RESERVED_SECTORS_COUNT: int = 1
FAT_TABLES_COUNT: int = 1
SECTORS_PER_CLUSTER: int = 1
SECTOR_SIZE: int = 0x1000
HIDDEN_SECTORS: int = 0
ENTRY_SIZE: int = 32
NUM_HEADS: int = 0xff
OEM_NAME: str = 'MSDOS5.0'
SEC_PER_TRACK: int = 0x3f
VOLUME_LABEL: str = 'Espressif'
FILE_SYS_TYPE: str = 'FAT'
ROOT_ENTRIES_COUNT: int = 512 # number of entries in the root directory, recommended 512
MEDIA_TYPE: int = 0xf8
SIGNATURE_WORD: bytes = b'\x55\xAA'
# wear levelling defaults
VERSION: int = 2
TEMP_BUFFER_SIZE: int = 32
UPDATE_RATE: int = 16
WR_SIZE: int = 16
WL_SECTOR_SIZE: int = 4096