Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor Pio filesystem download script #20544

Merged
merged 2 commits into from Jan 19, 2024
Merged

refactor Pio filesystem download script #20544

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
38e05d9
refactor FS download Pio script
Jason2866 Jan 19, 2024
da350ca
add authors
Jason2866 Jan 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
65 changes: 24 additions & 41 deletions pio-tools/custom_target.py
View file
Open in desktop
@@ -1,9 +1,11 @@
# Written by Maximilian Gerhardt <maximilian.gerhardt@rub.de>
# 29th December 2020
# and Christian Baars, Johann Obermeier
# 2023 / 2024
# License: Apache
# Expanded from functionality provided by PlatformIO's espressif32 and espressif8266 platforms, credited below.
# This script provides functions to download the filesystem (SPIFFS or LittleFS) from a running ESP32 / ESP8266
# over the serial bootloader using esptool.py, and mklittlefs / mkspiffs for extracting.
# This script provides functions to download the filesystem (LittleFS) from a running ESP32 / ESP8266
# over the serial bootloader using esptool.py, and mklittlefs for extracting.
# run by either using the VSCode task "Custom" -> "Download Filesystem"
# or by doing 'pio run -t downloadfs' (with optional '-e <environment>') from the commandline.
# output will be saved, by default, in the "unpacked_fs" of the project.
Expand All @@ -22,10 +24,6 @@
platform = env.PioPlatform()
board = env.BoardConfig()
mcu = board.get("build.mcu", "esp32")
# Hack for using mklittlefs instead of mkspiffs -> needed since littlefs is not supported with this for ESP32
if env["PIOPLATFORM"] == "espressif32":
#print("Replace MKSPIFFSTOOL with mklittlefs")
env.Replace( MKSPIFFSTOOL=platform.get_package_dir("tool-mklittlefs") + '/mklittlefs' )


class FSType(Enum):
Expand All @@ -48,30 +46,22 @@ def get_extract_cmd(self, input_file, output_dir):

class LittleFSInfo(FSInfo):
def __init__(self, start, length, page_size, block_size):
if env["PIOPLATFORM"] == "espressif32":
#for ESP32: retrieve and evaluate, e.g. to mkspiffs_espressif32_arduino
self.tool = env.subst(env["MKSPIFFSTOOL"])
else:
self.tool = env["MKFSTOOL"] # from mkspiffs package
self.tool = env["MKFSTOOL"]
self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool)
super().__init__(FSType.LITTLEFS, start, length, page_size, block_size)
def __repr__(self):
return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size} Tool: {self.tool}"
return f"{self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size}"
def get_extract_cmd(self, input_file, output_dir):
return [self.tool, "-b", str(self.block_size), "-p", str(self.page_size), "--unpack", output_dir, input_file]
return [self.tool, "-b", str(self.block_size), "-s", str(self.length), "-p", str(self.page_size), "--unpack", output_dir, input_file]


class SPIFFSInfo(FSInfo):
def __init__(self, start, length, page_size, block_size):
if env["PIOPLATFORM"] == "espressif32":
#for ESP32: retrieve and evaluate, e.g. to mkspiffs_espressif32_arduino
self.tool = env.subst(env["MKSPIFFSTOOL"])
else:
self.tool = env["MKFSTOOL"] # from mkspiffs package
self.tool = join(platform.get_package_dir("tool-mkspiffs"), self.tool)
self.tool = env["MKFSTOOL"]
self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool)
super().__init__(FSType.SPIFFS, start, length, page_size, block_size)
def __repr__(self):
return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size} Tool: {self.tool}"
return f"{self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size}"
def get_extract_cmd(self, input_file, output_dir):
return f'"{self.tool}" -b {self.block_size} -p {self.page_size} --unpack "{output_dir}" "{input_file}"'

Expand Down Expand Up @@ -169,15 +159,15 @@ def esp8266_get_esptoolpy_reset_flags(resetmethod):
## Script interface functions
def parse_partition_table(content):
entries = [e for e in content.split(b'\xaaP') if len(e) > 0]
print("Partition data:")
#print("Partition data:")
for entry in entries:
type = entry[1]
if type in [0x82,0x83]: # SPIFFS or LITTLEFS
offset = int.from_bytes(entry[2:5], byteorder='little', signed=False)
size = int.from_bytes(entry[6:9], byteorder='little', signed=False)
print("type:",hex(type))
print("address:",hex(offset))
print("size:",hex(size))
#print("type:",hex(type))
#print("address:",hex(offset))
#print("size:",hex(size))
env["SPIFFS_START"] = offset
env["SPIFFS_SIZE"] = size
env["SPIFFS_PAGE"] = int("0x100", 16)
Expand All @@ -203,8 +193,8 @@ def get_partition_table():
fs_file
]
esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
print("Executing flash download command to read partition table.")
print(esptoolpy_cmd)
#print("Executing flash download command to read partition table.")
#print(esptoolpy_cmd)
try:
returncode = subprocess.call(esptoolpy_cmd, shell=False)
except subprocess.CalledProcessError as exc:
Expand All @@ -217,14 +207,14 @@ def get_fs_type_start_and_length():
platform = env["PIOPLATFORM"]
if platform == "espressif32":
print(f"Retrieving filesystem info for {mcu}.")
print("Partition file: " + str(env.subst("$PARTITIONS_TABLE_CSV")))
#print("Partition file: " + str(env.subst("$PARTITIONS_TABLE_CSV")))
# esp32_fetch_spiffs_size(env)
get_partition_table()
return SPIFFSInfo(env["SPIFFS_START"], env["SPIFFS_SIZE"], env["SPIFFS_PAGE"], env["SPIFFS_BLOCK"])
elif platform == "espressif8266":
print("Retrieving filesystem info for ESP8266.")
filesystem = board.get("build.filesystem", "spiffs")
if filesystem not in ("spiffs", "littlefs"):
filesystem = board.get("build.filesystem", "littlefs")
if filesystem not in ("littlefs"):
print("Unrecognized board_build.filesystem option '" + str(filesystem) + "'.")
env.Exit(1)
# fetching sizes is the same for all filesystems
Expand All @@ -233,10 +223,7 @@ def get_fs_type_start_and_length():
print("FS_END: " + hex(env["FS_END"]))
print("FS_PAGE: " + hex(env["FS_PAGE"]))
print("FS_BLOCK: " + hex(env["FS_BLOCK"]))
if filesystem == "spiffs":
print("Recognized SPIFFS filesystem.")
return SPIFFSInfo(env["FS_START"], env["FS_END"] - env["FS_START"], env["FS_PAGE"], env["FS_BLOCK"])
elif filesystem == "littlefs":
if filesystem == "littlefs":
print("Recognized LittleFS filesystem.")
return LittleFSInfo(env["FS_START"], env["FS_END"] - env["FS_START"], env["FS_PAGE"], env["FS_BLOCK"])
else:
Expand Down Expand Up @@ -264,8 +251,8 @@ def download_fs(fs_info: FSInfo):
fs_file
]
esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
print("Executing flash download command.")
print(esptoolpy_cmd)
print("Download filesystem image")
#print(esptoolpy_cmd)
try:
returncode = subprocess.call(esptoolpy_cmd, shell=False)
# print("Launched download of filesystem binary.")
Expand All @@ -291,10 +278,10 @@ def unpack_fs(fs_info: FSInfo, downloaded_file: str):
os.makedirs(unpack_dir)

cmd = fs_info.get_extract_cmd(downloaded_file, unpack_dir)
print("Executing extraction command: " + str(cmd))
print("Unpack files from filesystem image")
#print("Extraction command: " + str(cmd))
try:
returncode = subprocess.call(cmd, shell=True)
print("Unpacked filesystem.")
return (True, unpack_dir)
except subprocess.CalledProcessError as exc:
print("Unpacking filesystem failed with " + str(exc))
Expand All @@ -307,11 +294,8 @@ def display_fs(extracted_dir):
print("Extracted " + str(file_count) + " file(s) from filesystem.")

def command_download_fs(*args, **kwargs):
#print("Entrypoint")
#print(env.Dump())
get_partition_table()
info = get_fs_type_start_and_length()
print("Parsed FS info: " + str(info))
download_ok, downloaded_file = download_fs(info)
# print("Download was okay: " + str(download_ok) + ". File at: "+ str(downloaded_file)) # this is wrong
unpack_ok, unpacked_dir = unpack_fs(info, downloaded_file)
Expand Down Expand Up @@ -358,4 +342,3 @@ def upload_factory(*args, **kwargs):
title="Flash factory",
description="Flash factory firmware"
)