refactor Pio filesystem download script (arendst#20544)

* refactor FS download Pio script

pio-tools/custom_target.py

@@ -1,9 +1,11 @@
 # Written by Maximilian Gerhardt <maximilian.gerhardt@rub.de>
 # 29th December 2020
+# and Christian Baars, Johann Obermeier
+# 2023 / 2024
 # License: Apache
 # Expanded from functionality provided by PlatformIO's espressif32 and espressif8266 platforms, credited below.
-# This script provides functions to download the filesystem (SPIFFS or LittleFS) from a running ESP32 / ESP8266
-# over the serial bootloader using esptool.py, and mklittlefs / mkspiffs for extracting.
+# This script provides functions to download the filesystem (LittleFS) from a running ESP32 / ESP8266
+# over the serial bootloader using esptool.py, and mklittlefs for extracting.
 # run by either using the VSCode task "Custom" -> "Download Filesystem"
 # or by doing 'pio run -t downloadfs' (with optional '-e <environment>') from the commandline.
 # output will be saved, by default, in the "unpacked_fs" of the project.
@@ -22,10 +24,6 @@
 platform = env.PioPlatform()
 board = env.BoardConfig()
 mcu = board.get("build.mcu", "esp32")
-# Hack for using mklittlefs instead of mkspiffs -> needed since littlefs is not supported with this for ESP32
-if env["PIOPLATFORM"] == "espressif32":
-    #print("Replace MKSPIFFSTOOL with mklittlefs")
-    env.Replace( MKSPIFFSTOOL=platform.get_package_dir("tool-mklittlefs") + '/mklittlefs' )
 
 
 class FSType(Enum):
@@ -48,30 +46,22 @@ def get_extract_cmd(self, input_file, output_dir):
 
 class LittleFSInfo(FSInfo):
     def __init__(self, start, length, page_size, block_size):
-        if env["PIOPLATFORM"] == "espressif32":
-            #for ESP32: retrieve and evaluate, e.g. to mkspiffs_espressif32_arduino
-            self.tool = env.subst(env["MKSPIFFSTOOL"])
-        else:
-            self.tool = env["MKFSTOOL"] # from mkspiffs package
+        self.tool = env["MKFSTOOL"]
         self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool)
         super().__init__(FSType.LITTLEFS, start, length, page_size, block_size)
     def __repr__(self):
-        return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size} Tool: {self.tool}"
+        return f"{self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size}"
     def get_extract_cmd(self, input_file, output_dir):
-        return [self.tool, "-b", str(self.block_size), "-p", str(self.page_size), "--unpack", output_dir, input_file]
+        return [self.tool, "-b", str(self.block_size), "-s", str(self.length), "-p", str(self.page_size), "--unpack", output_dir, input_file]
 
 
 class SPIFFSInfo(FSInfo):
     def __init__(self, start, length, page_size, block_size):
-        if env["PIOPLATFORM"] == "espressif32":
-            #for ESP32: retrieve and evaluate, e.g. to mkspiffs_espressif32_arduino
-            self.tool = env.subst(env["MKSPIFFSTOOL"])
-        else:
-            self.tool = env["MKFSTOOL"] # from mkspiffs package
-        self.tool = join(platform.get_package_dir("tool-mkspiffs"), self.tool)
+        self.tool = env["MKFSTOOL"]
+        self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool)
         super().__init__(FSType.SPIFFS, start, length, page_size, block_size)
     def __repr__(self):
-        return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size} Tool: {self.tool}"
+        return f"{self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size}"
     def get_extract_cmd(self, input_file, output_dir):
         return f'"{self.tool}" -b {self.block_size} -p {self.page_size} --unpack "{output_dir}" "{input_file}"'
 
@@ -169,15 +159,15 @@ def esp8266_get_esptoolpy_reset_flags(resetmethod):
 ## Script interface functions
 def parse_partition_table(content):
     entries = [e for e in content.split(b'\xaaP') if len(e) > 0]
-    print("Partition data:")
+    #print("Partition data:")
     for entry in entries:
         type = entry[1]
         if type in [0x82,0x83]: # SPIFFS or LITTLEFS
             offset = int.from_bytes(entry[2:5], byteorder='little', signed=False)
             size = int.from_bytes(entry[6:9], byteorder='little', signed=False)
-            print("type:",hex(type))
-            print("address:",hex(offset))
-            print("size:",hex(size))
+            #print("type:",hex(type))
+            #print("address:",hex(offset))
+            #print("size:",hex(size))
             env["SPIFFS_START"] = offset
             env["SPIFFS_SIZE"] = size
             env["SPIFFS_PAGE"] = int("0x100", 16)
@@ -203,8 +193,8 @@ def get_partition_table():
             fs_file
     ]
     esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
-    print("Executing flash download command to read partition table.")
-    print(esptoolpy_cmd)
+    #print("Executing flash download command to read partition table.")
+    #print(esptoolpy_cmd)
     try:
         returncode = subprocess.call(esptoolpy_cmd, shell=False)
     except subprocess.CalledProcessError as exc:
@@ -217,14 +207,14 @@ def get_fs_type_start_and_length():
     platform = env["PIOPLATFORM"]
     if platform == "espressif32":
         print(f"Retrieving filesystem info for {mcu}.")
-        print("Partition file: " + str(env.subst("$PARTITIONS_TABLE_CSV")))
+        #print("Partition file: " + str(env.subst("$PARTITIONS_TABLE_CSV")))
         # esp32_fetch_spiffs_size(env)
         get_partition_table()
         return SPIFFSInfo(env["SPIFFS_START"], env["SPIFFS_SIZE"], env["SPIFFS_PAGE"], env["SPIFFS_BLOCK"])
     elif platform == "espressif8266":
         print("Retrieving filesystem info for ESP8266.")
-        filesystem = board.get("build.filesystem", "spiffs")
-        if filesystem not in ("spiffs", "littlefs"):
+        filesystem = board.get("build.filesystem", "littlefs")
+        if filesystem not in ("littlefs"):
             print("Unrecognized board_build.filesystem option '" + str(filesystem) + "'.")
             env.Exit(1)
         # fetching sizes is the same for all filesystems
@@ -233,10 +223,7 @@ def get_fs_type_start_and_length():
         print("FS_END: " + hex(env["FS_END"]))
         print("FS_PAGE: " + hex(env["FS_PAGE"]))
         print("FS_BLOCK: " + hex(env["FS_BLOCK"]))
-        if filesystem == "spiffs":
-            print("Recognized SPIFFS filesystem.")
-            return SPIFFSInfo(env["FS_START"], env["FS_END"] - env["FS_START"], env["FS_PAGE"], env["FS_BLOCK"])
-        elif filesystem == "littlefs":
+        if filesystem == "littlefs":
             print("Recognized LittleFS filesystem.")
             return LittleFSInfo(env["FS_START"], env["FS_END"] - env["FS_START"], env["FS_PAGE"], env["FS_BLOCK"])
         else:
@@ -264,8 +251,8 @@ def download_fs(fs_info: FSInfo):
             fs_file
     ]
     esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
-    print("Executing flash download command.")
-    print(esptoolpy_cmd)
+    print("Download filesystem image")
+    #print(esptoolpy_cmd)
     try:
         returncode = subprocess.call(esptoolpy_cmd, shell=False)
         # print("Launched download of filesystem binary.")
@@ -291,10 +278,10 @@ def unpack_fs(fs_info: FSInfo, downloaded_file: str):
         os.makedirs(unpack_dir)
 
     cmd = fs_info.get_extract_cmd(downloaded_file, unpack_dir)
-    print("Executing extraction command: " + str(cmd))
+    print("Unpack files from filesystem image")
+    #print("Extraction command: " + str(cmd))
     try:
         returncode = subprocess.call(cmd, shell=True)
-        print("Unpacked filesystem.")
         return (True, unpack_dir)
     except subprocess.CalledProcessError as exc:
         print("Unpacking filesystem failed with " + str(exc))
@@ -307,11 +294,8 @@ def display_fs(extracted_dir):
     print("Extracted " + str(file_count) + " file(s) from filesystem.")
 
 def command_download_fs(*args, **kwargs):
-    #print("Entrypoint")
-    #print(env.Dump())
     get_partition_table()
     info = get_fs_type_start_and_length()
-    print("Parsed FS info: " + str(info))
     download_ok, downloaded_file = download_fs(info)
     # print("Download was okay: " + str(download_ok) + ". File at: "+ str(downloaded_file)) # this is wrong
     unpack_ok, unpacked_dir = unpack_fs(info, downloaded_file)
@@ -358,4 +342,3 @@ def upload_factory(*args, **kwargs):
     title="Flash factory",
     description="Flash factory firmware"
 )
-