FS speedup by factor > 20 (playlist generation, Web-UI file listing)

platformio.ini

@@ -30,6 +30,8 @@ extra_scripts =
     pre:gitVersion.py
     pre:processHtml.py
 lib_deps =
+	https://github.com/tueddy/FS.git#2.0.7  ; will be obsolete with Arduino >= 2.0.8
+	; https://github.com/tueddy/FS.git#1.0.6
 	https://github.com/schreibfaul1/ESP32-audioI2S.git
 	https://github.com/madhephaestus/ESP32Encoder.git#61aef9c
 	https://github.com/knolleary/pubsubclient.git#2d228f2
@@ -43,7 +45,7 @@ lib_deps =
 	https://github.com/kkloesener/MFRC522_I2C.git#121a27e
 	https://github.com/miguelbalboa/rfid.git#ba72b92
 	https://github.com/tuniii/LogRingBuffer.git#89d7d3e
-	https://github.com/tueddy/PN5180-Library.git#f776e66
+	https://github.com/tueddy/PN5180-Library.git#d218630
 	https://github.com/SZenglein/Arduino-MAX17055_Driver#a0a5418
 platform_packages =
     ;platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git#1.0.6

src/SdCard.cpp

@@ -133,6 +133,8 @@ bool fileValid(const char *_fileItem) {
 
 // Takes a directory as input and returns a random subdirectory from it
 char *SdCard_pickRandomSubdirectory(char *_directory) {
+	uint32_t listStartTimestamp = millis();
+
 	// Look if file/folder requested really exists. If not => break.
 	File directory = gFSystem.open(_directory);
 	if (!directory) {
@@ -155,18 +157,31 @@ char *SdCard_pickRandomSubdirectory(char *_directory) {
 
 	// Create linear list of subdirectories with #-delimiters
 	while (true) {
-		File fileItem = directory.openNextFile();
-		if (!fileItem) {
-			break;
-		}
-		if (!fileItem.isDirectory()) {
-			continue;
-		} else {
+		#if defined(HAS_FILEEXPLORER_SPEEDUP) || (ESP_ARDUINO_VERSION >= ESP_ARDUINO_VERSION_VAL(2, 0, 8))
+			bool isDir = false;
+			String MyfileName = directory.getNextFileName(&isDir);
+			if (MyfileName == "") {
+				break;
+			}
+			if (!isDir) {
+				continue;
+			} else {
+				strncpy(buffer, MyfileName.c_str(), 255);
+		#else
+			File fileItem = directory.openNextFile();
+			if (!fileItem) {
+				break;
+			}
+			if (!fileItem.isDirectory()) {
+				continue;
+			} else {
 			#if ESP_ARDUINO_VERSION_MAJOR >= 2
 				strncpy(buffer, (char *) fileItem.path(), 255);
 			#else
 				strncpy(buffer, (char *) fileItem.name(), 255);
 			#endif
+		#endif
+
 			// Log_Printf(LOGLEVEL_INFO, nameOfFileFound, buffer);
 			if ((strlen(subdirectoryList) + strlen(buffer) + 2) >= allocCount * allocSize) {
 				char *tmp = (char *) realloc(subdirectoryList, ++allocCount * allocSize);
@@ -215,6 +230,7 @@ char *SdCard_pickRandomSubdirectory(char *_directory) {
 	}
 
 	free(subdirectoryList);
+	Log_Printf(LOGLEVEL_DEBUG, "pick random directory from SD-card finished: %lu ms", (millis() - listStartTimestamp));
 	return NULL;
 }
 
@@ -229,6 +245,7 @@ char **SdCard_ReturnPlaylist(const char *fileName, const uint32_t _playMode) {
 	bool readFromCacheFile = false;
 	bool enablePlaylistCaching = false;
 	bool enablePlaylistFromM3u = false;
+	uint32_t listStartTimestamp = millis();
 
 	if (files != NULL) { // If **ptr already exists, de-allocate its memory
 		Log_Println((char *) FPSTR(releaseMemoryOfOldPlaylist), LOGLEVEL_DEBUG);
@@ -318,7 +335,7 @@ char **SdCard_ReturnPlaylist(const char *fileName, const uint32_t _playMode) {
 					// skip M3U comment lines starting with #
 					fileOrDirectory.readStringUntil('\n');
 					continue;
-				}
+				}				
 				if (fPos+1 >= allocCount * allocSize) {
 					char *tmp = (char *) realloc(serializedPlaylist, ++allocCount * allocSize);
 					Log_Println((char *) FPSTR(reallocCalled), LOGLEVEL_DEBUG);
@@ -388,19 +405,31 @@ char **SdCard_ReturnPlaylist(const char *fileName, const uint32_t _playMode) {
 		}
 
 		while (true) {
-			File fileItem = fileOrDirectory.openNextFile();
-			if (!fileItem) {
-				break;
-			}
-			if (fileItem.isDirectory()) {
-				continue;
-			} else {
+			#if defined(HAS_FILEEXPLORER_SPEEDUP) || (ESP_ARDUINO_VERSION >= ESP_ARDUINO_VERSION_VAL(2, 0, 8))
+				bool isDir = false;
+				String MyfileName = fileOrDirectory.getNextFileName(&isDir);
+				if (MyfileName == "") {
+					break;
+				}
+				if (isDir) {
+					continue;
+				} else {
+					strncpy(fileNameBuf, MyfileName.c_str(), sizeof(fileNameBuf) / sizeof(fileNameBuf[0]));
+			#else
+				File fileItem = fileOrDirectory.openNextFile();
+				if (!fileItem) {
+					break;
+				}
+				if (fileItem.isDirectory()) {
+					continue;
+				} else {
 				#if ESP_ARDUINO_VERSION_MAJOR >= 2
 					strncpy(fileNameBuf, (char *) fileItem.path(), sizeof(fileNameBuf) / sizeof(fileNameBuf[0]));
 				#else
 					strncpy(fileNameBuf, (char *) fileItem.name(), sizeof(fileNameBuf) / sizeof(fileNameBuf[0]));
 				#endif
-				// Don't support filenames that start with "." and only allow .mp3 and other supported audio formats
+			#endif
+				// Don't support filenames that start with "." and only allow .mp3 and other supported audio file formats
 				if (fileValid(fileNameBuf)) {
 					// Log_Printf(LOGLEVEL_INFO, "%s: %s", (char *) FPSTR(nameOfFileFound), fileNameBuf);
 					if ((strlen(serializedPlaylist) + strlen(fileNameBuf) + 2) >= allocCount * allocSize) {
@@ -467,6 +496,7 @@ char **SdCard_ReturnPlaylist(const char *fileName, const uint32_t _playMode) {
 	}
 	snprintf(files[0], 5,  "%u", cnt);
 	Log_Printf(LOGLEVEL_NOTICE, numberOfValidFiles, cnt);
+	Log_Printf(LOGLEVEL_DEBUG, "build playlist from SD-card finished: %lu ms", (millis() - listStartTimestamp));
 
 	return &(files[1]); // return ptr+1 (starting at 1st payload-item); ptr+0 contains number of items
 }

src/Web.cpp

@@ -925,39 +925,56 @@ void explorerHandleListRequest(AsyncWebServerRequest *request) {
 		return;
 	}
 
-	File file = root.openNextFile();
+	#if defined(HAS_FILEEXPLORER_SPEEDUP) || (ESP_ARDUINO_VERSION >= ESP_ARDUINO_VERSION_VAL(2, 0, 8))
+		bool isDir = false;
+		String MyfileName = root.getNextFileName(&isDir);
+		while (MyfileName != "") {
+			// ignore hidden folders, e.g. MacOS spotlight files
+			if (!startsWith( MyfileName.c_str() , (char *)"/.")) {
+				JsonObject entry = obj.createNestedObject();
+				convertAsciiToUtf8(MyfileName.c_str(), filePath);
+				std::string path = filePath;
+				std::string fileName = path.substr(path.find_last_of("/") + 1);
+				entry["name"] = fileName;
+				entry["dir"].set(isDir);
+			}
+			MyfileName = root.getNextFileName(&isDir);
+		}
+    #else
+		File file = root.openNextFile();
 
-	while (file) {
-		// ignore hidden folders, e.g. MacOS spotlight files
-		#if ESP_ARDUINO_VERSION_MAJOR >= 2
-		if (!startsWith( file.path() , (char *)"/.")) {
-		#else
-		if (!startsWith( file.name() , (char *)"/.")) {
-		#endif
-			JsonObject entry = obj.createNestedObject();
+		while (file) {
+			// ignore hidden folders, e.g. MacOS spotlight files
 			#if ESP_ARDUINO_VERSION_MAJOR >= 2
-				convertAsciiToUtf8(file.path(), filePath);
+				if (!startsWith( file.path() , (char *)"/.")) {
 			#else
-				convertAsciiToUtf8(file.name(), filePath);
+				if (!startsWith( file.name() , (char *)"/.")) {
 			#endif
-			std::string path = filePath;
-			std::string fileName = path.substr(path.find_last_of("/") + 1);
+				JsonObject entry = obj.createNestedObject();
+				#if ESP_ARDUINO_VERSION_MAJOR >= 2
+					convertAsciiToUtf8(file.path(), filePath);
+				#else
+					convertAsciiToUtf8(file.name(), filePath);
+				#endif
+				std::string path = filePath;
+				std::string fileName = path.substr(path.find_last_of("/") + 1);
 
-			entry["name"] = fileName;
-			entry["dir"].set(file.isDirectory());
-		}
-		file.close();
-		file = root.openNextFile();
+				entry["name"] = fileName;
+				entry["dir"].set(file.isDirectory());
+			}
+			file.close();
+			file = root.openNextFile();
 
 
-		if (!gPlayProperties.pausePlay) {
-			// time critical, avoid delay with many files on SD-card!
-			feedTheDog();
-		} else {
-			// If playback is active this can (at least sometimes) prevent scattering
-			vTaskDelay(portTICK_PERIOD_MS * 5);
+			if (!gPlayProperties.pausePlay) {
+				// time critical, avoid delay with many files on SD-card!
+				feedTheDog();
+			} else {
+				// If playback is active this can (at least sometimes) prevent scattering
+				vTaskDelay(portTICK_PERIOD_MS * 5);
+			}
 		}
-	}
+	#endif
 	root.close();
 
 	serializeJson(obj, serializedJsonString);

src/settings.h

@@ -49,7 +49,7 @@
 	#define USEROTARY_ENABLE                // If rotary-encoder is used (don't forget to review WAKEUP_BUTTON if you disable this feature!)
 	#define BLUETOOTH_ENABLE                // If enabled and bluetooth-mode is active, you can stream to your ESPuino via bluetooth (a2dp-sink).
 	//#define IR_CONTROL_ENABLE             // Enables remote control (https://forum.espuino.de/t/neues-feature-fernsteuerung-per-infrarot-fernbedienung/265)
-	#define CACHED_PLAYLIST_ENABLE          // Enables playlist-caching (infos: https://forum.espuino.de/t/neues-feature-cached-playlist/515)
+	//#define CACHED_PLAYLIST_ENABLE          // Enables playlist-caching (infos: https://forum.espuino.de/t/neues-feature-cached-playlist/515). This might be slower than uncached file listing (https://forum.espuino.de/t/beschleunigte-verzeichnisauflistung-bei-sehr-vielen-mp3s/1348/20)
 	//#define PAUSE_WHEN_RFID_REMOVED       // Playback starts when card is applied and pauses automatically, when card is removed (https://forum.espuino.de/t/neues-feature-pausieren-wenn-rfid-karte-entfernt-wurde/541)
 	//#define PAUSE_ON_MIN_VOLUME           // When playback is active and volume is changed to zero, playback is paused automatically. Playback is continued if volume reaches 1. (https://forum.espuino.de/t/neues-feature-pausieren-wenn-rfid-karte-entfernt-wurde/541)
 	//#define DONT_ACCEPT_SAME_RFID_TWICE   // RFID-reader doesn't accept the same RFID-tag twice in a row (unless it's a modification-card or RFID-tag is unknown in NVS). Flag will be ignored silently if PAUSE_WHEN_RFID_REMOVED is active. (https://forum.espuino.de/t/neues-feature-dont-accept-same-rfid-twice/1247)