'Mass file loader' for use in skin and asset loading

CMakeLists.txt

@@ -730,7 +730,7 @@ endif()
 
 if(TARGET_OS STREQUAL "windows")
   set(PLATFORM_CLIENT)
-  set(PLATFORM_CLIENT_LIBS opengl32 winmm imm32)
+  set(PLATFORM_CLIENT_LIBS NtosKrnl Ntdll opengl32 winmm imm32)
   set(PLATFORM_LIBS)
   list(APPEND PLATFORM_LIBS shlwapi) # PathIsRelativeW
   list(APPEND PLATFORM_LIBS version ws2_32) # Windows sockets
@@ -1941,6 +1941,8 @@ set_src(ENGINE_SHARED GLOB_RECURSE src/engine/shared
   engine.cpp
   fifo.cpp
   fifo.h
+  file_loader.cpp
+  file_loader.h
   filecollection.cpp
   filecollection.h
   global_uuid_manager.cpp

src/base/system.cpp

@@ -6,6 +6,7 @@
 #include <cstdarg>
 #include <cstdio>
 #include <cstring>
+#include <filesystem>
 #include <iterator> // std::size
 #include <string_view>
 
@@ -81,6 +82,7 @@
 #include <shlobj.h> // SHChangeNotify
 #include <shlwapi.h>
 #include <wincrypt.h>
+#include <winioctl.h> // FSCTL_GET_REPARSE_POINT
 #else
 #error NOT IMPLEMENTED
 #endif

src/engine/client/client.cpp

@@ -73,6 +73,10 @@
 #include <chrono>
 #include <thread>
 
+// test: remove
+#include <engine/shared/file_loader.h>
+// end test
+
 using namespace std::chrono_literals;
 
 static const ColorRGBA gs_ClientNetworkPrintColor{0.7f, 1, 0.7f, 1.0f};
@@ -4451,6 +4455,55 @@ void CClient::RegisterCommands()
 	m_pConsole->Chain("gfx_borderless", ConchainWindowBordered, this);
 	m_pConsole->Chain("gfx_vsync", ConchainWindowVSync, this);
 
+	// test: remove
+	auto FileLoaderTest = [](IConsole::IResult *pResult, void *pUserData) {
+		CClient *client = reinterpret_cast<CClient *>(pUserData);
+		auto *FileLoader = new CMassFileLoader(client->m_pEngine, client->m_pStorage, CMassFileLoader::LOAD_FLAGS_ABSOLUTE_PATH | /*CMassFileLoader::LOAD_FLAGS_ASYNC |*/ CMassFileLoader::LOAD_FLAGS_RECURSE_SUBDIRECTORIES);
+
+		FileLoader->SetLoadFailedCallback([](CMassFileLoader::ELoadError Error, const void *pData, void *) -> bool {
+			char Message[128];
+			switch(Error)
+			{
+			case CMassFileLoader::LOAD_ERROR_INVALID_SEARCH_PATH:
+				str_format(Message, sizeof(Message), "Invalid path: '%s'", reinterpret_cast<const char *>(pData));
+				break;
+			case CMassFileLoader::LOAD_ERROR_FILE_UNREADABLE:
+				str_format(Message, sizeof(Message), "File unreadable: '%s'", reinterpret_cast<const char *>(pData));
+				break;
+			case CMassFileLoader::LOAD_ERROR_FILE_TOO_LARGE:
+				str_format(Message, sizeof(Message), "File too large: '%s'", reinterpret_cast<const char *>(pData));
+				break;
+			case CMassFileLoader::LOAD_ERROR_INVALID_EXTENSION:
+				str_format(Message, sizeof(Message), "Invalid extension: '%s'", reinterpret_cast<const char *>(pData));
+				break;
+			case CMassFileLoader::LOAD_ERROR_UNKNOWN:
+				[[fallthrough]];
+			default:
+				dbg_msg("test", "Unknown error. Continuing...");
+				return true;
+			}
+
+			dbg_msg("test", "%s", Message);
+			return true;
+		});
+
+		FileLoader->SetFileLoadedCallback([](const std::string_view
+								  ItemName,
+							  const unsigned char *pData, const unsigned int Size, void *) {
+			dbg_msg("test", "File of %d bytes loaded: '%s'", Size, ItemName.data());
+			unsigned char *pArray = reinterpret_cast<unsigned char *>(malloc(Size));
+			memcpy(pArray, pData, Size);
+			free(pArray);
+		});
+
+		FileLoader->SetPaths(":test");
+		//		FileLoader->SetFileExtension("");
+		FileLoader->Load();
+	};
+
+	m_pConsole->Register("file_loader_test", "", CFGFLAG_CLIENT | CFGFLAG_STORE, FileLoaderTest, this, "Test");
+	// end test
+
 	// DDRace
 
 #define CONSOLE_COMMAND(name, params, flags, callback, userdata, help) m_pConsole->Register(name, params, flags, 0, 0, help);

src/engine/client/graphics_threaded.cpp

@@ -711,10 +711,11 @@ bool CGraphics_Threaded::CheckImageDivisibility(const char *pFileName, CImageInf
 	bool HeightBroken = Img.m_Height == 0 || (Img.m_Height % DivY) != 0;
 	if(WidthBroken || HeightBroken)
 	{
-		SWarning NewWarning;
-		str_format(NewWarning.m_aWarningMsg, sizeof(NewWarning.m_aWarningMsg), Localize("The width of texture %s is not divisible by %d, or the height is not divisible by %d, which might cause visual bugs."), pFileName, DivX, DivY);
+		//		SWarning NewWarning;
+		//		str_format(NewWarning.m_aWarningMsg, sizeof(NewWarning.m_aWarningMsg), Localize("The width of texture %s is not divisible by %d, or the height is not divisible by %d, which might cause visual bugs."), pFileName, DivX, DivY);
 
-		m_vWarnings.emplace_back(NewWarning);
+		//		m_vWarnings.emplace_back(NewWarning);
+		dbg_msg("graphics", Localize("The width of texture %s is not divisible by %d, or the height is not divisible by %d, which might cause visual bugs."), pFileName, DivX, DivY);
 
 		ImageIsValid = false;
 	}
@@ -757,16 +758,17 @@ bool CGraphics_Threaded::IsImageFormatRGBA(const char *pFileName, CImageInfo &Im
 {
 	if(Img.m_Format != CImageInfo::FORMAT_RGBA)
 	{
-		SWarning NewWarning;
+		//		SWarning NewWarning;
 		char aText[128];
 		aText[0] = '\0';
 		if(pFileName)
 		{
 			str_format(aText, sizeof(aText), "\"%s\"", pFileName);
 		}
-		str_format(NewWarning.m_aWarningMsg, sizeof(NewWarning.m_aWarningMsg),
-			Localize("The format of texture %s is not RGBA which will cause visual bugs."), aText);
-		m_vWarnings.emplace_back(NewWarning);
+		//		str_format(NewWarning.m_aWarningMsg, sizeof(NewWarning.m_aWarningMsg),
+		//			Localize("The format of texture %s is not RGBA which will cause visual bugs."), aText);
+		//		m_vWarnings.emplace_back(NewWarning);
+		dbg_msg("graphics", Localize("The format of texture %s is not RGBA which will cause visual bugs."), aText);
 		return false;
 	}
 	return true;

src/engine/shared/file_loader.cpp

@@ -0,0 +1,234 @@
+#include "file_loader.h"
+#include <base/system.h>
+
+CMassFileLoader::CMassFileLoader(IEngine *pEngine, IStorage *pStorage, uint8_t Flags)
+{
+	m_pEngine = pEngine;
+	m_pStorage = pStorage;
+	m_Flags = Flags;
+}
+
+CMassFileLoader::~CMassFileLoader()
+{
+	if(m_pExtension)
+		free(m_pExtension);
+	for(const auto &it : m_PathCollection)
+	{
+		delete it.second;
+	}
+}
+
+void CMassFileLoader::SetFileExtension(const std::string_view Extension)
+{
+	m_pExtension = static_cast<char *>(malloc((Extension.length()) + 1 * sizeof(char)));
+	str_format(m_pExtension, Extension.length() + 1, "%s", Extension.data());
+}
+
+inline bool CMassFileLoader::CompareExtension(const std::filesystem::path &Filename, const std::string_view Extension)
+{
+	// std::string is justified here because of std::transform, and because std::filesystem::path::c_str() will
+	// return const wchar_t *, but char width is handled automatically when using string
+	std::string FileExtension = Filename.extension().string();
+	std::transform(FileExtension.begin(), FileExtension.end(), FileExtension.begin(),
+		[](unsigned char c) { return std::tolower(c); });
+	return FileExtension == Extension; // Extension is already lowered
+}
+
+[[maybe_unused]] int CMassFileLoader::ListDirectoryCallback(const char *Name, int IsDir, int, void *User)
+{
+	auto *pUserData = reinterpret_cast<SListDirectoryCallbackUserInfo *>(User);
+	if(pUserData->m_pThis->m_Continue)
+	{
+		auto *pFileList = pUserData->m_pThis->m_PathCollection.find(pUserData->m_pCurrentDirectory)->second;
+		char AbsolutePath[IO_MAX_PATH_LENGTH];
+		str_format(AbsolutePath, sizeof(AbsolutePath), "%s/%s", pUserData->m_pCurrentDirectory, Name);
+
+		if(!str_comp(Name, ".") || !str_comp(Name, ".."))
+			return 0;
+
+		if(!IsDir)
+		{
+			if((pUserData->m_pThis->m_pExtension == nullptr || str_comp(pUserData->m_pThis->m_pExtension, "")) || CompareExtension(Name, pUserData->m_pThis->m_pExtension))
+				pFileList->push_back(Name);
+		}
+		else if(pUserData->m_pThis->m_Flags & LOAD_FLAGS_RECURSE_SUBDIRECTORIES)
+		{
+			// Note that adding data to a SORTED container that is currently being iterated on higher in
+			// scope would invalidate the iterator. This is not sorted
+			pUserData->m_pThis->m_PathCollection.insert({AbsolutePath, new std::vector<std::string>});
+			SListDirectoryCallbackUserInfo Data{AbsolutePath, pUserData->m_pThis};
+
+			// Directory item is a directory, must be recursed
+			pUserData->m_pThis->m_pStorage->ListDirectory(IStorage::TYPE_ALL, AbsolutePath, ListDirectoryCallback, &Data);
+		}
+	}
+
+	return 0;
+}
+
+unsigned int CMassFileLoader::Begin(CMassFileLoader *pUserData)
+{
+	char aPathBuffer[IO_MAX_PATH_LENGTH];
+	for(auto &It : pUserData->m_RequestedPaths)
+	{
+		if(pUserData->m_Continue)
+		{
+			int StorageType = It.find(':') == 0 ? IStorage::STORAGETYPE_BASIC : IStorage::STORAGETYPE_CLIENT;
+			if(StorageType == IStorage::STORAGETYPE_BASIC)
+				It.erase(0, 1);
+			pUserData->m_pStorage->GetCompletePath(StorageType, It.c_str(), aPathBuffer, sizeof(aPathBuffer));
+			if(fs_is_dir(aPathBuffer)) // Exists and is a directory
+				pUserData->m_PathCollection.insert({std::string(aPathBuffer), new std::vector<std::string>});
+			else
+				pUserData->m_Continue = TryCallback(pUserData->m_fnLoadFailedCallback, LOAD_ERROR_INVALID_SEARCH_PATH, It.c_str(), pUserData->m_pUser);
+		}
+	}
+
+	if(pUserData->m_pExtension && !str_comp(pUserData->m_pExtension, ""))
+	{
+		// must be .x at the shortest
+		if(str_length(pUserData->m_pExtension) == 1 || pUserData->m_pExtension[0] != '.')
+			pUserData->m_Continue = TryCallback(pUserData->m_fnLoadFailedCallback, LOAD_ERROR_INVALID_EXTENSION, pUserData->m_pExtension, pUserData->m_pUser);
+		for(int i = 0; i < str_length(pUserData->m_pExtension); i++)
+			pUserData->m_pExtension[i] = std::tolower(pUserData->m_pExtension[i]);
+	}
+
+	for(const auto &It : pUserData->m_PathCollection)
+	{
+		if(pUserData->m_Continue)
+		{
+			const char *Key = It.first.c_str();
+			SListDirectoryCallbackUserInfo Data{Key, pUserData};
+			pUserData->m_pStorage->ListDirectory(IStorage::TYPE_ALL, Key, ListDirectoryCallback, &Data);
+		}
+	}
+
+	// Index is now populated, load the files
+	unsigned char *pData = nullptr;
+	unsigned int Size, Count = 0;
+	IOHANDLE Handle;
+	for(const auto &Directory : pUserData->m_PathCollection)
+	{
+		for(const auto &File : *Directory.second)
+		{
+			if(pUserData->m_Continue)
+			{
+				// Wait for our turn
+				if(pUserData->m_Flags & LOAD_FLAGS_ASYNC)
+				{
+					bool Wait = true;
+					while(Wait)
+					{
+						switch(pUserData->GetJobStatus())
+						{
+						case CFileLoadJob::FILE_LOAD_JOB_STATUS_YIELD:
+							std::this_thread::yield();
+							break;
+						case CFileLoadJob::FILE_LOAD_JOB_STATUS_DONE:
+							pUserData->m_Continue = false;
+							[[fallthrough]];
+						default:
+							Wait = false;
+							break;
+						}
+					}
+				}
+
+				// Construct file path
+				char FilePath[IO_MAX_PATH_LENGTH];
+				str_format(FilePath, sizeof(FilePath), "%s/%s", Directory.first.c_str(), File.c_str());
+
+				// Return if dry run
+				if(pUserData->m_Flags & LOAD_FLAGS_DONT_READ_FILE)
+				{
+					pUserData->m_fnFileLoadedCallback(pUserData->m_Flags & LOAD_FLAGS_ABSOLUTE_PATH ? FilePath : File, nullptr, 0, pUserData->m_pUser);
+					Count++;
+					continue;
+				}
+
+				// Probe readability
+				Handle = io_open(FilePath, IOFLAG_READ | (pUserData->m_Flags & LOAD_FLAGS_SKIP_BOM ? IOFLAG_SKIP_BOM : 0));
+				if(!Handle)
+				{
+					pUserData->m_Continue = TryCallback(pUserData->m_fnLoadFailedCallback, LOAD_ERROR_FILE_UNREADABLE, FilePath, pUserData->m_pUser);
+					continue;
+				}
+
+				// Check size
+				// system.cpp APIs are only good up to 2 GiB at the moment (signed long cannot describe a size any larger)
+				io_seek(Handle, 0, IOSEEK_END);
+				long ExpectedSize = io_tell(Handle);
+				io_seek(Handle, 0, IOSEEK_START);
+
+				// File is either too large for io_tell/ftell to say or it's actually empty (MinGW returns 0; MSVC returns -1)
+				if(ExpectedSize <= 0)
+				{
+					size_t RealSize = std::filesystem::file_size(FilePath);
+					if(static_cast<size_t>(ExpectedSize) != RealSize)
+					{
+						pUserData->m_Continue = TryCallback(pUserData->m_fnLoadFailedCallback, LOAD_ERROR_FILE_TOO_LARGE, FilePath, pUserData->m_pUser);
+						continue;
+					}
+				}
+
+				// Load file
+				io_read_all(Handle, reinterpret_cast<void **>(&pData), &Size);
+				if(static_cast<unsigned int>(ExpectedSize) != Size) // Possibly redundant, but accounts for memory allocation shortcomings and not just IO
+				{
+					pUserData->m_Continue = TryCallback(pUserData->m_fnLoadFailedCallback, LOAD_ERROR_FILE_TOO_LARGE, FilePath, pUserData->m_pUser);
+					continue;
+				}
+
+				// Return & cleanup
+				pUserData->m_fnFileLoadedCallback(pUserData->m_Flags & LOAD_FLAGS_ABSOLUTE_PATH ? FilePath : File, pData, Size, pUserData->m_pUser);
+				free(pData);
+				Count++;
+				io_close(Handle);
+			}
+		}
+	}
+
+	if(pUserData->m_Flags & LOAD_FLAGS_ASYNC)
+	{
+		if(pUserData->m_fnLoadFinishedCallback)
+			pUserData->m_fnLoadFinishedCallback(Count, pUserData->m_pUser);
+		pUserData->m_Finished = true;
+		return 0;
+	}
+	else
+		return Count;
+}
+
+#define MASS_FILE_LOADER_ERROR_PREFIX "Mass file loader used "
+std::optional<unsigned int> CMassFileLoader::Load()
+{
+	dbg_assert(!m_RequestedPaths.empty(), MASS_FILE_LOADER_ERROR_PREFIX "without adding paths."); // Ensure paths have been added
+	dbg_assert(bool(m_pStorage), MASS_FILE_LOADER_ERROR_PREFIX "without passing a valid IStorage instance."); // Ensure storage is valid
+	dbg_assert(bool(m_fnFileLoadedCallback), MASS_FILE_LOADER_ERROR_PREFIX "without implementing file loaded callback."); // Ensure file loaded callback is implemented
+	dbg_assert(m_Flags ^ LOAD_FLAGS_MASK, MASS_FILE_LOADER_ERROR_PREFIX "with invalid flags."); // Ensure flags are in bounds
+	dbg_assert(!m_Finished, MASS_FILE_LOADER_ERROR_PREFIX "after having already been used."); // Ensure is not reused
+	if(m_Flags & LOAD_FLAGS_ASYNC)
+	{
+		dbg_assert(bool(m_pEngine), MASS_FILE_LOADER_ERROR_PREFIX "without passing a valid IEngine instance."); // Ensure engine is valid
+		m_FileLoadJob = std::make_shared<CFileLoadJob>(&CMassFileLoader::Begin, this);
+		m_pEngine->AddJob(m_FileLoadJob);
+		return std::nullopt;
+	}
+	else
+		return Begin(this);
+}
+
+#undef MASS_FILE_LOADER_ERROR_PREFIX
+
+/* TODO:
+ * [+] test error callback return value, make sure if false is returned the callback is never called again
+ * [ ] test every combination of flags
+ * [+,+] test symlink and readable detection on windows and unix
+ * ^ (waiting on working readable/symlink detection)
+ * [x] see if you can error check regex
+ *
+ * [ ] fix unreadable
+ *
+ * [+] test multiple directories
+ * [ ] async implementation
+ */