Add easy way to verify games against the Redump database, supplied as…

… CSV.

CMakeLists.txt

@@ -2265,6 +2265,8 @@ add_library(${CoreLibName} ${CoreLinkType}
 	Core/Util/AudioFormat.h
 	Core/Util/GameManager.cpp
 	Core/Util/GameManager.h
+	Core/Util/GameDB.cpp
+	Core/Util/GameDB.h
 	Core/Util/PortManager.cpp
 	Core/Util/PortManager.h
 	Core/Util/BlockAllocator.cpp

Common/StringUtils.h

@@ -45,6 +45,14 @@ inline bool startsWith(const std::string &str, const char *key) {
 	return !memcmp(str.data(), key, keyLen);
 }
 
+// Optimized for string views.
+inline bool startsWith(std::string_view str, std::string_view key) {
+	size_t keyLen = key.size();
+	if (str.size() < keyLen)
+		return false;
+	return !memcmp(str.data(), key.data(), keyLen);
+}
+
 inline bool startsWith(const std::string &str, const std::string &what) {
 	if (str.size() < what.size())
 		return false;

Core/Core.vcxproj

@@ -1076,6 +1076,7 @@
     <ClCompile Include="Util\AudioFormat.cpp" />
     <ClCompile Include="Util\BlockAllocator.cpp" />
     <ClCompile Include="Util\DisArm64.cpp" />
+    <ClCompile Include="Util\GameDB.cpp" />
     <ClCompile Include="Util\GameManager.cpp" />
     <ClCompile Include="Util\PortManager.cpp" />
     <ClCompile Include="Util\PPGeDraw.cpp" />
@@ -1444,6 +1445,7 @@
     <ClInclude Include="Util\AudioFormat.h" />
     <ClInclude Include="Util\BlockAllocator.h" />
     <ClInclude Include="Util\DisArm64.h" />
+    <ClInclude Include="Util\GameDB.h" />
     <ClInclude Include="Util\GameManager.h" />
     <ClInclude Include="Util\PortManager.h" />
     <ClInclude Include="Util\PPGeDraw.h" />

Core/Core.vcxproj.filters

@@ -1297,6 +1297,9 @@
     <ClCompile Include="MIPS\ARM64\Arm64IRCompFPU.cpp">
       <Filter>MIPS\ARM64</Filter>
     </ClCompile>
+    <ClCompile Include="Util\GameDB.cpp">
+      <Filter>Util</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="ELF\ElfReader.h">
@@ -2070,6 +2073,9 @@
     <ClInclude Include="MIPS\ARM64\Arm64IRRegCache.h">
       <Filter>MIPS\ARM64</Filter>
     </ClInclude>
+    <ClInclude Include="Util\GameDB.h">
+      <Filter>Util</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\LICENSE.TXT" />

Core/FileSystems/BlockDevices.h

@@ -45,8 +45,11 @@ class BlockDevice {
 		return true;
 	}
 	int GetBlockSize() const { return 2048;}  // forced, it cannot be changed by subclasses
-	virtual u32 GetNumBlocks() = 0;
-	virtual bool IsDisc() = 0;
+	virtual u32 GetNumBlocks() const = 0;
+	u64 GetUncompressedSize() const {
+		return (u64)GetNumBlocks() * (u64)GetBlockSize();
+	}
+	virtual bool IsDisc() const = 0;
 
 	u32 CalculateCRC(volatile bool *cancel = nullptr);
 	void NotifyReadError();
@@ -62,8 +65,8 @@ class CISOFileBlockDevice : public BlockDevice {
 	~CISOFileBlockDevice();
 	bool ReadBlock(int blockNumber, u8 *outPtr, bool uncached = false) override;
 	bool ReadBlocks(u32 minBlock, int count, u8 *outPtr) override;
-	u32 GetNumBlocks() override { return numBlocks; }
-	bool IsDisc() override { return true; }
+	u32 GetNumBlocks() const override { return numBlocks; }
+	bool IsDisc() const override { return true; }
 
 private:
 	u32 *index;
@@ -85,8 +88,8 @@ class FileBlockDevice : public BlockDevice {
 	~FileBlockDevice();
 	bool ReadBlock(int blockNumber, u8 *outPtr, bool uncached = false) override;
 	bool ReadBlocks(u32 minBlock, int count, u8 *outPtr) override;
-	u32 GetNumBlocks() override {return (u32)(filesize_ / GetBlockSize());}
-	bool IsDisc() override { return true; }
+	u32 GetNumBlocks() const override {return (u32)(filesize_ / GetBlockSize());}
+	bool IsDisc() const override { return true; }
 
 private:
 	u64 filesize_;
@@ -109,8 +112,8 @@ class NPDRMDemoBlockDevice : public BlockDevice {
 	~NPDRMDemoBlockDevice();
 
 	bool ReadBlock(int blockNumber, u8 *outPtr, bool uncached = false) override;
-	u32 GetNumBlocks() override {return (u32)lbaSize;}
-	bool IsDisc() override { return false; }
+	u32 GetNumBlocks() const override {return (u32)lbaSize;}
+	bool IsDisc() const override { return false; }
 
 private:
 	static std::mutex mutex_;
@@ -138,8 +141,8 @@ class CHDFileBlockDevice : public BlockDevice {
 	~CHDFileBlockDevice();
 	bool ReadBlock(int blockNumber, u8 *outPtr, bool uncached = false) override;
 	bool ReadBlocks(u32 minBlock, int count, u8 *outPtr) override;
-	u32 GetNumBlocks() override { return numBlocks; }
-	bool IsDisc() override { return true; }
+	u32 GetNumBlocks() const override { return numBlocks; }
+	bool IsDisc() const override { return true; }
 
 private:
 	std::unique_ptr<CHDImpl> impl_;

Core/Util/GameDB.cpp

@@ -0,0 +1,140 @@
+#include <cstdint>
+
+#include "Core/Util/GameDB.h"
+#include "Common/Log.h"
+#include "Common/File/VFS/VFS.h"
+#include "Common/StringUtils.h"
+
+GameDB g_gameDB;
+
+void SplitCSVLine(const std::string_view str, const char delim, std::vector<std::string_view> &result) {
+	result.clear();
+
+	int indexCommaToLeftOfColumn = 0;
+	int indexCommaToRightOfColumn = -1;
+
+	bool inQuote = false;
+
+	for (int i = 0; i < static_cast<int>(str.size()); i++) {
+		if (str[i] == '\"') {
+			inQuote = !inQuote;
+		} else if (str[i] == ',' && !inQuote) {
+			indexCommaToLeftOfColumn = indexCommaToRightOfColumn;
+			indexCommaToRightOfColumn = i;
+			int index = indexCommaToLeftOfColumn + 1;
+			int length = indexCommaToRightOfColumn - index;
+			std::string_view column(str.data() + index, length);
+			// Remove quotes if possible
+			column = StripQuotes(column);
+			result.push_back(column);
+		}
+	}
+
+	const std::string_view finalColumn(str.data() + indexCommaToRightOfColumn + 1, str.size() - indexCommaToRightOfColumn - 1);
+	result.push_back(finalColumn);
+}
+
+static std::vector<std::string_view> splitSV(std::string_view strv, char delim, bool removeWhiteSpace) {
+	std::vector<std::string_view> output;
+	size_t first = 0;
+	while (first < strv.size()) {
+		const auto second = strv.find(delim, first);
+		if (first != second) {
+			std::string_view line = strv.substr(first, second - first);
+			if (line.back() == '\r') {
+				line = strv.substr(first, second - first - 1);
+			}
+			if (removeWhiteSpace) {
+				line = StripSpaces(line);
+			}
+			output.emplace_back(line);
+		}
+		if (second == std::string_view::npos)
+			break;
+		first = second + 1;
+	}
+	return output;
+}
+
+bool GameDB::LoadFromVFS(VFSInterface &vfs, const char *filename) {
+	size_t size;
+	uint8_t *data = vfs.ReadFile(filename, &size);
+	if (!data)
+		return false;
+	contents_ = std::string((const char *)data, size);
+	delete[] data;
+
+	// Split the string into views of each line, keeping the original.
+	std::vector<std::string_view> lines = splitSV(contents_, '\n', false);
+
+	SplitCSVLine(lines[0], ',', columns_);
+
+	const size_t titleColumn = GetColumnIndex("Title");
+	const size_t foreignTitleColumn = GetColumnIndex("Foreign Title");
+	const size_t serialColumn = GetColumnIndex("Serial");
+	const size_t crcColumn = GetColumnIndex("CRC32");
+	const size_t sizeColumn = GetColumnIndex("Size");
+
+	std::vector<std::string_view> items;
+	for (size_t i = 1; i < lines.size(); i++) {
+		auto &lineString = lines[i];
+		SplitCSVLine(lineString, ',', items);
+		if (items.size() != columns_.size()) {
+			// Bad line
+			ERROR_LOG(SYSTEM, "Bad line in CSV file: %s", std::string(lineString).c_str());
+			continue;
+		}
+
+		Line line;
+		line.title = items[titleColumn];
+		line.foreignTitle = items[foreignTitleColumn];
+		line.serials = splitSV(items[serialColumn], ',', true);
+		line.crc = items[crcColumn];
+		line.size = items[sizeColumn];
+		lines_.push_back(line);
+	}
+	return true;
+}
+
+size_t GameDB::GetColumnIndex(std::string_view name) const {
+	for (size_t i = 0; i < columns_.size(); i++) {
+		if (name == columns_[i]) {
+			return i;
+		}
+	}
+	return (size_t)-1;
+}
+
+// Our IDs are ULUS12345, while the DB has them in some different forms, with a space or dash as separator.
+// TODO: report to redump
+static bool IDMatches(std::string_view id, std::string_view dbId) {
+	if (id.size() < 9 || dbId.size() < 10)
+		return false;
+	if (id.substr(0, 4) != dbId.substr(0, 4))
+		return false;
+	if (id.substr(4, 5) != dbId.substr(5, 5))
+		return false;
+	return true;
+}
+
+bool GameDB::GetGameInfos(std::string_view id, std::vector<GameDBInfo> *infos) {
+	if (id.size() < 9) {
+		// Not a game.
+		return false;
+	}
+
+	for (auto &line : lines_) {
+		for (auto serial : line.serials) {
+			// Ignore version and stuff for now
+			if (IDMatches(id, serial)) {
+				GameDBInfo info;
+				sscanf(line.crc.data(), "%08x", &info.crc);
+				sscanf(line.size.data(), "%llu", &info.size);
+				info.title = line.title;
+				info.foreignTitle = line.foreignTitle;
+				infos->push_back(info);
+			}
+		}
+	}
+	return !infos->empty();
+}

Core/Util/GameDB.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+#include <string_view>
+
+class VFSInterface;
+
+// Serial/id doesn't need including here since we look up by it.
+struct GameDBInfo {
+	std::string title;
+	std::string foreignTitle;
+	uint32_t crc;
+	uint64_t size;
+};
+
+class GameDB {
+public:
+	bool LoadFromVFS(VFSInterface &vfs, const char *filename);
+	bool GetGameInfos(std::string_view id, std::vector<GameDBInfo> *infos);
+
+private:
+	size_t GetColumnIndex(std::string_view name) const;
+
+	struct Line {
+		// The exact same ISO can have multiple serials.
+		std::vector<std::string_view> serials;
+		// The below fields should match GameDBInfo.
+		std::string_view title;
+		std::string_view foreignTitle;
+		std::string_view size;
+		std::string_view crc;
+	};
+
+	std::string contents_;
+	std::vector<Line> lines_;
+	std::vector<std::string_view> columns_;
+};
+
+extern GameDB g_gameDB;

UI/GameInfoCache.cpp

@@ -111,7 +111,7 @@ bool GameInfo::Delete() {
 	}
 }
 
-u64 GameInfo::GetGameSizeInBytes() {
+u64 GameInfo::GetGameSizeOnDiskInBytes() {
 	switch (fileType) {
 	case IdentifiedFileType::PSP_PBP_DIRECTORY:
 	case IdentifiedFileType::PSP_SAVEDATA_DIRECTORY:
@@ -122,6 +122,22 @@ u64 GameInfo::GetGameSizeInBytes() {
 	}
 }
 
+u64 GameInfo::GetGameSizeUncompressedInBytes() {
+	switch (fileType) {
+	case IdentifiedFileType::PSP_PBP_DIRECTORY:
+	case IdentifiedFileType::PSP_SAVEDATA_DIRECTORY:
+		return File::ComputeRecursiveDirectorySize(ResolvePBPDirectory(filePath_));
+
+	default:
+	{
+		BlockDevice *blockDevice = constructBlockDevice(GetFileLoader().get());
+		u64 size = blockDevice->GetUncompressedSize();
+		delete blockDevice;
+		return size;
+	}
+	}
+}
+
 // Not too meaningful if the object itself is a savedata directory...
 std::vector<Path> GameInfo::GetSaveDataDirectories() {
 	Path memc = GetSysDirectory(DIRECTORY_SAVEDATA);
@@ -659,10 +675,13 @@ class GameInfoWorkItem : public Task {
 
 		if (info_->wantFlags & GAMEINFO_WANTSIZE) {
 			std::lock_guard<std::mutex> lock(info_->lock);
-			info_->gameSize = info_->GetGameSizeInBytes();
+			info_->gameSizeOnDisk = info_->GetGameSizeOnDiskInBytes();
 			info_->saveDataSize = info_->GetSaveDataSizeInBytes();
 			info_->installDataSize = info_->GetInstallDataSizeInBytes();
 		}
+		if (info_->wantFlags & GAMEINFO_WANTUNCOMPRESSEDSIZE) {
+			info_->gameSizeUncompressed = info_->GetGameSizeUncompressedInBytes();
+		}
 
 		// INFO_LOG(SYSTEM, "Completed writing info for %s", info_->GetTitle().c_str());
 	}

UI/GameInfoCache.h

@@ -56,6 +56,7 @@ enum GameInfoWantFlags {
 	GAMEINFO_WANTSIZE = 0x02,
 	GAMEINFO_WANTSND = 0x04,
 	GAMEINFO_WANTBGDATA = 0x08, // Use with WANTBG.
+	GAMEINFO_WANTUNCOMPRESSEDSIZE = 0x10,
 };
 
 class FileLoader;
@@ -94,7 +95,8 @@ class GameInfo {
 	std::shared_ptr<FileLoader> GetFileLoader();
 	void DisposeFileLoader();
 
-	u64 GetGameSizeInBytes();
+	u64 GetGameSizeUncompressedInBytes();  // NOTE: More expensive than GetGameSizeOnDiskInBytes().
+	u64 GetGameSizeOnDiskInBytes();
 	u64 GetSaveDataSizeInBytes();
 	u64 GetInstallDataSizeInBytes();
 
@@ -144,7 +146,8 @@ class GameInfo {
 
 	double lastAccessedTime = 0.0;
 
-	u64 gameSize = 0;
+	u64 gameSizeUncompressed = 0;
+	u64 gameSizeOnDisk = 0;  // compressed size, in case of CSO
 	u64 saveDataSize = 0;
 	u64 installDataSize = 0;