From 8da5d0c4fee307b3a195dc5ef55c86dfbb40df34 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 28 Sep 2018 17:04:16 +0200 Subject: [PATCH 01/36] Add an early version of WIABlobReader It can currently only read the first 0x80 bytes of a disc image, which is enough for identifying it but not for doing anything else. --- .../dolphinemu/utils/FileBrowserHelper.java | 2 +- Source/Core/Core/Boot/Boot.cpp | 2 +- Source/Core/DiscIO/Blob.cpp | 3 + Source/Core/DiscIO/Blob.h | 3 +- Source/Core/DiscIO/CMakeLists.txt | 2 + Source/Core/DiscIO/DiscIO.vcxproj | 2 + Source/Core/DiscIO/DiscIO.vcxproj.filters | 6 ++ Source/Core/DiscIO/WIABlob.cpp | 92 ++++++++++++++++ Source/Core/DiscIO/WIABlob.h | 102 ++++++++++++++++++ .../Core/DolphinQt/GameList/GameTracker.cpp | 4 +- Source/Core/DolphinQt/Info.plist.in | 1 + Source/Core/DolphinQt/MainWindow.cpp | 4 +- Source/Core/DolphinQt/Settings/PathPane.cpp | 2 +- Source/Core/UICommon/GameFileCache.cpp | 2 +- 14 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 Source/Core/DiscIO/WIABlob.cpp create mode 100644 Source/Core/DiscIO/WIABlob.h diff --git a/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java b/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java index 96658720c741..a27b2f2e0563 100644 --- a/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java +++ b/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java @@ -22,7 +22,7 @@ public final class FileBrowserHelper { public static final HashSet GAME_EXTENSIONS = new HashSet<>(Arrays.asList( - "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wad", "dol", "elf", "dff")); + "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "wad", "dol", "elf", "dff")); public static final HashSet RAW_EXTENSION = new HashSet<>(Collections.singletonList( "raw")); diff --git a/Source/Core/Core/Boot/Boot.cpp b/Source/Core/Core/Boot/Boot.cpp index a4f85208e3d6..f06c1077f92f 100644 --- a/Source/Core/Core/Boot/Boot.cpp +++ b/Source/Core/Core/Boot/Boot.cpp @@ -159,7 +159,7 @@ BootParameters::GenerateFromFile(std::vector paths, paths.clear(); static const std::unordered_set disc_image_extensions = { - {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".dol", ".elf"}}; + {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".dol", ".elf"}}; if (disc_image_extensions.find(extension) != disc_image_extensions.end() || is_drive) { std::unique_ptr disc = DiscIO::CreateDisc(path); diff --git a/Source/Core/DiscIO/Blob.cpp b/Source/Core/DiscIO/Blob.cpp index 20fe23e9ca45..97e48eb047c5 100644 --- a/Source/Core/DiscIO/Blob.cpp +++ b/Source/Core/DiscIO/Blob.cpp @@ -20,6 +20,7 @@ #include "DiscIO/DriveBlob.h" #include "DiscIO/FileBlob.h" #include "DiscIO/TGCBlob.h" +#include "DiscIO/WIABlob.h" #include "DiscIO/WbfsBlob.h" namespace DiscIO @@ -205,6 +206,8 @@ std::unique_ptr CreateBlobReader(const std::string& filename) return TGCFileReader::Create(std::move(file)); case WBFS_MAGIC: return WbfsFileReader::Create(std::move(file), filename); + case WIA_MAGIC: + return WIAFileReader::Create(std::move(file), filename); default: if (auto directory_blob = DirectoryBlobReader::Create(filename)) return std::move(directory_blob); diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index e8846753d168..5371478d6582 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -34,7 +34,8 @@ enum class BlobType GCZ, CISO, WBFS, - TGC + TGC, + WIA }; class BlobReader diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index 6437bf35d108..1e234bea4d1c 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -42,6 +42,8 @@ add_library(discio VolumeWii.h WbfsBlob.cpp WbfsBlob.h + WIABlob.cpp + WIABlob.h WiiEncryptionCache.cpp WiiEncryptionCache.h WiiSaveBanner.cpp diff --git a/Source/Core/DiscIO/DiscIO.vcxproj b/Source/Core/DiscIO/DiscIO.vcxproj index ee1d2ecd3663..9c2c9f7954a4 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj +++ b/Source/Core/DiscIO/DiscIO.vcxproj @@ -65,6 +65,7 @@ + @@ -91,6 +92,7 @@ + diff --git a/Source/Core/DiscIO/DiscIO.vcxproj.filters b/Source/Core/DiscIO/DiscIO.vcxproj.filters index 43cf82fa03b4..b7990b19d676 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj.filters +++ b/Source/Core/DiscIO/DiscIO.vcxproj.filters @@ -90,6 +90,9 @@ Volume\Blob + + Volume\Blob + @@ -164,6 +167,9 @@ Volume\Blob + + Volume\Blob + diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp new file mode 100644 index 000000000000..85f7f8ce38a1 --- /dev/null +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -0,0 +1,92 @@ +// Copyright 2018 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#include "DiscIO/WIABlob.h" + +#include + +#include "Common/CommonTypes.h" +#include "Common/File.h" +#include "Common/Logging/Log.h" +#include "Common/StringUtil.h" +#include "Common/Swap.h" + +#include "DiscIO/VolumeWii.h" + +namespace DiscIO +{ +WIAFileReader::WIAFileReader(File::IOFile file, const std::string& path) : m_file(std::move(file)) +{ + m_valid = Initialize(path); +} + +WIAFileReader::~WIAFileReader() = default; + +bool WIAFileReader::Initialize(const std::string& path) +{ + if (!m_file.Seek(0, SEEK_SET) || !m_file.ReadArray(&m_header_1, 1)) + return false; + + if (m_header_1.magic != WIA_MAGIC) + return false; + + const u32 version = Common::swap32(m_header_1.version); + const u32 version_compatible = Common::swap32(m_header_1.version_compatible); + if (WIA_VERSION < version_compatible || WIA_VERSION_READ_COMPATIBLE > version) + { + ERROR_LOG(DISCIO, "Unsupported WIA version %s in %s", VersionToString(version).c_str(), + path.c_str()); + return false; + } + + if (Common::swap64(m_header_1.wia_file_size) != m_file.GetSize()) + { + ERROR_LOG(DISCIO, "File size is incorrect for %s", path.c_str()); + return false; + } + + if (Common::swap32(m_header_1.header_2_size) < sizeof(WIAHeader2)) + return false; + + if (!m_file.ReadArray(&m_header_2, 1)) + return false; + + const u32 chunk_size = Common::swap32(m_header_2.chunk_size); + if (chunk_size % VolumeWii::GROUP_TOTAL_SIZE != 0) + return false; + + return true; +} + +std::unique_ptr WIAFileReader::Create(File::IOFile file, const std::string& path) +{ + std::unique_ptr blob(new WIAFileReader(std::move(file), path)); + return blob->m_valid ? std::move(blob) : nullptr; +} + +bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) +{ + if (offset + size <= sizeof(WIAHeader2::disc_header)) + { + std::memcpy(out_ptr, m_header_2.disc_header.data() + offset, size); + return true; + } + + // Not implemented + return false; +} + +std::string WIAFileReader::VersionToString(u32 version) +{ + const u8 a = version >> 24; + const u8 b = (version >> 16) & 0xff; + const u8 c = (version >> 8) & 0xff; + const u8 d = version & 0xff; + + if (d == 0 || d == 0xff) + return StringFromFormat("%u.%02x.%02x", a, b, c); + else + return StringFromFormat("%u.%02x.%02x.beta%u", a, b, c, d); +} +} // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h new file mode 100644 index 000000000000..edd7d0b77633 --- /dev/null +++ b/Source/Core/DiscIO/WIABlob.h @@ -0,0 +1,102 @@ +// Copyright 2018 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#pragma once + +#include +#include + +#include "Common/CommonTypes.h" +#include "Common/File.h" +#include "Common/Swap.h" +#include "DiscIO/Blob.h" + +namespace DiscIO +{ +constexpr u32 WIA_MAGIC = 0x01414957; // "WIA\x1" (byteswapped to little endian) + +class WIAFileReader : public BlobReader +{ +public: + ~WIAFileReader(); + + static std::unique_ptr Create(File::IOFile file, const std::string& path); + + BlobType GetBlobType() const override { return BlobType::WIA; } + + u64 GetRawSize() const override { return Common::swap64(m_header_1.wia_file_size); } + u64 GetDataSize() const override { return Common::swap64(m_header_1.iso_file_size); } + bool IsDataSizeAccurate() const override { return true; } + + u64 GetBlockSize() const override { return Common::swap32(m_header_2.chunk_size); } + bool HasFastRandomAccessInBlock() const override { return false; } + + bool Read(u64 offset, u64 size, u8* out_ptr) override; + +private: + explicit WIAFileReader(File::IOFile file, const std::string& path); + bool Initialize(const std::string& path); + + static std::string VersionToString(u32 version); + + using SHA1 = std::array; + +#pragma pack(push, 1) + struct WIAHeader1 + { + u32 magic; + u32 version; + u32 version_compatible; + u32 header_2_size; + SHA1 header_2_hash; + u64 iso_file_size; + u64 wia_file_size; + SHA1 header_1_hash; + }; +#pragma pack(pop) + static_assert(sizeof(WIAHeader1) == 0x48, "Wrong size for WIA header 1"); + +#pragma pack(push, 1) + struct WIAHeader2 + { + u32 disc_type; + u32 compression_type; + u32 compression_level; // Informative only + u32 chunk_size; + + std::array disc_header; + + u32 number_of_partitions_entries; + u32 partition_entry_size; + u64 partition_entries_offset; + SHA1 partition_entries_hash; + + u32 number_of_raw_data_entries; + u64 raw_data_entries_offset; + u32 raw_data_entries_size; + + u32 number_of_group_entries; + u64 group_entries_offset; + u32 group_entries_size; + }; +#pragma pack(pop) + static_assert(sizeof(WIAHeader2) == 0xd4, "Wrong size for WIA header 2"); + + bool m_valid; + + File::IOFile m_file; + + WIAHeader1 m_header_1; + WIAHeader2 m_header_2; + + static constexpr u32 WIA_VERSION = 0x01000000; + static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000; + static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000; + + // Perhaps we could set WIA_VERSION_WRITE_COMPATIBLE to 0.9, but WIA version 0.9 was never in + // any official release of wit, and interim versions (either source or binaries) are hard to find. + // Since we've been unable to check if we're write compatible with 0.9, we set it 1.0 to be safe. +}; + +} // namespace DiscIO diff --git a/Source/Core/DolphinQt/GameList/GameTracker.cpp b/Source/Core/DolphinQt/GameList/GameTracker.cpp index 4caf643dd542..4aa930140c93 100644 --- a/Source/Core/DolphinQt/GameList/GameTracker.cpp +++ b/Source/Core/DolphinQt/GameList/GameTracker.cpp @@ -24,8 +24,8 @@ static const QStringList game_filters{ QStringLiteral("*.[gG][cC][mM]"), QStringLiteral("*.[iI][sS][oO]"), QStringLiteral("*.[tT][gG][cC]"), QStringLiteral("*.[cC][iI][sS][oO]"), QStringLiteral("*.[gG][cC][zZ]"), QStringLiteral("*.[wW][bB][fF][sS]"), - QStringLiteral("*.[wW][aA][dD]"), QStringLiteral("*.[eE][lL][fF]"), - QStringLiteral("*.[dD][oO][lL]")}; + QStringLiteral("*.[wW][iI][aA]"), QStringLiteral("*.[wW][aA][dD]"), + QStringLiteral("*.[eE][lL][fF]"), QStringLiteral("*.[dD][oO][lL]")}; GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent) { diff --git a/Source/Core/DolphinQt/Info.plist.in b/Source/Core/DolphinQt/Info.plist.in index 7fd60fa8e0d3..979f5f734882 100644 --- a/Source/Core/DolphinQt/Info.plist.in +++ b/Source/Core/DolphinQt/Info.plist.in @@ -16,6 +16,7 @@ m3u tgc wad + wia wbfs CFBundleTypeIconFile diff --git a/Source/Core/DolphinQt/MainWindow.cpp b/Source/Core/DolphinQt/MainWindow.cpp index 8b8eb37f4097..508b8224271d 100644 --- a/Source/Core/DolphinQt/MainWindow.cpp +++ b/Source/Core/DolphinQt/MainWindow.cpp @@ -686,8 +686,8 @@ QStringList MainWindow::PromptFileNames() QStringList paths = QFileDialog::getOpenFileNames( this, tr("Select a File"), settings.value(QStringLiteral("mainwindow/lastdir"), QString{}).toString(), - tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wad *.dff *.m3u);;" - "All Files (*)")); + tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.wad *.dff " + "*.m3u);;All Files (*)")); if (!paths.isEmpty()) { diff --git a/Source/Core/DolphinQt/Settings/PathPane.cpp b/Source/Core/DolphinQt/Settings/PathPane.cpp index ab59f185ccf1..e77f4d09c33e 100644 --- a/Source/Core/DolphinQt/Settings/PathPane.cpp +++ b/Source/Core/DolphinQt/Settings/PathPane.cpp @@ -44,7 +44,7 @@ void PathPane::BrowseDefaultGame() { QString file = QDir::toNativeSeparators(QFileDialog::getOpenFileName( this, tr("Select a Game"), Settings::Instance().GetDefaultGame(), - tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wad *.m3u);;" + tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.wad *.m3u);;" "All Files (*)"))); if (!file.isEmpty()) diff --git a/Source/Core/UICommon/GameFileCache.cpp b/Source/Core/UICommon/GameFileCache.cpp index 9b2729407ed1..bb12eb3ccd50 100644 --- a/Source/Core/UICommon/GameFileCache.cpp +++ b/Source/Core/UICommon/GameFileCache.cpp @@ -33,7 +33,7 @@ std::vector FindAllGamePaths(const std::vector& direct bool recursive_scan) { static const std::vector search_extensions = { - ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wad", ".dol", ".elf"}; + ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wia", ".wad", ".dol", ".elf"}; // TODO: We could process paths iteratively as they are found return Common::DoFileSearch(directories_to_scan, search_extensions, recursive_scan); From 2a5fcc9c252697840ed7e75a0c04759b2a952537 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 2 Jan 2019 22:24:18 +0100 Subject: [PATCH 02/36] WIA: Add reading raw data --- Source/Core/DiscIO/WIABlob.cpp | 142 +++++++++++++++++++++++++++++++-- Source/Core/DiscIO/WIABlob.h | 52 +++++++++++- 2 files changed, 183 insertions(+), 11 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 85f7f8ce38a1..4151b7bc7f8f 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -4,6 +4,8 @@ #include "DiscIO/WIABlob.h" +#include +#include #include #include "Common/CommonTypes.h" @@ -46,16 +48,92 @@ bool WIAFileReader::Initialize(const std::string& path) return false; } - if (Common::swap32(m_header_1.header_2_size) < sizeof(WIAHeader2)) + const u32 header_2_size = Common::swap32(m_header_1.header_2_size); + const u32 header_2_min_size = sizeof(WIAHeader2) - sizeof(WIAHeader2::compressor_data); + if (header_2_size < header_2_min_size) return false; - if (!m_file.ReadArray(&m_header_2, 1)) + std::vector header_2(header_2_size); + if (!m_file.ReadBytes(header_2.data(), header_2.size())) return false; + std::memcpy(&m_header_2, header_2.data(), std::min(header_2.size(), sizeof(WIAHeader2))); + + if (m_header_2.compressor_data_size > sizeof(WIAHeader2::compressor_data) || + header_2_size < header_2_min_size + m_header_2.compressor_data_size) + { + return false; + } + const u32 chunk_size = Common::swap32(m_header_2.chunk_size); if (chunk_size % VolumeWii::GROUP_TOTAL_SIZE != 0) return false; + const u32 compression_type = Common::swap32(m_header_2.compression_type); + if (compression_type != 0) + { + ERROR_LOG(DISCIO, "Unsupported WIA compression type %u in %s", compression_type, path.c_str()); + return false; + } + + const size_t number_of_partition_entries = Common::swap32(m_header_2.number_of_partition_entries); + const size_t partition_entry_size = Common::swap32(m_header_2.partition_entry_size); + std::vector partition_entries(partition_entry_size * number_of_partition_entries); + if (!m_file.Seek(Common::swap64(m_header_2.partition_entries_offset), SEEK_SET)) + return false; + if (!m_file.ReadBytes(partition_entries.data(), partition_entries.size())) + return false; + // TODO: Check hash + + const size_t copy_length = std::min(partition_entry_size, sizeof(PartitionEntry)); + const size_t memset_length = sizeof(PartitionEntry) - copy_length; + u8* ptr = partition_entries.data(); + m_partition_entries.resize(number_of_partition_entries); + for (size_t i = 0; i < number_of_partition_entries; ++i, ptr += partition_entry_size) + { + std::memcpy(&m_partition_entries[i], ptr, copy_length); + std::memset(reinterpret_cast(&m_partition_entries[i]) + copy_length, 0, memset_length); + } + + for (const PartitionEntry& partition : m_partition_entries) + { + if (Common::swap32(partition.data_entries[1].number_of_sectors) != 0) + { + const u32 first_end = Common::swap32(partition.data_entries[0].first_sector) + + Common::swap32(partition.data_entries[0].number_of_sectors); + const u32 second_start = Common::swap32(partition.data_entries[1].first_sector); + if (first_end > second_start) + return false; + } + } + + std::sort(m_partition_entries.begin(), m_partition_entries.end(), + [](const PartitionEntry& a, const PartitionEntry& b) { + return Common::swap32(a.data_entries[0].first_sector) < + Common::swap32(b.data_entries[0].first_sector); + }); + + // TODO: Compression + const u32 number_of_raw_data_entries = Common::swap32(m_header_2.number_of_raw_data_entries); + m_raw_data_entries.resize(number_of_raw_data_entries); + if (!m_file.Seek(Common::swap64(m_header_2.raw_data_entries_offset), SEEK_SET)) + return false; + if (!m_file.ReadArray(m_raw_data_entries.data(), number_of_raw_data_entries)) + return false; + + std::sort(m_raw_data_entries.begin(), m_raw_data_entries.end(), + [](const RawDataEntry& a, const RawDataEntry& b) { + return Common::swap64(a.data_offset) < Common::swap64(b.data_offset); + }); + + // TODO: Compression + const u32 number_of_group_entries = Common::swap32(m_header_2.number_of_group_entries); + m_group_entries.resize(number_of_group_entries); + if (!m_file.Seek(Common::swap64(m_header_2.group_entries_offset), SEEK_SET)) + return false; + if (!m_file.ReadArray(m_group_entries.data(), number_of_group_entries)) + return false; + return true; } @@ -67,14 +145,64 @@ std::unique_ptr WIAFileReader::Create(File::IOFile file, const st bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) { - if (offset + size <= sizeof(WIAHeader2::disc_header)) + if (offset + size > Common::swap64(m_header_1.iso_file_size)) + return false; + + if (offset < sizeof(WIAHeader2::disc_header)) + { + const u64 bytes_to_read = std::min(sizeof(WIAHeader2::disc_header) - offset, size); + std::memcpy(out_ptr, m_header_2.disc_header.data() + offset, bytes_to_read); + offset += bytes_to_read; + size -= bytes_to_read; + out_ptr += bytes_to_read; + } + + const u32 chunk_size = Common::swap32(m_header_2.chunk_size); + for (RawDataEntry raw_data : m_raw_data_entries) { - std::memcpy(out_ptr, m_header_2.disc_header.data() + offset, size); - return true; + if (size == 0) + return true; + + u64 data_offset = Common::swap64(raw_data.data_offset); + u64 data_size = Common::swap64(raw_data.data_size); + + if (data_offset + data_size <= offset) + continue; + + if (offset < data_offset) + return false; + + const u64 skipped_data = data_offset % VolumeWii::BLOCK_TOTAL_SIZE; + data_offset -= skipped_data; + data_size += skipped_data; + + const u32 number_of_groups = Common::swap32(raw_data.number_of_groups); + const u64 group_index = (offset - data_offset) / chunk_size; + for (u64 i = group_index; i < number_of_groups && size > 0; ++i) + { + const u64 total_group_index = Common::swap32(raw_data.group_index) + i; + if (total_group_index >= m_group_entries.size()) + return false; + + const GroupEntry group = m_group_entries[total_group_index]; + const u64 group_offset_on_disc = data_offset + i * chunk_size; + const u64 offset_in_group = offset - group_offset_on_disc; + + // TODO: Compression + + const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; + const u64 offset_in_file = group_offset_in_file + offset_in_group; + const u64 bytes_to_read = std::min(chunk_size - offset_in_group, size); + if (!m_file.Seek(offset_in_file, SEEK_SET) || !m_file.ReadBytes(out_ptr, bytes_to_read)) + return false; + + offset += bytes_to_read; + size -= bytes_to_read; + out_ptr += bytes_to_read; + } } - // Not implemented - return false; + return size == 0; } std::string WIAFileReader::VersionToString(u32 version) diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index edd7d0b77633..c8ec442488b0 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -41,6 +41,7 @@ class WIAFileReader : public BlobReader static std::string VersionToString(u32 version); using SHA1 = std::array; + using WiiKey = std::array; #pragma pack(push, 1) struct WIAHeader1 @@ -54,10 +55,8 @@ class WIAFileReader : public BlobReader u64 wia_file_size; SHA1 header_1_hash; }; -#pragma pack(pop) static_assert(sizeof(WIAHeader1) == 0x48, "Wrong size for WIA header 1"); -#pragma pack(push, 1) struct WIAHeader2 { u32 disc_type; @@ -67,7 +66,7 @@ class WIAFileReader : public BlobReader std::array disc_header; - u32 number_of_partitions_entries; + u32 number_of_partition_entries; u32 partition_entry_size; u64 partition_entries_offset; SHA1 partition_entries_hash; @@ -79,9 +78,51 @@ class WIAFileReader : public BlobReader u32 number_of_group_entries; u64 group_entries_offset; u32 group_entries_size; + + u8 compressor_data_size; + u8 compressor_data[7]; + }; + static_assert(sizeof(WIAHeader2) == 0xdc, "Wrong size for WIA header 2"); + + struct PartitionDataEntry + { + u32 first_sector; + u32 number_of_sectors; + u32 group_index; + u32 number_of_groups; + }; + static_assert(sizeof(PartitionDataEntry) == 0x10, "Wrong size for WIA partition data entry"); + + struct PartitionEntry + { + WiiKey partition_key; + std::array data_entries; + }; + static_assert(sizeof(PartitionEntry) == 0x30, "Wrong size for WIA partition entry"); + + struct RawDataEntry + { + u64 data_offset; + u64 data_size; + u32 group_index; + u32 number_of_groups; + }; + static_assert(sizeof(RawDataEntry) == 0x18, "Wrong size for WIA raw data entry"); + + struct GroupEntry + { + u32 data_offset; // >> 2 + u32 data_size; + }; + static_assert(sizeof(GroupEntry) == 0x08, "Wrong size for WIA group entry"); + + struct HashExceptionEntry + { + u16 offset; + SHA1 hash; }; + static_assert(sizeof(HashExceptionEntry) == 0x16, "Wrong size for WIA hash exception entry"); #pragma pack(pop) - static_assert(sizeof(WIAHeader2) == 0xd4, "Wrong size for WIA header 2"); bool m_valid; @@ -89,6 +130,9 @@ class WIAFileReader : public BlobReader WIAHeader1 m_header_1; WIAHeader2 m_header_2; + std::vector m_partition_entries; + std::vector m_raw_data_entries; + std::vector m_group_entries; static constexpr u32 WIA_VERSION = 0x01000000; static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000; From 3672bd79f37ac865e24e8a23bfeb20d7ee61ea02 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 27 Dec 2019 22:26:00 +0100 Subject: [PATCH 03/36] WIA: Implement ReadWiiDecrypted --- Source/Core/DiscIO/WIABlob.cpp | 113 +++++++++++++++++++++++++-------- Source/Core/DiscIO/WIABlob.h | 6 ++ 2 files changed, 93 insertions(+), 26 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 4151b7bc7f8f..74f7f4293e47 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -8,6 +8,7 @@ #include #include +#include "Common/Align.h" #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/Logging/Log.h" @@ -163,46 +164,106 @@ bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) if (size == 0) return true; - u64 data_offset = Common::swap64(raw_data.data_offset); - u64 data_size = Common::swap64(raw_data.data_size); + if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_TOTAL_SIZE, + Common::swap64(raw_data.data_offset), Common::swap64(raw_data.data_size), + Common::swap32(raw_data.group_index), + Common::swap32(raw_data.number_of_groups), false)) + { + return false; + } + } - if (data_offset + data_size <= offset) - continue; + return size == 0; +} - if (offset < data_offset) - return false; +bool WIAFileReader::SupportsReadWiiDecrypted() const +{ + return !m_partition_entries.empty(); +} - const u64 skipped_data = data_offset % VolumeWii::BLOCK_TOTAL_SIZE; - data_offset -= skipped_data; - data_size += skipped_data; +bool WIAFileReader::ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) +{ + const u64 chunk_size = Common::swap32(m_header_2.chunk_size) * VolumeWii::BLOCK_DATA_SIZE / + VolumeWii::BLOCK_TOTAL_SIZE; + for (const PartitionEntry& partition : m_partition_entries) + { + const u32 partition_first_sector = Common::swap32(partition.data_entries[0].first_sector); + if (partition_data_offset != partition_first_sector * VolumeWii::BLOCK_TOTAL_SIZE) + continue; - const u32 number_of_groups = Common::swap32(raw_data.number_of_groups); - const u64 group_index = (offset - data_offset) / chunk_size; - for (u64 i = group_index; i < number_of_groups && size > 0; ++i) + for (const PartitionDataEntry& data : partition.data_entries) { - const u64 total_group_index = Common::swap32(raw_data.group_index) + i; - if (total_group_index >= m_group_entries.size()) + if (size == 0) + return true; + + const u64 data_offset = + (Common::swap32(data.first_sector) - partition_first_sector) * VolumeWii::BLOCK_DATA_SIZE; + const u64 data_size = Common::swap32(data.number_of_sectors) * VolumeWii::BLOCK_DATA_SIZE; + + if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_DATA_SIZE, + data_offset, data_size, Common::swap32(data.group_index), + Common::swap32(data.number_of_groups), true)) + { return false; + } + } + + return size == 0; + } + + return false; +} + +bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, + u32 sector_size, u64 data_offset, u64 data_size, u32 group_index, + u32 number_of_groups, bool exception_list) +{ + if (data_offset + data_size <= *offset) + return true; + + if (*offset < data_offset) + return false; + + const u64 skipped_data = data_offset % sector_size; + data_offset -= skipped_data; + data_size += skipped_data; + + const u64 start_group_index = (*offset - data_offset) / chunk_size; + for (u64 i = start_group_index; i < number_of_groups && (*size) > 0; ++i) + { + const u64 total_group_index = group_index + i; + if (total_group_index >= m_group_entries.size()) + return false; + + const GroupEntry group = m_group_entries[total_group_index]; + const u64 group_offset = data_offset + i * chunk_size; + const u64 offset_in_group = *offset - group_offset; - const GroupEntry group = m_group_entries[total_group_index]; - const u64 group_offset_on_disc = data_offset + i * chunk_size; - const u64 offset_in_group = offset - group_offset_on_disc; + // TODO: Compression - // TODO: Compression + u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; - const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; - const u64 offset_in_file = group_offset_in_file + offset_in_group; - const u64 bytes_to_read = std::min(chunk_size - offset_in_group, size); - if (!m_file.Seek(offset_in_file, SEEK_SET) || !m_file.ReadBytes(out_ptr, bytes_to_read)) + if (exception_list) + { + u16 exceptions; + if (!m_file.Seek(group_offset_in_file, SEEK_SET) || !m_file.ReadArray(&exceptions, 1)) return false; - offset += bytes_to_read; - size -= bytes_to_read; - out_ptr += bytes_to_read; + group_offset_in_file += Common::AlignUp( + sizeof(exceptions) + Common::swap16(exceptions) * sizeof(HashExceptionEntry), 4); } + + const u64 offset_in_file = group_offset_in_file + offset_in_group; + const u64 bytes_to_read = std::min(chunk_size - offset_in_group, *size); + if (!m_file.Seek(offset_in_file, SEEK_SET) || !m_file.ReadBytes(*out_ptr, bytes_to_read)) + return false; + + *offset += bytes_to_read; + *size -= bytes_to_read; + *out_ptr += bytes_to_read; } - return size == 0; + return true; } std::string WIAFileReader::VersionToString(u32 version) diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index c8ec442488b0..fd00520dd1f2 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -33,11 +33,17 @@ class WIAFileReader : public BlobReader bool HasFastRandomAccessInBlock() const override { return false; } bool Read(u64 offset, u64 size, u8* out_ptr) override; + bool SupportsReadWiiDecrypted() const override; + bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override; private: explicit WIAFileReader(File::IOFile file, const std::string& path); bool Initialize(const std::string& path); + bool ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size, + u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups, + bool exception_list); + static std::string VersionToString(u32 version); using SHA1 = std::array; From 36991e2ddec67b97ba0306d13f90385f7748e72f Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 29 Dec 2019 23:29:51 +0100 Subject: [PATCH 04/36] WIA: Implement PURGE decompression --- Source/Core/DiscIO/WIABlob.cpp | 165 ++++++++++++++++++++++++++++----- Source/Core/DiscIO/WIABlob.h | 25 +++++ 2 files changed, 168 insertions(+), 22 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 74f7f4293e47..6c56af5ad10e 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -5,8 +5,10 @@ #include "DiscIO/WIABlob.h" #include +#include #include #include +#include #include "Common/Align.h" #include "Common/CommonTypes.h" @@ -71,7 +73,8 @@ bool WIAFileReader::Initialize(const std::string& path) return false; const u32 compression_type = Common::swap32(m_header_2.compression_type); - if (compression_type != 0) + m_compression_type = static_cast(compression_type); + if (m_compression_type > CompressionType::Purge) { ERROR_LOG(DISCIO, "Unsupported WIA compression type %u in %s", compression_type, path.c_str()); return false; @@ -114,26 +117,30 @@ bool WIAFileReader::Initialize(const std::string& path) Common::swap32(b.data_entries[0].first_sector); }); - // TODO: Compression const u32 number_of_raw_data_entries = Common::swap32(m_header_2.number_of_raw_data_entries); m_raw_data_entries.resize(number_of_raw_data_entries); - if (!m_file.Seek(Common::swap64(m_header_2.raw_data_entries_offset), SEEK_SET)) - return false; - if (!m_file.ReadArray(m_raw_data_entries.data(), number_of_raw_data_entries)) + if (!ReadCompressedData(number_of_raw_data_entries * sizeof(RawDataEntry), + Common::swap64(m_header_2.raw_data_entries_offset), + Common::swap32(m_header_2.raw_data_entries_size), + reinterpret_cast(m_raw_data_entries.data()), false)) + { return false; + } std::sort(m_raw_data_entries.begin(), m_raw_data_entries.end(), [](const RawDataEntry& a, const RawDataEntry& b) { return Common::swap64(a.data_offset) < Common::swap64(b.data_offset); }); - // TODO: Compression const u32 number_of_group_entries = Common::swap32(m_header_2.number_of_group_entries); m_group_entries.resize(number_of_group_entries); - if (!m_file.Seek(Common::swap64(m_header_2.group_entries_offset), SEEK_SET)) - return false; - if (!m_file.ReadArray(m_group_entries.data(), number_of_group_entries)) + if (!ReadCompressedData(number_of_group_entries * sizeof(GroupEntry), + Common::swap64(m_header_2.group_entries_offset), + Common::swap32(m_header_2.group_entries_size), + reinterpret_cast(m_group_entries.data()), false)) + { return false; + } return true; } @@ -239,31 +246,145 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu const u64 group_offset = data_offset + i * chunk_size; const u64 offset_in_group = *offset - group_offset; - // TODO: Compression + const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; + const u64 bytes_to_read = std::min(chunk_size - offset_in_group, *size); + if (!ReadCompressedData(chunk_size, group_offset_in_file, Common::swap32(group.data_size), + offset_in_group, bytes_to_read, *out_ptr, exception_list)) + { + return false; + } + + *offset += bytes_to_read; + *size -= bytes_to_read; + *out_ptr += bytes_to_read; + } + + return true; +} + +bool WIAFileReader::ReadCompressedData(u32 decompressed_data_size, u64 data_offset, u64 data_size, + u8* out_ptr, bool exception_list) +{ + switch (m_compression_type) + { + case CompressionType::None: + { + return ReadCompressedData(decompressed_data_size, data_offset, data_size, 0, + decompressed_data_size, out_ptr, exception_list); + } - u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; + case CompressionType::Purge: + { + if (!m_file.Seek(data_offset, SEEK_SET)) + return false; if (exception_list) { - u16 exceptions; - if (!m_file.Seek(group_offset_in_file, SEEK_SET) || !m_file.ReadArray(&exceptions, 1)) + const std::optional exception_size = ReadExceptionListFromFile(); + if (!exception_size) return false; - group_offset_in_file += Common::AlignUp( - sizeof(exceptions) + Common::swap16(exceptions) * sizeof(HashExceptionEntry), 4); + data_size -= *exception_size; } - const u64 offset_in_file = group_offset_in_file + offset_in_group; - const u64 bytes_to_read = std::min(chunk_size - offset_in_group, *size); - if (!m_file.Seek(offset_in_file, SEEK_SET) || !m_file.ReadBytes(*out_ptr, bytes_to_read)) + const u64 hash_offset = data_size - sizeof(SHA1); + u32 offset_in_data = 0; + u32 offset_in_decompressed_data = 0; + + while (offset_in_data < hash_offset) + { + PurgeSegment purge_segment; + if (!m_file.ReadArray(&purge_segment, 1)) + return false; + + const u32 segment_offset = Common::swap32(purge_segment.offset); + const u32 segment_size = Common::swap32(purge_segment.size); + + if (segment_offset < offset_in_decompressed_data) + return false; + + const u32 blank_bytes = segment_offset - offset_in_decompressed_data; + std::memset(out_ptr, 0, blank_bytes); + out_ptr += blank_bytes; + + if (segment_size != 0 && !m_file.ReadBytes(out_ptr, segment_size)) + return false; + out_ptr += segment_size; + + offset_in_data += sizeof(PurgeSegment) + segment_size; + offset_in_decompressed_data = segment_offset + segment_size; + } + + if (offset_in_data != hash_offset || offset_in_decompressed_data > decompressed_data_size) return false; - *offset += bytes_to_read; - *size -= bytes_to_read; - *out_ptr += bytes_to_read; + std::memset(out_ptr, 0, decompressed_data_size - offset_in_decompressed_data); + + SHA1 expected_hash; + if (!m_file.ReadArray(&expected_hash, 1)) + return false; + + // TODO: Check hash + + return true; + } } - return true; + return false; +} + +bool WIAFileReader::ReadCompressedData(u32 decompressed_data_size, u64 data_offset, u64 data_size, + u64 offset_in_data, u64 size_in_data, u8* out_ptr, + bool exception_list) +{ + if (m_compression_type == CompressionType::None) + { + if (!m_file.Seek(data_offset, SEEK_SET)) + return false; + + if (exception_list) + { + const std::optional exception_list_size = ReadExceptionListFromFile(); + if (!exception_list_size) + return false; + + data_size -= *exception_list_size; + } + + if (!m_file.Seek(offset_in_data, SEEK_CUR) || !m_file.ReadBytes(out_ptr, size_in_data)) + return false; + + return true; + } + else + { + // TODO: Caching + std::vector buffer(decompressed_data_size); + if (!ReadCompressedData(decompressed_data_size, data_offset, data_size, buffer.data(), + exception_list)) + { + return false; + } + std::memcpy(out_ptr, buffer.data() + offset_in_data, size_in_data); + return true; + } +} + +std::optional WIAFileReader::ReadExceptionListFromFile() +{ + u16 exceptions; + if (!m_file.ReadArray(&exceptions, 1)) + return std::nullopt; + + const u64 exception_list_size = Common::AlignUp( + sizeof(exceptions) + Common::swap16(exceptions) * sizeof(HashExceptionEntry), 4); + + if (!m_file.Seek(exception_list_size - sizeof(exceptions), SEEK_CUR)) + return std::nullopt; + + // TODO: Actually handle the exceptions + + return exception_list_size; } std::string WIAFileReader::VersionToString(u32 version) diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index fd00520dd1f2..b355437d5b13 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -6,6 +6,7 @@ #include #include +#include #include "Common/CommonTypes.h" #include "Common/File.h" @@ -43,6 +44,13 @@ class WIAFileReader : public BlobReader bool ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size, u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups, bool exception_list); + bool ReadCompressedData(u32 decompressed_data_size, u64 data_offset, u64 data_size, u8* out_ptr, + bool exception_list); + bool ReadCompressedData(u32 decompressed_data_size, u64 data_offset, u64 data_size, + u64 offset_in_data, u64 size_in_data, u8* out_ptr, bool exception_list); + + // Returns the number of bytes read + std::optional ReadExceptionListFromFile(); static std::string VersionToString(u32 version); @@ -128,9 +136,26 @@ class WIAFileReader : public BlobReader SHA1 hash; }; static_assert(sizeof(HashExceptionEntry) == 0x16, "Wrong size for WIA hash exception entry"); + + struct PurgeSegment + { + u32 offset; + u32 size; + }; + static_assert(sizeof(PurgeSegment) == 0x08, "Wrong size for WIA purge segment"); #pragma pack(pop) + enum class CompressionType : u32 + { + None = 0, + Purge = 1, + Bzip2 = 2, + LZMA = 3, + LZMA2 = 4, + }; + bool m_valid; + CompressionType m_compression_type; File::IOFile m_file; From 3c373c8aa82a6178993f8181c4621540141eb49e Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 3 Jan 2020 23:27:39 +0100 Subject: [PATCH 05/36] WIA: Treat groups with size 0 as containing only zeroes --- Source/Core/DiscIO/WIABlob.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 6c56af5ad10e..c6c1cae22ee2 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -246,12 +246,20 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu const u64 group_offset = data_offset + i * chunk_size; const u64 offset_in_group = *offset - group_offset; - const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; const u64 bytes_to_read = std::min(chunk_size - offset_in_group, *size); - if (!ReadCompressedData(chunk_size, group_offset_in_file, Common::swap32(group.data_size), - offset_in_group, bytes_to_read, *out_ptr, exception_list)) + const u32 group_data_size = Common::swap32(group.data_size); + if (group_data_size == 0) { - return false; + std::memset(*out_ptr, 0, bytes_to_read); + } + else + { + const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; + if (!ReadCompressedData(chunk_size, group_offset_in_file, group_data_size, offset_in_group, + bytes_to_read, *out_ptr, exception_list)) + { + return false; + } } *offset += bytes_to_read; From 1579e061a3111b638c08e927b6416995ac75ec43 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 3 Jan 2020 23:57:35 +0100 Subject: [PATCH 06/36] WIA: Correctly handle data with size not divisible by chunk size --- Source/Core/DiscIO/WIABlob.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index c6c1cae22ee2..692462b2af47 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -246,6 +246,8 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu const u64 group_offset = data_offset + i * chunk_size; const u64 offset_in_group = *offset - group_offset; + chunk_size = std::min(chunk_size, data_offset + data_size - group_offset); + const u64 bytes_to_read = std::min(chunk_size - offset_in_group, *size); const u32 group_data_size = Common::swap32(group.data_size); if (group_data_size == 0) From b59ef81a7efa7e0190cf12c3b792ed4a492cbd72 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Tue, 31 Dec 2019 17:17:11 +0100 Subject: [PATCH 07/36] WIA: Implement bzip2, LZMA, and LZMA2 decompression --- Source/Core/DiscIO/CMakeLists.txt | 4 + Source/Core/DiscIO/DiscIO.vcxproj | 6 + Source/Core/DiscIO/WIABlob.cpp | 201 +++++++++++++++++++++++++++++- Source/Core/DiscIO/WIABlob.h | 59 +++++++++ 4 files changed, 269 insertions(+), 1 deletion(-) diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index 1e234bea4d1c..46d66d217dc3 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -51,6 +51,10 @@ add_library(discio ) target_link_libraries(discio +PUBLIC + BZip2::BZip2 + LibLZMA::LibLZMA + PRIVATE minizip pugixml diff --git a/Source/Core/DiscIO/DiscIO.vcxproj b/Source/Core/DiscIO/DiscIO.vcxproj index 9c2c9f7954a4..7d94fcff278b 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj +++ b/Source/Core/DiscIO/DiscIO.vcxproj @@ -112,6 +112,12 @@ {38fee76f-f347-484b-949c-b4649381cffb} + + {055a775f-b4f5-4970-9240-f6cf7661f37b} + + + {1d8c51d2-ffa4-418e-b183-9f42b6a6717e} + diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 692462b2af47..d0f3eb490b68 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -10,6 +10,9 @@ #include #include +#include +#include + #include "Common/Align.h" #include "Common/CommonTypes.h" #include "Common/File.h" @@ -74,7 +77,7 @@ bool WIAFileReader::Initialize(const std::string& path) const u32 compression_type = Common::swap32(m_header_2.compression_type); m_compression_type = static_cast(compression_type); - if (m_compression_type > CompressionType::Purge) + if (m_compression_type > CompressionType::LZMA2) { ERROR_LOG(DISCIO, "Unsupported WIA compression type %u in %s", compression_type, path.c_str()); return false; @@ -338,6 +341,60 @@ bool WIAFileReader::ReadCompressedData(u32 decompressed_data_size, u64 data_offs return true; } + + case CompressionType::Bzip2: + case CompressionType::LZMA: + case CompressionType::LZMA2: + { + std::vector compressed_data(data_size); + if (!m_file.Seek(data_offset, SEEK_SET) || !m_file.ReadBytes(compressed_data.data(), data_size)) + return false; + + std::unique_ptr decompressor; + switch (m_compression_type) + { + case CompressionType::Bzip2: + decompressor = std::make_unique(); + break; + case CompressionType::LZMA: + decompressor = std::make_unique(false, m_header_2.compressor_data, + m_header_2.compressor_data_size); + break; + case CompressionType::LZMA2: + decompressor = std::make_unique(true, m_header_2.compressor_data, + m_header_2.compressor_data_size); + break; + } + + if (!decompressor->Start(compressed_data.data(), compressed_data.size())) + return false; + + if (exception_list) + { + u16 exceptions; + if (decompressor->Read(reinterpret_cast(&exceptions), sizeof(exceptions)) != + sizeof(exceptions)) + { + return false; + } + + std::vector exception_entries(Common::swap16(exceptions)); + u8* exceptions_data = reinterpret_cast(exception_entries.data()); + const size_t exceptions_size = exception_entries.size() * sizeof(HashExceptionEntry); + if (decompressor->Read(exceptions_data, exceptions_size) != exceptions_size) + return false; + + // TODO: Actually handle the exceptions + } + + if (decompressor->Read(out_ptr, decompressed_data_size) != decompressed_data_size) + return false; + + if (!decompressor->DoneReading()) + return false; + + return true; + } } return false; @@ -409,4 +466,146 @@ std::string WIAFileReader::VersionToString(u32 version) else return StringFromFormat("%u.%02x.%02x.beta%u", a, b, c, d); } + +WIAFileReader::Decompressor::~Decompressor() = default; + +WIAFileReader::Bzip2Decompressor::~Bzip2Decompressor() +{ + End(); +} + +bool WIAFileReader::Bzip2Decompressor::Start(const u8* in_ptr, u64 size) +{ + if (m_started) + return false; + + m_stream.bzalloc = nullptr; + m_stream.bzfree = nullptr; + m_stream.opaque = nullptr; + + m_started = BZ2_bzDecompressInit(&m_stream, 0, 0) == BZ_OK; + + m_stream.next_in = reinterpret_cast(const_cast(in_ptr)); + m_stream.avail_in = size; + + return m_started; +} + +u64 WIAFileReader::Bzip2Decompressor::Read(u8* out_ptr, u64 size) +{ + if (!m_started || m_error_occurred || m_stream.avail_in == 0) + return 0; + + m_stream.next_out = reinterpret_cast(out_ptr); + m_stream.avail_out = size; + + const int result = BZ2_bzDecompress(&m_stream); + m_error_occurred = result != BZ_OK && result != BZ_STREAM_END; + + return m_error_occurred ? 0 : m_stream.next_out - reinterpret_cast(out_ptr); +} + +bool WIAFileReader::Bzip2Decompressor::DoneReading() const +{ + return m_started && !m_error_occurred && m_stream.avail_in == 0; +} + +void WIAFileReader::Bzip2Decompressor::End() +{ + if (m_started && !m_ended) + { + BZ2_bzDecompressEnd(&m_stream); + m_ended = true; + } +} + +WIAFileReader::LZMADecompressor::LZMADecompressor(bool lzma2, const u8* filter_options, + size_t filter_options_size) +{ + m_options.preset_dict = nullptr; + + if (!lzma2 && filter_options_size == 5) + { + // The dictionary size is stored as a 32-bit little endian unsigned integer + static_assert(sizeof(m_options.dict_size) == sizeof(u32)); + std::memcpy(&m_options.dict_size, filter_options + 1, sizeof(u32)); + + const u8 d = filter_options[0]; + if (d >= (9 * 5 * 5)) + { + m_error_occurred = true; + } + else + { + m_options.lc = d % 9; + const u8 e = d / 9; + m_options.pb = e / 5; + m_options.lp = e % 5; + } + } + else if (lzma2 && filter_options_size == 1) + { + const u8 d = filter_options[0]; + if (d > 40) + m_error_occurred = true; + else + m_options.dict_size = d == 40 ? 0xFFFFFFFF : (static_cast(2) | (d & 1)) << (d / 2 + 11); + } + else + { + m_error_occurred = true; + } + + m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1; + m_filters[0].options = &m_options; + m_filters[1].id = LZMA_VLI_UNKNOWN; + m_filters[1].options = nullptr; +} + +WIAFileReader::LZMADecompressor::~LZMADecompressor() +{ + End(); +} + +bool WIAFileReader::LZMADecompressor::Start(const u8* in_ptr, u64 size) +{ + if (m_started || m_error_occurred) + return false; + + m_started = lzma_raw_decoder(&m_stream, m_filters) == LZMA_OK; + + m_stream.next_in = in_ptr; + m_stream.avail_in = size; + + return m_started; +} + +u64 WIAFileReader::LZMADecompressor::Read(u8* out_ptr, u64 size) +{ + if (!m_started || m_error_occurred || m_stream.avail_in == 0) + return 0; + + m_stream.next_out = out_ptr; + m_stream.avail_out = size; + + const lzma_ret result = lzma_code(&m_stream, LZMA_RUN); + m_error_occurred = result != LZMA_OK && result != LZMA_STREAM_END; + + return m_error_occurred ? 0 : m_stream.next_out - out_ptr; +} + +bool WIAFileReader::LZMADecompressor::DoneReading() const +{ + return m_started && !m_error_occurred && m_stream.avail_in == 0; +} + +void WIAFileReader::LZMADecompressor::End() +{ + if (m_started && !m_ended) + { + lzma_end(&m_stream); + m_ended = true; + } +} + } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index b355437d5b13..e7198e24679a 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -8,6 +8,9 @@ #include #include +#include +#include + #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/Swap.h" @@ -154,6 +157,62 @@ class WIAFileReader : public BlobReader LZMA2 = 4, }; + class Decompressor + { + public: + virtual ~Decompressor(); + + // Specifies the compressed data to read. The data must still be in memory when calling Read. + virtual bool Start(const u8* in_ptr, u64 size) = 0; + + // Reads the specified number of bytes into out_ptr (or less, if there aren't that many bytes + // to output). Returns the number of bytes read. Start must be called before this. + virtual u64 Read(u8* out_ptr, u64 size) = 0; + + // Returns whether every byte of the input data has been read. + virtual bool DoneReading() const = 0; + + // Will be called automatically upon destruction, but can be called earlier if desired. + virtual void End() = 0; + }; + + class Bzip2Decompressor final : public Decompressor + { + public: + ~Bzip2Decompressor(); + + bool Start(const u8* in_ptr, u64 size) override; + u64 Read(u8* out_ptr, u64 size) override; + bool DoneReading() const override; + void End() override; + + private: + bz_stream m_stream; + bool m_started = false; + bool m_ended = false; + bool m_error_occurred = false; + }; + + class LZMADecompressor final : public Decompressor + { + public: + LZMADecompressor(bool lzma2, const u8* filter_options, size_t filter_options_size); + ~LZMADecompressor(); + + bool Start(const u8* in_ptr, u64 size) override; + u64 Read(u8* out_ptr, u64 size) override; + bool DoneReading() const override; + void End() override; + + private: + lzma_stream m_stream = LZMA_STREAM_INIT; + lzma_options_lzma m_options = {}; + lzma_filter m_filters[2]; + bool m_started = false; + bool m_ended = false; + bool m_error_occurred = false; + }; + bool m_valid; CompressionType m_compression_type; From 01a77ae8a133bd9ac4969df8f94b3fa42e498feb Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 5 Jan 2020 22:26:28 +0100 Subject: [PATCH 08/36] WIA: Implement caching and partial decompression --- Source/Core/DiscIO/WIABlob.cpp | 509 ++++++++++++++++++--------------- Source/Core/DiscIO/WIABlob.h | 130 ++++++--- 2 files changed, 371 insertions(+), 268 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index d0f3eb490b68..d58543cdb328 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -122,13 +123,12 @@ bool WIAFileReader::Initialize(const std::string& path) const u32 number_of_raw_data_entries = Common::swap32(m_header_2.number_of_raw_data_entries); m_raw_data_entries.resize(number_of_raw_data_entries); - if (!ReadCompressedData(number_of_raw_data_entries * sizeof(RawDataEntry), - Common::swap64(m_header_2.raw_data_entries_offset), - Common::swap32(m_header_2.raw_data_entries_size), - reinterpret_cast(m_raw_data_entries.data()), false)) - { + Chunk& raw_data_entries = + ReadCompressedData(Common::swap64(m_header_2.raw_data_entries_offset), + Common::swap32(m_header_2.raw_data_entries_size), + number_of_raw_data_entries * sizeof(RawDataEntry), false); + if (!raw_data_entries.ReadAll(&m_raw_data_entries)) return false; - } std::sort(m_raw_data_entries.begin(), m_raw_data_entries.end(), [](const RawDataEntry& a, const RawDataEntry& b) { @@ -137,13 +137,11 @@ bool WIAFileReader::Initialize(const std::string& path) const u32 number_of_group_entries = Common::swap32(m_header_2.number_of_group_entries); m_group_entries.resize(number_of_group_entries); - if (!ReadCompressedData(number_of_group_entries * sizeof(GroupEntry), - Common::swap64(m_header_2.group_entries_offset), - Common::swap32(m_header_2.group_entries_size), - reinterpret_cast(m_group_entries.data()), false)) - { + Chunk& group_entries = ReadCompressedData(Common::swap64(m_header_2.group_entries_offset), + Common::swap32(m_header_2.group_entries_size), + number_of_group_entries * sizeof(GroupEntry), false); + if (!group_entries.ReadAll(&m_group_entries)) return false; - } return true; } @@ -260,9 +258,11 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu else { const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; - if (!ReadCompressedData(chunk_size, group_offset_in_file, group_data_size, offset_in_group, - bytes_to_read, *out_ptr, exception_list)) + Chunk& chunk = + ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, exception_list); + if (!chunk.Read(offset_in_group, bytes_to_read, *out_ptr)) { + m_cached_chunk_offset = std::numeric_limits::max(); // Invalidate the cache return false; } } @@ -275,248 +275,193 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu return true; } -bool WIAFileReader::ReadCompressedData(u32 decompressed_data_size, u64 data_offset, u64 data_size, - u8* out_ptr, bool exception_list) +WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 compressed_size, + u64 decompressed_size, bool exception_list) { + if (offset_in_file == m_cached_chunk_offset) + return m_cached_chunk; + + std::unique_ptr decompressor; switch (m_compression_type) { case CompressionType::None: - { - return ReadCompressedData(decompressed_data_size, data_offset, data_size, 0, - decompressed_data_size, out_ptr, exception_list); - } - + decompressor = std::make_unique(); + break; case CompressionType::Purge: - { - if (!m_file.Seek(data_offset, SEEK_SET)) - return false; - - if (exception_list) - { - const std::optional exception_size = ReadExceptionListFromFile(); - if (!exception_size) - return false; - - data_size -= *exception_size; - } - - const u64 hash_offset = data_size - sizeof(SHA1); - u32 offset_in_data = 0; - u32 offset_in_decompressed_data = 0; - - while (offset_in_data < hash_offset) - { - PurgeSegment purge_segment; - if (!m_file.ReadArray(&purge_segment, 1)) - return false; + decompressor = std::make_unique(decompressed_size); + break; + case CompressionType::Bzip2: + decompressor = std::make_unique(); + break; + case CompressionType::LZMA: + decompressor = std::make_unique(false, m_header_2.compressor_data, + m_header_2.compressor_data_size); + break; + case CompressionType::LZMA2: + decompressor = std::make_unique(true, m_header_2.compressor_data, + m_header_2.compressor_data_size); + break; + } - const u32 segment_offset = Common::swap32(purge_segment.offset); - const u32 segment_size = Common::swap32(purge_segment.size); + const bool compressed_exception_list = m_compression_type > CompressionType::Purge; - if (segment_offset < offset_in_decompressed_data) - return false; + m_cached_chunk = Chunk(&m_file, offset_in_file, compressed_size, decompressed_size, + exception_list, compressed_exception_list, std::move(decompressor)); + m_cached_chunk_offset = offset_in_file; + return m_cached_chunk; +} - const u32 blank_bytes = segment_offset - offset_in_decompressed_data; - std::memset(out_ptr, 0, blank_bytes); - out_ptr += blank_bytes; +std::string WIAFileReader::VersionToString(u32 version) +{ + const u8 a = version >> 24; + const u8 b = (version >> 16) & 0xff; + const u8 c = (version >> 8) & 0xff; + const u8 d = version & 0xff; - if (segment_size != 0 && !m_file.ReadBytes(out_ptr, segment_size)) - return false; - out_ptr += segment_size; + if (d == 0 || d == 0xff) + return StringFromFormat("%u.%02x.%02x", a, b, c); + else + return StringFromFormat("%u.%02x.%02x.beta%u", a, b, c, d); +} - offset_in_data += sizeof(PurgeSegment) + segment_size; - offset_in_decompressed_data = segment_offset + segment_size; - } +WIAFileReader::Decompressor::~Decompressor() = default; - if (offset_in_data != hash_offset || offset_in_decompressed_data > decompressed_data_size) - return false; +bool WIAFileReader::NoneDecompressor::Decompress(const DecompressionBuffer& in, + DecompressionBuffer* out, size_t* in_bytes_read) +{ + const size_t length = + std::min(in.bytes_written - *in_bytes_read, out->data.size() - out->bytes_written); - std::memset(out_ptr, 0, decompressed_data_size - offset_in_decompressed_data); + std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read, length); - SHA1 expected_hash; - if (!m_file.ReadArray(&expected_hash, 1)) - return false; + *in_bytes_read += length; + out->bytes_written += length; - // TODO: Check hash + m_done = in.data.size() == *in_bytes_read; + return true; +} - return true; - } +WIAFileReader::PurgeDecompressor::PurgeDecompressor(u64 decompressed_size) + : m_decompressed_size(decompressed_size) +{ +} - case CompressionType::Bzip2: - case CompressionType::LZMA: - case CompressionType::LZMA2: +bool WIAFileReader::PurgeDecompressor::Decompress(const DecompressionBuffer& in, + DecompressionBuffer* out, size_t* in_bytes_read) +{ + while (!m_done && in.bytes_written != *in_bytes_read && + (m_segment_bytes_written < sizeof(m_segment) || out->data.size() != out->bytes_written)) { - std::vector compressed_data(data_size); - if (!m_file.Seek(data_offset, SEEK_SET) || !m_file.ReadBytes(compressed_data.data(), data_size)) - return false; - - std::unique_ptr decompressor; - switch (m_compression_type) + if (m_segment_bytes_written == 0 && *in_bytes_read == in.data.size() - sizeof(SHA1)) { - case CompressionType::Bzip2: - decompressor = std::make_unique(); - break; - case CompressionType::LZMA: - decompressor = std::make_unique(false, m_header_2.compressor_data, - m_header_2.compressor_data_size); - break; - case CompressionType::LZMA2: - decompressor = std::make_unique(true, m_header_2.compressor_data, - m_header_2.compressor_data_size); - break; - } + const size_t zeroes_to_write = std::min(m_decompressed_size - m_out_bytes_written, + out->data.size() - out->bytes_written); - if (!decompressor->Start(compressed_data.data(), compressed_data.size())) - return false; + std::memset(out->data.data() + out->bytes_written, 0, zeroes_to_write); - if (exception_list) - { - u16 exceptions; - if (decompressor->Read(reinterpret_cast(&exceptions), sizeof(exceptions)) != - sizeof(exceptions)) + out->bytes_written += zeroes_to_write; + m_out_bytes_written += zeroes_to_write; + + if (m_out_bytes_written == m_decompressed_size) { - return false; - } + *in_bytes_read += sizeof(SHA1); + m_done = true; - std::vector exception_entries(Common::swap16(exceptions)); - u8* exceptions_data = reinterpret_cast(exception_entries.data()); - const size_t exceptions_size = exception_entries.size() * sizeof(HashExceptionEntry); - if (decompressor->Read(exceptions_data, exceptions_size) != exceptions_size) - return false; + // TODO: Check hash + } - // TODO: Actually handle the exceptions + return true; } - if (decompressor->Read(out_ptr, decompressed_data_size) != decompressed_data_size) - return false; + if (m_segment_bytes_written < sizeof(m_segment)) + { + const size_t bytes_to_copy = + std::min(in.bytes_written - *in_bytes_read, sizeof(m_segment) - m_segment_bytes_written); - if (!decompressor->DoneReading()) - return false; + std::memcpy(reinterpret_cast(&m_segment) + m_segment_bytes_written, + in.data.data() + *in_bytes_read, bytes_to_copy); - return true; - } - } + *in_bytes_read += bytes_to_copy; + m_bytes_read += bytes_to_copy; + m_segment_bytes_written += bytes_to_copy; + } - return false; -} + if (m_segment_bytes_written < sizeof(m_segment)) + return true; -bool WIAFileReader::ReadCompressedData(u32 decompressed_data_size, u64 data_offset, u64 data_size, - u64 offset_in_data, u64 size_in_data, u8* out_ptr, - bool exception_list) -{ - if (m_compression_type == CompressionType::None) - { - if (!m_file.Seek(data_offset, SEEK_SET)) - return false; + const size_t offset = Common::swap32(m_segment.offset); + const size_t size = Common::swap32(m_segment.size); - if (exception_list) + if (m_out_bytes_written < offset) { - const std::optional exception_list_size = ReadExceptionListFromFile(); - if (!exception_list_size) - return false; + const size_t zeroes_to_write = + std::min(offset - m_out_bytes_written, out->data.size() - out->bytes_written); - data_size -= *exception_list_size; - } - - if (!m_file.Seek(offset_in_data, SEEK_CUR) || !m_file.ReadBytes(out_ptr, size_in_data)) - return false; + std::memset(out->data.data() + out->bytes_written, 0, zeroes_to_write); - return true; - } - else - { - // TODO: Caching - std::vector buffer(decompressed_data_size); - if (!ReadCompressedData(decompressed_data_size, data_offset, data_size, buffer.data(), - exception_list)) - { - return false; + out->bytes_written += zeroes_to_write; + m_out_bytes_written += zeroes_to_write; } - std::memcpy(out_ptr, buffer.data() + offset_in_data, size_in_data); - return true; - } -} -std::optional WIAFileReader::ReadExceptionListFromFile() -{ - u16 exceptions; - if (!m_file.ReadArray(&exceptions, 1)) - return std::nullopt; - - const u64 exception_list_size = Common::AlignUp( - sizeof(exceptions) + Common::swap16(exceptions) * sizeof(HashExceptionEntry), 4); - - if (!m_file.Seek(exception_list_size - sizeof(exceptions), SEEK_CUR)) - return std::nullopt; + if (m_out_bytes_written >= offset && m_out_bytes_written < offset + size) + { + const size_t bytes_to_copy = std::min( + std::min(offset + size - m_out_bytes_written, out->data.size() - out->bytes_written), + in.bytes_written - *in_bytes_read); - // TODO: Actually handle the exceptions + std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read, + bytes_to_copy); - return exception_list_size; -} + *in_bytes_read += bytes_to_copy; + m_bytes_read += bytes_to_copy; + out->bytes_written += bytes_to_copy; + m_out_bytes_written += bytes_to_copy; + } -std::string WIAFileReader::VersionToString(u32 version) -{ - const u8 a = version >> 24; - const u8 b = (version >> 16) & 0xff; - const u8 c = (version >> 8) & 0xff; - const u8 d = version & 0xff; + if (m_out_bytes_written >= offset + size) + m_segment_bytes_written = 0; + } - if (d == 0 || d == 0xff) - return StringFromFormat("%u.%02x.%02x", a, b, c); - else - return StringFromFormat("%u.%02x.%02x.beta%u", a, b, c, d); + return true; } -WIAFileReader::Decompressor::~Decompressor() = default; - WIAFileReader::Bzip2Decompressor::~Bzip2Decompressor() { - End(); + if (m_started) + BZ2_bzDecompressEnd(&m_stream); } -bool WIAFileReader::Bzip2Decompressor::Start(const u8* in_ptr, u64 size) +bool WIAFileReader::Bzip2Decompressor::Decompress(const DecompressionBuffer& in, + DecompressionBuffer* out, size_t* in_bytes_read) { - if (m_started) - return false; - - m_stream.bzalloc = nullptr; - m_stream.bzfree = nullptr; - m_stream.opaque = nullptr; + if (!m_started) + { + if (BZ2_bzDecompressInit(&m_stream, 0, 0) != BZ_OK) + return false; - m_started = BZ2_bzDecompressInit(&m_stream, 0, 0) == BZ_OK; + m_started = true; + } - m_stream.next_in = reinterpret_cast(const_cast(in_ptr)); - m_stream.avail_in = size; + constexpr auto clamped_cast = [](size_t x) { + return static_cast( + std::min(std::numeric_limits().max(), x)); + }; - return m_started; -} - -u64 WIAFileReader::Bzip2Decompressor::Read(u8* out_ptr, u64 size) -{ - if (!m_started || m_error_occurred || m_stream.avail_in == 0) - return 0; + char* const in_ptr = reinterpret_cast(const_cast(in.data.data() + *in_bytes_read)); + m_stream.next_in = in_ptr; + m_stream.avail_in = clamped_cast(in.bytes_written - *in_bytes_read); - m_stream.next_out = reinterpret_cast(out_ptr); - m_stream.avail_out = size; + char* const out_ptr = reinterpret_cast(out->data.data() + out->bytes_written); + m_stream.next_out = out_ptr; + m_stream.avail_out = clamped_cast(out->data.size() - out->bytes_written); const int result = BZ2_bzDecompress(&m_stream); - m_error_occurred = result != BZ_OK && result != BZ_STREAM_END; - - return m_error_occurred ? 0 : m_stream.next_out - reinterpret_cast(out_ptr); -} -bool WIAFileReader::Bzip2Decompressor::DoneReading() const -{ - return m_started && !m_error_occurred && m_stream.avail_in == 0; -} + *in_bytes_read += m_stream.next_in - in_ptr; + out->bytes_written += m_stream.next_out - out_ptr; -void WIAFileReader::Bzip2Decompressor::End() -{ - if (m_started && !m_ended) - { - BZ2_bzDecompressEnd(&m_stream); - m_ended = true; - } + m_done = result == BZ_STREAM_END; + return result == BZ_OK || result == BZ_STREAM_END; } WIAFileReader::LZMADecompressor::LZMADecompressor(bool lzma2, const u8* filter_options, @@ -564,48 +509,162 @@ WIAFileReader::LZMADecompressor::LZMADecompressor(bool lzma2, const u8* filter_o WIAFileReader::LZMADecompressor::~LZMADecompressor() { - End(); + if (m_started) + lzma_end(&m_stream); } -bool WIAFileReader::LZMADecompressor::Start(const u8* in_ptr, u64 size) +bool WIAFileReader::LZMADecompressor::Decompress(const DecompressionBuffer& in, + DecompressionBuffer* out, size_t* in_bytes_read) { - if (m_started || m_error_occurred) - return false; + if (!m_started) + { + if (m_error_occurred || lzma_raw_decoder(&m_stream, m_filters) != LZMA_OK) + return false; - m_started = lzma_raw_decoder(&m_stream, m_filters) == LZMA_OK; + m_started = true; + } + const u8* const in_ptr = in.data.data() + *in_bytes_read; m_stream.next_in = in_ptr; - m_stream.avail_in = size; - - return m_started; -} - -u64 WIAFileReader::LZMADecompressor::Read(u8* out_ptr, u64 size) -{ - if (!m_started || m_error_occurred || m_stream.avail_in == 0) - return 0; + m_stream.avail_in = in.bytes_written - *in_bytes_read; + u8* const out_ptr = out->data.data() + out->bytes_written; m_stream.next_out = out_ptr; - m_stream.avail_out = size; + m_stream.avail_out = out->data.size() - out->bytes_written; const lzma_ret result = lzma_code(&m_stream, LZMA_RUN); - m_error_occurred = result != LZMA_OK && result != LZMA_STREAM_END; - return m_error_occurred ? 0 : m_stream.next_out - out_ptr; + *in_bytes_read += m_stream.next_in - in_ptr; + out->bytes_written += m_stream.next_out - out_ptr; + + m_done = result == LZMA_STREAM_END; + return result == LZMA_OK || result == LZMA_STREAM_END; } -bool WIAFileReader::LZMADecompressor::DoneReading() const +WIAFileReader::Chunk::Chunk() = default; + +WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, + u64 decompressed_size, bool exception_list, + bool compressed_exception_list, + std::unique_ptr decompressor) + : m_file(file), m_offset_in_file(offset_in_file), m_exception_list(exception_list), + m_compressed_exception_list(compressed_exception_list), + m_decompressor(std::move(decompressor)) { - return m_started && !m_error_occurred && m_stream.avail_in == 0; + m_in.data.resize(compressed_size); + m_out.data.resize(decompressed_size); } -void WIAFileReader::LZMADecompressor::End() +bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) { - if (m_started && !m_ended) + if (offset + size > m_out.data.size() || !m_decompressor || !m_file) + return false; + + if (m_exception_list && !m_compressed_exception_list) { - lzma_end(&m_stream); - m_ended = true; + u16 exceptions; + if (!m_file->Seek(m_offset_in_file, SEEK_SET) || !m_file->ReadArray(&exceptions, 1)) + return false; + + m_exceptions.data.resize(Common::swap16(exceptions) * sizeof(HashExceptionEntry)); + if (!m_file->ReadBytes(m_exceptions.data.data(), m_exceptions.data.size())) + return false; + m_exceptions.bytes_written = m_exceptions.data.size(); + + m_in.bytes_written = Common::AlignUp(sizeof(exceptions) + m_exceptions.data.size(), 4); + m_in_bytes_read = m_in.bytes_written; + m_exception_list = false; + + // TODO: Actually handle the exceptions } + + while (offset + size > m_out.bytes_written) + { + u64 bytes_to_read; + if (offset + size == m_out.data.size()) + { + // Read all the remaining data. + bytes_to_read = m_in.data.size() - m_in.bytes_written; + } + else + { + // Pick a suitable amount of compressed data to read. The std::min line has to + // be as it is, but the rest is a bit arbitrary and can be changed if desired. + + // The compressed data is probably not much bigger than the decompressed data. + // Add a few bytes for possible compression overhead and for the exception list. + bytes_to_read = offset + size - m_out.bytes_written + 0x100; + + // Align the access in an attempt to gain speed. But we don't actually know the + // block size of the underlying storage device, so we just use the Wii block size. + bytes_to_read = + Common::AlignUp(bytes_to_read + m_offset_in_file, VolumeWii::BLOCK_TOTAL_SIZE) - + m_offset_in_file; + + // Ensure we don't read too much. + bytes_to_read = std::min(m_in.data.size() - m_in.bytes_written, bytes_to_read); + } + + if (bytes_to_read == 0) + { + // Compressed size is larger than expected or decompressed size is smaller than expected + return false; + } + + if (!m_file->Seek(m_offset_in_file, SEEK_SET)) + return false; + if (!m_file->ReadBytes(m_in.data.data() + m_in.bytes_written, bytes_to_read)) + return false; + + m_offset_in_file += bytes_to_read; + m_in.bytes_written += bytes_to_read; + + if (m_exception_list) + { + if (m_exceptions.data.empty()) + m_exceptions.data.resize(sizeof(u16)); + + if (m_exceptions.data.size() == sizeof(u16)) + { + if (!m_decompressor->Decompress(m_in, &m_exceptions, &m_in_bytes_read)) + return false; + + if (m_exceptions.bytes_written == m_exceptions.data.size()) + { + u16 exceptions; + std::memcpy(&exceptions, m_exceptions.data.data(), sizeof(exceptions)); + m_exceptions.data.resize(Common::swap16(exceptions) * sizeof(HashExceptionEntry)); + m_exceptions.bytes_written = 0; + } + } + + if (m_exceptions.data.size() != sizeof(u16)) + { + if (!m_decompressor->Decompress(m_in, &m_exceptions, &m_in_bytes_read)) + return false; + + if (m_exceptions.bytes_written == m_exceptions.data.size()) + m_exception_list = false; + + // TODO: Actually handle the exceptions + } + } + + if (!m_exception_list) + { + if (!m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read)) + return false; + + if (m_out.bytes_written == m_out.data.size() && !m_decompressor->Done()) + return false; // Decompressed size is larger than expected + + if (m_decompressor->Done() && m_in_bytes_read != m_in.data.size()) + return false; // Compressed size is smaller than expected + } + } + + std::memcpy(out_ptr, m_out.data.data() + offset, size); + return true; } } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index e7198e24679a..205c48e127a7 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include @@ -41,25 +42,18 @@ class WIAFileReader : public BlobReader bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override; private: - explicit WIAFileReader(File::IOFile file, const std::string& path); - bool Initialize(const std::string& path); - - bool ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size, - u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups, - bool exception_list); - bool ReadCompressedData(u32 decompressed_data_size, u64 data_offset, u64 data_size, u8* out_ptr, - bool exception_list); - bool ReadCompressedData(u32 decompressed_data_size, u64 data_offset, u64 data_size, - u64 offset_in_data, u64 size_in_data, u8* out_ptr, bool exception_list); - - // Returns the number of bytes read - std::optional ReadExceptionListFromFile(); - - static std::string VersionToString(u32 version); - using SHA1 = std::array; using WiiKey = std::array; + enum class CompressionType : u32 + { + None = 0, + Purge = 1, + Bzip2 = 2, + LZMA = 3, + LZMA2 = 4, + }; + #pragma pack(push, 1) struct WIAHeader1 { @@ -148,13 +142,10 @@ class WIAFileReader : public BlobReader static_assert(sizeof(PurgeSegment) == 0x08, "Wrong size for WIA purge segment"); #pragma pack(pop) - enum class CompressionType : u32 + struct DecompressionBuffer { - None = 0, - Purge = 1, - Bzip2 = 2, - LZMA = 3, - LZMA2 = 4, + std::vector data; + size_t bytes_written = 0; }; class Decompressor @@ -162,18 +153,36 @@ class WIAFileReader : public BlobReader public: virtual ~Decompressor(); - // Specifies the compressed data to read. The data must still be in memory when calling Read. - virtual bool Start(const u8* in_ptr, u64 size) = 0; + virtual bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) = 0; + virtual bool Done() const { return m_done; }; + + protected: + bool m_done = false; + }; - // Reads the specified number of bytes into out_ptr (or less, if there aren't that many bytes - // to output). Returns the number of bytes read. Start must be called before this. - virtual u64 Read(u8* out_ptr, u64 size) = 0; + class NoneDecompressor final : public Decompressor + { + public: + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; + }; - // Returns whether every byte of the input data has been read. - virtual bool DoneReading() const = 0; + // This class assumes that more bytes won't be added to in once in.bytes_written == in.data.size() + class PurgeDecompressor final : public Decompressor + { + public: + PurgeDecompressor(u64 decompressed_size); + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; - // Will be called automatically upon destruction, but can be called earlier if desired. - virtual void End() = 0; + private: + PurgeSegment m_segment = {}; + size_t m_bytes_read = 0; + size_t m_segment_bytes_written = 0; + size_t m_out_bytes_written = 0; + + const u64 m_decompressed_size; }; class Bzip2Decompressor final : public Decompressor @@ -181,16 +190,12 @@ class WIAFileReader : public BlobReader public: ~Bzip2Decompressor(); - bool Start(const u8* in_ptr, u64 size) override; - u64 Read(u8* out_ptr, u64 size) override; - bool DoneReading() const override; - void End() override; + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; private: - bz_stream m_stream; + bz_stream m_stream = {}; bool m_started = false; - bool m_ended = false; - bool m_error_occurred = false; }; class LZMADecompressor final : public Decompressor @@ -199,24 +204,63 @@ class WIAFileReader : public BlobReader LZMADecompressor(bool lzma2, const u8* filter_options, size_t filter_options_size); ~LZMADecompressor(); - bool Start(const u8* in_ptr, u64 size) override; - u64 Read(u8* out_ptr, u64 size) override; - bool DoneReading() const override; - void End() override; + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; private: lzma_stream m_stream = LZMA_STREAM_INIT; lzma_options_lzma m_options = {}; lzma_filter m_filters[2]; bool m_started = false; - bool m_ended = false; bool m_error_occurred = false; }; + class Chunk + { + public: + Chunk(); + Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size, + bool exception_list, bool compressed_exception_list, + std::unique_ptr decompressor); + + bool Read(u64 offset, u64 size, u8* out_ptr); + + template + bool ReadAll(std::vector* vector) + { + return Read(0, vector->size() * sizeof(T), reinterpret_cast(vector->data())); + } + + private: + DecompressionBuffer m_in; + DecompressionBuffer m_out; + DecompressionBuffer m_exceptions; + size_t m_in_bytes_read = 0; + + std::unique_ptr m_decompressor = nullptr; + File::IOFile* m_file = nullptr; + u64 m_offset_in_file = 0; + bool m_exception_list = false; + bool m_compressed_exception_list = false; + }; + + explicit WIAFileReader(File::IOFile file, const std::string& path); + bool Initialize(const std::string& path); + + bool ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size, + u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups, + bool exception_list); + Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size, + bool exception_list); + + static std::string VersionToString(u32 version); + bool m_valid; CompressionType m_compression_type; File::IOFile m_file; + Chunk m_cached_chunk; + u64 m_cached_chunk_offset = std::numeric_limits::max(); WIAHeader1 m_header_1; WIAHeader2 m_header_2; From 0b407228b7a27fe4cd5600db910d677bba895de1 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Mon, 6 Jan 2020 22:34:11 +0100 Subject: [PATCH 09/36] WIA: Add documentation --- Source/Core/DiscIO/WIABlob.h | 2 + docs/WIA.md | 170 +++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 docs/WIA.md diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 205c48e127a7..5fcdc7e24c3b 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -54,6 +54,8 @@ class WIAFileReader : public BlobReader LZMA2 = 4, }; + // See docs/WIA.md for details about the format + #pragma pack(push, 1) struct WIAHeader1 { diff --git a/docs/WIA.md b/docs/WIA.md new file mode 100644 index 000000000000..894078cd52ca --- /dev/null +++ b/docs/WIA.md @@ -0,0 +1,170 @@ +# WIA file format description + +This document describes the WIA disc image format, version 1.00, as implemented in wit v2.40a. A few notes about Dolphin's implementation of the format are also included, where Dolphin differs from wit. The unique features of WIA compared to older formats like GCZ are: + + - Support for the compression algorithms bzip2, LZMA, and LZMA2 + - Wii partition data is stored decrypted and without hashes, making it compressible + +Like essentially all compressed GC/Wii disc image formats, WIA divides the data into blocks (called chunks in wit). Each chunk is compressed separately, making random access of compressed data possible. + +The struct names and variable names below are taken directly from wit. Data in WIA files can be stored in any order unless otherwise noted. All integers are big endian unless otherwise noted. The type `sha1_hash_t` refers to an array of 20 bytes. + +## `wia_file_head_t` + +This struct is stored at offset 0x0 and is 0x48 bytes long. The wit source code says its format will never be changed. + +A short note from the wit source code about how version numbers are encoded: + +``` +//----------------------------------------------------- +// Format of version number: AABBCCDD = A.BB | A.BB.CC +// If D != 0x00 && D != 0xff => append: 'beta' D +//----------------------------------------------------- +``` + +|Type and name|Description| +|--|--| +|`char magic[4]`|Always contains `"WIA\x1"`.| +|`u32 version`|The WIA format version.| +|`u32 version_compatible`|If the reading program supports the version of WIA indicated here, it can read the file. `version` can be higher than `version_compatible` (wit v2.40a sets the former to `0x01000000` and the latter to `0x00090000`).| +|`u32 disc_size`|The size of the `wia_disc_t` struct. wit v2.40a always includes the full 7 bytes of `compr_data` when writing this.| +|`sha1_hash_t disc_hash`|The SHA-1 hash of the `wia_disc_t` struct. The number of bytes to hash is determined by `disc_size`. For instance, you may have to hash all 7 bytes of `compr_data` regardless of what `compr_data_len` says.| +|`u64 iso_file_size`|The original size of the disc (or in other words, the size of the ISO file that has the same contents as this WIA file).| +|`u64 wia_file_size`|The size of this file.| +|`sha1_hash_t file_head_hash`|The SHA-1 hash of this struct, up to but not including `file_head_hash` itself.| + +## `wia_disc_t` + +This struct is stored at offset 0x48, immediately after `wia_file_head_t`. + +|Type and name|Description| +|--|--| +|`u32 disc_type`|wit sets this to 0 for "unknown" (does this ever happen in practice?), 1 for GameCube discs, 2 for Wii discs.| +|`u32 compression`|0 for NONE, 1 for PURGE (see the `wia_exception_t` section), 2 for BZIP2, 3 for LZMA, 4 for LZMA2. +|`u32 compr_level`|The compression level used by the compressor. The possible values are compressor-specific. For informational purposes only.| +|`u32 chunk_size`|The size of the chunks that data is divided into. Must be a multiple of 2 MiB.| +|`u8 dhead[0x80]`|The first 0x80 bytes of the disc image. +|`u32 n_part`|The number of `wia_part_t` structs.| +|`u32 part_t_size`|The size of one `wia_part_t` struct. If this is smaller than `sizeof(wia_part_t)`, fill the missing bytes with `0x00`.| +|`u64 part_off`|The offset in the file where the `wia_part_t` structs are stored (uncompressed).| +|`sha1_hash_t part_hash`|The SHA-1 hash of the `wia_part_t` structs. The number of bytes to hash is determined by `n_part * part_t_size`.| +|`u32 n_raw_data`|The number of `wia_raw_data_t` structs.| +|`u64 raw_data_off`|The offset in the file where the `wia_raw_data_t` structs are stored (compressed).| +|`u32 raw_data_size`|The total compressed size of the `wia_raw_data_t` structs.| +|`u32 n_groups`|The number of `wia_group_t` structs.| +|`u64 group_off`|The offset in the file where the `wia_group_t` structs are stored (compressed).| +|`u32 group_size`|The total compressed size of the `wia_group_t` structs.| +|`u8 compr_data_len`|The number of used bytes in the `compr_data` array.| +|`u8 compr_data[7]`|Compressor specific data (see below).| + +If the compression method is NONE, PURGE or BZIP2, `compr_data_len`is 0. If the compression method is LZMA or LZMA2, the compressor specific data is stored in the format used by the 7-Zip SDK. It needs to be converted if you are using e.g. liblzma. + +For LZMA, the data is 5 bytes long. The first byte encodes the `lc`, `pb`, and `lp` parameters, and the four other bytes encode the dictionary size in little endian. The first byte can be decoded as follows (code from the 7-Zip SDK): + +``` +d = data[0]; +if (d >= (9 * 5 * 5)) + return SZ_ERROR_UNSUPPORTED; + +p->lc = d % 9; +d /= 9; +p->pb = d / 5; +p->lp = d % 5; +``` + +For LZMA2, the data consists of a single byte that encodes the dictionary size. It can be decoded as follows (code from the 7-Zip SDK): + +``` +#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) + +if (prop > 40) + return SZ_ERROR_UNSUPPORTED; +dicSize = (prop == 40) ? 0xFFFFFFFF : LZMA2_DIC_SIZE_FROM_PROP(prop); +``` + +Preset dictionaries are not used for any compression method. + +## `wia_part_data_t` + +|Type and name|Description| +|--|--| +|`u32 first_sector`|The sector on the disc at which this data starts. One sector is 32 KiB (or 31 KiB excluding hashes).| +|`u32 n_sectors`|The number of sectors on the disc covered by this struct. One sector is 32 KiB (or 31 KiB excluding hashes).| +|`u32 group_index`|The index of the first `wia_group_t` struct that points to the data covered by this struct. The other `wia_group_t` indices follow sequentially.| +|`u32 n_groups`|The number of `wia_group_t` structs used for this data.| + +## `wia_part_t` + +This struct is used for keeping track of Wii partition data that on the actual disc is encrypted and hashed. This does not include the unencrypted area at the beginning of partitions that contains the ticket, TMD, certificate chain, and H3 table. So for a typical game partition, `pd[0].first_sector * 0x8000` would be 0x0F820000, not 0x0F800000. + +Wii partition data is stored decrypted and with hashes removed. For each 0x8000 bytes on the disc, 0x7C00 bytes are stored in the WIA file (prior to compression). If the hashes are desired, the reading program must first recalculate the hashes as done when creating a Wii disc image from scratch (see https://wiibrew.org/wiki/Wii_Disc), and must then apply the hash exceptions which are stored along with the data (see the `wia_except_list_t` section). + +|Type and name|Description| +|--|--| +|`u8 part_key[16]`|The title key for this partition (128-bit AES), which can be used for re-encrypting the partition data. This key can be used directly, without decrypting it using the Wii common key.| +|`wia_part_data_t pd[2]`|To quote the wit source code: `segment 0 is small and defined for management data (boot .. fst). segment 1 takes the remaining data`. The point at which wit splits the two segments is the FST end offset rounded up to the next 2 MiB. Giving the first segment a size which is not a multiple of 2 MiB is likely a bad idea (unless the second segment has a size of 0).| + +## `wia_raw_data_t` + +This struct is used for keeping track of disc data that is not stored as `wia_part_t`. The data is stored as is (other than compression being applied). + +The first `wia_raw_data_t` has `raw_data_off` set to 0x80 and `raw_data_size` set to 0x4FF80, but despite this, it actually contains 0x50000 bytes of data. (However, the first 0x80 bytes should be read from `wia_disc_t` instead.) This should be handled by rounding the offset down to the previous multiple of 0x8000 (and adding the equivalent amount to the size so that the end offset stays the same), not by special casing the first `wia_raw_data_t`. + +|Type and name|Description| +|--|--| +|`u64 raw_data_off`|The offset on the disc at which this data starts.| +|`u64 raw_data_size`|The number of bytes on the disc covered by this struct.| +|`u32 group_index`|The index of the first `wia_group_t` struct that points to the data covered by this struct. The other `wia_group_t` indices follow sequentially.| +|`u32 n_groups`|The number of `wia_group_t` structs used for this data.| + +## `wia_group_t` + +This struct points directly to the actual disc data, stored compressed. The data is interpreted differently depending on whether the `wia_group_t` is referenced by a `wia_part_data_t` or a `wia_raw_data_t` (see the `wia_part_t` section for details). + +A `wia_group_t` normally contains `chunk_size` bytes of decompressed data (or `chunk_size / 0x8000 * 0x7C00` for Wii partition data when not counting hashes), not counting any `wia_except_list_t` structs. However, the last `wia_group_t` of a `wia_part_data_t` or `wia_raw_data_t` contains less data than that if `n_sectors * 0x8000` (for `wia_part_data_t`) or `raw_data_size` (for `wia_raw_data_t`) is not evenly divisible by `chunk_size`. + +|Type and name|Description| +|--|--| +|`u32 data_off4`|The offset in the file where the compressed data is stored, divided by 4.| +|`u32 data_size`|The size of the compressed data, including any `wia_except_list_t` structs. 0 is a special case meaning that every byte of the decompressed data is `0x00` and the `wia_except_list_t` structs (if there are supposed to be any) contain 0 exceptions.| + +## `wia_exception_t` + +This struct represents a 20-byte difference between the recalculated hash data and the original hash data. (See also `wia_except_list_t` below.) + +When recalculating hashes for a `wia_group_t` with a size which is not evenly divisible by 2 MiB (with the size of the hashes included), the missing bytes should be treated as zeroes for the purpose of hashing. (wit's writing code seems to act as if the reading code does not assume that these missing bytes are zero, but both wit's and Dolphin's reading code treat them as zero. Dolphin's writing code assumes that the reading code treats them as zero.) + +wit's writing code only outputs `wia_exception_t` structs for mismatches in the actual hash data, not in the padding data (which normally only contains zeroes). Dolphin's writing code outputs `wia_exception_t` structs for both hash data and padding data. When Dolphin needs to write `wia_exception_t` structs for a padding area which is 32 bytes long, it writes one which covers the first 20 bytes of the padding area and one which covers the last 20 bytes of the padding area, generating 12 bytes of overlap between the `wia_exception_t` structs. + +|Type and name|Description| +|--|--| +|`u16 offset`|The offset among the hashes. The offsets `0x0000`-`0x0400` here map to the offsets `0x0000`-`0x0400` in the full 2 MiB of data, the offsets `0x0400`-`0x0800` here map to the offsets `0x8000`-`0x8400` in the full 2 MiB of data, and so on. The offsets start over at 0 for each new `wia_except_list_t`.| +|`sha1_hash_t hash`|The hash that the automatically generated hash at the given offset needs to be replaced with. The replacement should happen after calculating all hashes for the current 2 MiB of data but before encrypting the hashes.| + +## `wia_except_list_t` + +Each `wia_group_t` of Wii partition data contains one or more `wia_except_list_t` structs before the actual data, one for each 2 MiB of data in the `wia_group_t`. The number of `wia_except_list_t` structs per `wia_group_t` is always `chunk_size / 0x200000`, even for a `wia_group_t` which contains less data than normal due to it being at the end of a partition. + +For memory management reasons, programs which read WIA files might place a limit on how many exceptions there can be in a `wia_except_list_t`. Dolphin's reading code has a limit of 52×64=3328 (unless the compression method is NONE or PURGE, in which case there is no limit), which is enough to cover all hashes and all padding. wit's reading code seems to be written as if 47×64=3008 is the maximum it needs to be able to handle, which is enough to cover all hashes but not any padding. However, because wit allocates more memory than needed, it seems to be possible to exceed 3008 by some amount without problems. It should be safe for writing code to assume that reading code can handle at least 3328 exceptions per `wia_except_list_t`. + +|Type and name|Description| +|--|--| +|`u16 n_exceptions`|The number of `wia_exception_t` structs.| +|`wia_exception_t exception[n_exceptions]`|Each `wia_exception_t` describes one difference between the hashes obtained by hashing the partition data and the original hashes.| + +Somewhat ironically, there are exceptions to how `wia_except_list_t` structs are handled: + + - For the compression method PURGE, the `wia_except_list_t` structs are stored uncompressed (in other words, before the first `wia_segment_t`). For BZIP2, LZMA and LZMA2, they are compressed along with the rest of the data. + - For the compression methods NONE and PURGE, if the end offset of the last ``wia_except_list_t`` is not evenly divisible by 4, padding is inserted after it so that the data afterwards will start at a 4 byte boundary. This padding is not inserted for the other compression methods. + +## `wia_segment_t` + +This struct is used by the simple compression method PURGE, which stores runs of zeroes efficiently and stores other data as is. + +|Type and name|Description| +|--|--| +|`u32 offset`|The offset of `data` within the decompressed data. (Any `wia_except_list_t` structs are not counted as part of the decompressed data.)| +|`u32 size`|The number of bytes in `data`.| +|`u8 data[size]`|Data.| + +Each PURGE chunk contains zero or more `wia_segment_t` structs stored in order of ascending `offset`, followed by a SHA-1 hash (0x14 bytes) of the `wia_except_list_t` structs (if any) and the `wia_segment_t` structs. Bytes in the decompressed data that are not covered by any `wia_segment_t` struct are set to `0x00`. From 827437c03638ced506c717f9966272884749fbe1 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Tue, 7 Jan 2020 19:28:11 +0100 Subject: [PATCH 10/36] WIA: Fix the handling of chunk sizes larger than 2 MiB --- Source/Core/DiscIO/WIABlob.cpp | 147 +++++++++++++++++++-------------- Source/Core/DiscIO/WIABlob.h | 18 ++-- 2 files changed, 99 insertions(+), 66 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index d58543cdb328..afefc63f37d2 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -175,7 +175,7 @@ bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_TOTAL_SIZE, Common::swap64(raw_data.data_offset), Common::swap64(raw_data.data_size), Common::swap32(raw_data.group_index), - Common::swap32(raw_data.number_of_groups), false)) + Common::swap32(raw_data.number_of_groups), 0)) { return false; } @@ -210,7 +210,8 @@ bool WIAFileReader::ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 part if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_DATA_SIZE, data_offset, data_size, Common::swap32(data.group_index), - Common::swap32(data.number_of_groups), true)) + Common::swap32(data.number_of_groups), + chunk_size / VolumeWii::GROUP_DATA_SIZE)) { return false; } @@ -224,7 +225,7 @@ bool WIAFileReader::ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 part bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size, u64 data_offset, u64 data_size, u32 group_index, - u32 number_of_groups, bool exception_list) + u32 number_of_groups, u32 exception_lists) { if (data_offset + data_size <= *offset) return true; @@ -259,7 +260,7 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu { const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; Chunk& chunk = - ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, exception_list); + ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, exception_lists); if (!chunk.Read(offset_in_group, bytes_to_read, *out_ptr)) { m_cached_chunk_offset = std::numeric_limits::max(); // Invalidate the cache @@ -276,7 +277,7 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu } WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 compressed_size, - u64 decompressed_size, bool exception_list) + u64 decompressed_size, u32 exception_lists) { if (offset_in_file == m_cached_chunk_offset) return m_cached_chunk; @@ -303,10 +304,10 @@ WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 break; } - const bool compressed_exception_list = m_compression_type > CompressionType::Purge; + const bool compressed_exception_lists = m_compression_type > CompressionType::Purge; m_cached_chunk = Chunk(&m_file, offset_in_file, compressed_size, decompressed_size, - exception_list, compressed_exception_list, std::move(decompressor)); + exception_lists, compressed_exception_lists, std::move(decompressor)); m_cached_chunk_offset = offset_in_file; return m_cached_chunk; } @@ -544,41 +545,34 @@ bool WIAFileReader::LZMADecompressor::Decompress(const DecompressionBuffer& in, WIAFileReader::Chunk::Chunk() = default; WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, - u64 decompressed_size, bool exception_list, - bool compressed_exception_list, + u64 decompressed_size, u32 exception_lists, + bool compressed_exception_lists, std::unique_ptr decompressor) - : m_file(file), m_offset_in_file(offset_in_file), m_exception_list(exception_list), - m_compressed_exception_list(compressed_exception_list), + : m_file(file), m_offset_in_file(offset_in_file), m_exception_lists(exception_lists), + m_compressed_exception_lists(compressed_exception_lists), m_decompressor(std::move(decompressor)) { + constexpr size_t MAX_SIZE_PER_EXCEPTION_LIST = + Common::AlignUp(VolumeWii::BLOCK_HEADER_SIZE, sizeof(SHA1)) / sizeof(SHA1) * + VolumeWii::BLOCKS_PER_GROUP * sizeof(HashExceptionEntry) + + sizeof(u16); + + m_out_bytes_allocated_for_exceptions = + m_compressed_exception_lists ? MAX_SIZE_PER_EXCEPTION_LIST * m_exception_lists : 0; + m_in.data.resize(compressed_size); - m_out.data.resize(decompressed_size); + m_out.data.resize(decompressed_size + m_out_bytes_allocated_for_exceptions); } bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) { - if (offset + size > m_out.data.size() || !m_decompressor || !m_file) - return false; - - if (m_exception_list && !m_compressed_exception_list) + if (!m_decompressor || !m_file || + offset + size > m_out.data.size() - m_out_bytes_allocated_for_exceptions) { - u16 exceptions; - if (!m_file->Seek(m_offset_in_file, SEEK_SET) || !m_file->ReadArray(&exceptions, 1)) - return false; - - m_exceptions.data.resize(Common::swap16(exceptions) * sizeof(HashExceptionEntry)); - if (!m_file->ReadBytes(m_exceptions.data.data(), m_exceptions.data.size())) - return false; - m_exceptions.bytes_written = m_exceptions.data.size(); - - m_in.bytes_written = Common::AlignUp(sizeof(exceptions) + m_exceptions.data.size(), 4); - m_in_bytes_read = m_in.bytes_written; - m_exception_list = false; - - // TODO: Actually handle the exceptions + return false; } - while (offset + size > m_out.bytes_written) + while (offset + size > m_out.bytes_written - m_out_bytes_used_for_exceptions) { u64 bytes_to_read; if (offset + size == m_out.data.size()) @@ -592,8 +586,9 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) // be as it is, but the rest is a bit arbitrary and can be changed if desired. // The compressed data is probably not much bigger than the decompressed data. - // Add a few bytes for possible compression overhead and for the exception list. - bytes_to_read = offset + size - m_out.bytes_written + 0x100; + // Add a few bytes for possible compression overhead and for any hash exceptions. + bytes_to_read = + offset + size - (m_out.bytes_written - m_out_bytes_used_for_exceptions) + 0x100; // Align the access in an attempt to gain speed. But we don't actually know the // block size of the underlying storage device, so we just use the Wii block size. @@ -619,43 +614,41 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) m_offset_in_file += bytes_to_read; m_in.bytes_written += bytes_to_read; - if (m_exception_list) + if (m_exception_lists > 0 && !m_compressed_exception_lists) { - if (m_exceptions.data.empty()) - m_exceptions.data.resize(sizeof(u16)); - - if (m_exceptions.data.size() == sizeof(u16)) + if (!HandleExceptions(m_in.data.data(), m_in.data.size(), m_in.bytes_written, + &m_in_bytes_used_for_exceptions, true)) { - if (!m_decompressor->Decompress(m_in, &m_exceptions, &m_in_bytes_read)) - return false; - - if (m_exceptions.bytes_written == m_exceptions.data.size()) - { - u16 exceptions; - std::memcpy(&exceptions, m_exceptions.data.data(), sizeof(exceptions)); - m_exceptions.data.resize(Common::swap16(exceptions) * sizeof(HashExceptionEntry)); - m_exceptions.bytes_written = 0; - } + return false; } - if (m_exceptions.data.size() != sizeof(u16)) - { - if (!m_decompressor->Decompress(m_in, &m_exceptions, &m_in_bytes_read)) - return false; + m_in_bytes_read = m_in_bytes_used_for_exceptions; + } - if (m_exceptions.bytes_written == m_exceptions.data.size()) - m_exception_list = false; + if (m_exception_lists == 0 || m_compressed_exception_lists) + { + if (!m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read)) + return false; + } - // TODO: Actually handle the exceptions + if (m_exception_lists > 0 && m_compressed_exception_lists) + { + if (!HandleExceptions(m_out.data.data(), m_out_bytes_allocated_for_exceptions, + m_out.bytes_written, &m_out_bytes_used_for_exceptions, false)) + { + return false; } } - if (!m_exception_list) + if (m_exception_lists == 0) { - if (!m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read)) - return false; + const size_t expected_out_bytes = m_out.data.size() - m_out_bytes_allocated_for_exceptions + + m_out_bytes_used_for_exceptions; + + if (m_out.bytes_written > expected_out_bytes) + return false; // Decompressed size is larger than expected - if (m_out.bytes_written == m_out.data.size() && !m_decompressor->Done()) + if (m_out.bytes_written == expected_out_bytes && !m_decompressor->Done()) return false; // Decompressed size is larger than expected if (m_decompressor->Done() && m_in_bytes_read != m_in.data.size()) @@ -663,7 +656,41 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) } } - std::memcpy(out_ptr, m_out.data.data() + offset, size); + std::memcpy(out_ptr, m_out.data.data() + offset + m_out_bytes_used_for_exceptions, size); + return true; +} + +bool WIAFileReader::Chunk::HandleExceptions(const u8* data, size_t bytes_allocated, + size_t bytes_written, size_t* bytes_used, bool align) +{ + while (m_exception_lists > 0) + { + if (sizeof(u16) + *bytes_used > bytes_allocated) + { + ERROR_LOG(DISCIO, "More hash exceptions than expected"); + return false; + } + if (sizeof(u16) + *bytes_used > bytes_written) + return true; + + const u16 exceptions = Common::swap16(data + *bytes_used); + + size_t exception_list_size = exceptions * sizeof(HashExceptionEntry) + sizeof(u16); + if (align && m_exception_lists == 1) + exception_list_size = Common::AlignUp(*bytes_used + exception_list_size, 4) - *bytes_used; + + if (exception_list_size + *bytes_used > bytes_allocated) + { + ERROR_LOG(DISCIO, "More hash exceptions than expected"); + return false; + } + if (exception_list_size + *bytes_used > bytes_written) + return true; + + *bytes_used += exception_list_size; + --m_exception_lists; + } + return true; } diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 5fcdc7e24c3b..65ea5b53435c 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -222,7 +222,7 @@ class WIAFileReader : public BlobReader public: Chunk(); Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size, - bool exception_list, bool compressed_exception_list, + u32 exception_lists, bool compressed_exception_lists, std::unique_ptr decompressor); bool Read(u64 offset, u64 size, u8* out_ptr); @@ -234,16 +234,22 @@ class WIAFileReader : public BlobReader } private: + bool HandleExceptions(const u8* data, size_t bytes_allocated, size_t bytes_written, + size_t* bytes_used, bool align); + DecompressionBuffer m_in; DecompressionBuffer m_out; - DecompressionBuffer m_exceptions; size_t m_in_bytes_read = 0; std::unique_ptr m_decompressor = nullptr; File::IOFile* m_file = nullptr; u64 m_offset_in_file = 0; - bool m_exception_list = false; - bool m_compressed_exception_list = false; + + size_t m_out_bytes_allocated_for_exceptions = 0; + size_t m_out_bytes_used_for_exceptions = 0; + size_t m_in_bytes_used_for_exceptions = 0; + u32 m_exception_lists = 0; + bool m_compressed_exception_lists = false; }; explicit WIAFileReader(File::IOFile file, const std::string& path); @@ -251,9 +257,9 @@ class WIAFileReader : public BlobReader bool ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size, u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups, - bool exception_list); + u32 exception_lists); Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size, - bool exception_list); + u32 exception_lists); static std::string VersionToString(u32 version); From e3d291a5299c4f6506b88f5f2ab983744f0b34bf Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 12 Jan 2020 00:54:00 +0100 Subject: [PATCH 11/36] WIA: Check the internal WIA hashes --- Source/Core/DiscIO/WIABlob.cpp | 45 +++++++++++++++++++++++++++++++--- Source/Core/DiscIO/WIABlob.h | 7 +++++- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index afefc63f37d2..4d6d501832e4 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "Common/Align.h" #include "Common/CommonTypes.h" @@ -49,6 +50,12 @@ bool WIAFileReader::Initialize(const std::string& path) return false; } + SHA1 header_1_actual_hash; + mbedtls_sha1_ret(reinterpret_cast(&m_header_1), sizeof(m_header_1) - sizeof(SHA1), + header_1_actual_hash.data()); + if (m_header_1.header_1_hash != header_1_actual_hash) + return false; + if (Common::swap64(m_header_1.wia_file_size) != m_file.GetSize()) { ERROR_LOG(DISCIO, "File size is incorrect for %s", path.c_str()); @@ -64,6 +71,11 @@ bool WIAFileReader::Initialize(const std::string& path) if (!m_file.ReadBytes(header_2.data(), header_2.size())) return false; + SHA1 header_2_actual_hash; + mbedtls_sha1_ret(header_2.data(), header_2.size(), header_2_actual_hash.data()); + if (m_header_1.header_2_hash != header_2_actual_hash) + return false; + std::memcpy(&m_header_2, header_2.data(), std::min(header_2.size(), sizeof(WIAHeader2))); if (m_header_2.compressor_data_size > sizeof(WIAHeader2::compressor_data) || @@ -91,7 +103,12 @@ bool WIAFileReader::Initialize(const std::string& path) return false; if (!m_file.ReadBytes(partition_entries.data(), partition_entries.size())) return false; - // TODO: Check hash + + SHA1 partition_entries_actual_hash; + mbedtls_sha1_ret(reinterpret_cast(partition_entries.data()), partition_entries.size(), + partition_entries_actual_hash.data()); + if (m_header_2.partition_entries_hash != partition_entries_actual_hash) + return false; const size_t copy_length = std::min(partition_entry_size, sizeof(PartitionEntry)); const size_t memset_length = sizeof(PartitionEntry) - copy_length; @@ -345,11 +362,22 @@ bool WIAFileReader::NoneDecompressor::Decompress(const DecompressionBuffer& in, WIAFileReader::PurgeDecompressor::PurgeDecompressor(u64 decompressed_size) : m_decompressed_size(decompressed_size) { + mbedtls_sha1_init(&m_sha1_context); } bool WIAFileReader::PurgeDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, size_t* in_bytes_read) { + if (!m_started) + { + mbedtls_sha1_starts_ret(&m_sha1_context); + + // Include the exception lists in the SHA-1 calculation (but not in the compression...) + mbedtls_sha1_update_ret(&m_sha1_context, in.data.data(), *in_bytes_read); + + m_started = true; + } + while (!m_done && in.bytes_written != *in_bytes_read && (m_segment_bytes_written < sizeof(m_segment) || out->data.size() != out->bytes_written)) { @@ -363,12 +391,19 @@ bool WIAFileReader::PurgeDecompressor::Decompress(const DecompressionBuffer& in, out->bytes_written += zeroes_to_write; m_out_bytes_written += zeroes_to_write; - if (m_out_bytes_written == m_decompressed_size) + if (m_out_bytes_written == m_decompressed_size && in.bytes_written == in.data.size()) { - *in_bytes_read += sizeof(SHA1); + SHA1 actual_hash; + mbedtls_sha1_finish_ret(&m_sha1_context, actual_hash.data()); + + SHA1 expected_hash; + std::memcpy(expected_hash.data(), in.data.data() + *in_bytes_read, expected_hash.size()); + + *in_bytes_read += expected_hash.size(); m_done = true; - // TODO: Check hash + if (actual_hash != expected_hash) + return false; } return true; @@ -381,6 +416,7 @@ bool WIAFileReader::PurgeDecompressor::Decompress(const DecompressionBuffer& in, std::memcpy(reinterpret_cast(&m_segment) + m_segment_bytes_written, in.data.data() + *in_bytes_read, bytes_to_copy); + mbedtls_sha1_update_ret(&m_sha1_context, in.data.data() + *in_bytes_read, bytes_to_copy); *in_bytes_read += bytes_to_copy; m_bytes_read += bytes_to_copy; @@ -412,6 +448,7 @@ bool WIAFileReader::PurgeDecompressor::Decompress(const DecompressionBuffer& in, std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read, bytes_to_copy); + mbedtls_sha1_update_ret(&m_sha1_context, in.data.data() + *in_bytes_read, bytes_to_copy); *in_bytes_read += bytes_to_copy; m_bytes_read += bytes_to_copy; diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 65ea5b53435c..df0b841058dc 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -11,6 +11,7 @@ #include #include +#include #include "Common/CommonTypes.h" #include "Common/File.h" @@ -171,6 +172,7 @@ class WIAFileReader : public BlobReader }; // This class assumes that more bytes won't be added to in once in.bytes_written == in.data.size() + // and that *in_bytes_read initially will be equal to the size of the exception lists class PurgeDecompressor final : public Decompressor { public: @@ -179,12 +181,15 @@ class WIAFileReader : public BlobReader size_t* in_bytes_read) override; private: + const u64 m_decompressed_size; + PurgeSegment m_segment = {}; size_t m_bytes_read = 0; size_t m_segment_bytes_written = 0; size_t m_out_bytes_written = 0; + bool m_started = false; - const u64 m_decompressed_size; + mbedtls_sha1_context m_sha1_context; }; class Bzip2Decompressor final : public Decompressor From 04089f24f99d46c68a5346bcc4b09a53c144e50b Mon Sep 17 00:00:00 2001 From: JosJuice Date: Thu, 23 Jan 2020 16:47:49 +0100 Subject: [PATCH 12/36] WIA: Implement re-encryption of Wii partition data --- Source/Core/DiscIO/WIABlob.cpp | 164 +++++++++++++++++++++++---------- Source/Core/DiscIO/WIABlob.h | 19 ++++ 2 files changed, 136 insertions(+), 47 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 4d6d501832e4..d92aa8f0f106 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -23,10 +24,12 @@ #include "Common/Swap.h" #include "DiscIO/VolumeWii.h" +#include "DiscIO/WiiEncryptionCache.h" namespace DiscIO { -WIAFileReader::WIAFileReader(File::IOFile file, const std::string& path) : m_file(std::move(file)) +WIAFileReader::WIAFileReader(File::IOFile file, const std::string& path) + : m_file(std::move(file)), m_encryption_cache(this) { m_valid = Initialize(path); } @@ -120,24 +123,33 @@ bool WIAFileReader::Initialize(const std::string& path) std::memset(reinterpret_cast(&m_partition_entries[i]) + copy_length, 0, memset_length); } - for (const PartitionEntry& partition : m_partition_entries) + for (size_t i = 0; i < m_partition_entries.size(); ++i) { - if (Common::swap32(partition.data_entries[1].number_of_sectors) != 0) + const std::array& entries = m_partition_entries[i].data_entries; + + size_t non_empty_entries = 0; + for (size_t j = 0; j < entries.size(); ++j) + { + const u32 number_of_sectors = Common::swap32(entries[j].number_of_sectors); + if (number_of_sectors != 0) + { + ++non_empty_entries; + + const u32 last_sector = Common::swap32(entries[j].first_sector) + number_of_sectors; + m_data_entries.emplace(last_sector * VolumeWii::BLOCK_TOTAL_SIZE, DataEntry(i, j)); + } + } + + if (non_empty_entries > 1) { - const u32 first_end = Common::swap32(partition.data_entries[0].first_sector) + - Common::swap32(partition.data_entries[0].number_of_sectors); - const u32 second_start = Common::swap32(partition.data_entries[1].first_sector); + const u32 first_end = + Common::swap32(entries[0].first_sector) + Common::swap32(entries[0].number_of_sectors); + const u32 second_start = Common::swap32(entries[1].first_sector); if (first_end > second_start) return false; } } - std::sort(m_partition_entries.begin(), m_partition_entries.end(), - [](const PartitionEntry& a, const PartitionEntry& b) { - return Common::swap32(a.data_entries[0].first_sector) < - Common::swap32(b.data_entries[0].first_sector); - }); - const u32 number_of_raw_data_entries = Common::swap32(m_header_2.number_of_raw_data_entries); m_raw_data_entries.resize(number_of_raw_data_entries); Chunk& raw_data_entries = @@ -147,10 +159,13 @@ bool WIAFileReader::Initialize(const std::string& path) if (!raw_data_entries.ReadAll(&m_raw_data_entries)) return false; - std::sort(m_raw_data_entries.begin(), m_raw_data_entries.end(), - [](const RawDataEntry& a, const RawDataEntry& b) { - return Common::swap64(a.data_offset) < Common::swap64(b.data_offset); - }); + for (size_t i = 0; i < m_raw_data_entries.size(); ++i) + { + const RawDataEntry& entry = m_raw_data_entries[i]; + const u64 data_size = Common::swap64(entry.data_size); + if (data_size != 0) + m_data_entries.emplace(Common::swap64(entry.data_offset) + data_size, DataEntry(i)); + } const u32 number_of_group_entries = Common::swap32(m_header_2.number_of_group_entries); m_group_entries.resize(number_of_group_entries); @@ -184,21 +199,75 @@ bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) } const u32 chunk_size = Common::swap32(m_header_2.chunk_size); - for (RawDataEntry raw_data : m_raw_data_entries) + while (size > 0) { - if (size == 0) - return true; + const auto it = m_data_entries.upper_bound(offset); + if (it == m_data_entries.end()) + return false; - if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_TOTAL_SIZE, - Common::swap64(raw_data.data_offset), Common::swap64(raw_data.data_size), - Common::swap32(raw_data.group_index), - Common::swap32(raw_data.number_of_groups), 0)) + const DataEntry& data = it->second; + if (data.is_partition) { - return false; + const PartitionEntry& partition = m_partition_entries[it->second.index]; + + const u32 partition_first_sector = Common::swap32(partition.data_entries[0].first_sector); + const u64 partition_data_offset = partition_first_sector * VolumeWii::BLOCK_TOTAL_SIZE; + + const u32 second_number_of_sectors = + Common::swap32(partition.data_entries[1].number_of_sectors); + const u32 partition_total_sectors = + second_number_of_sectors ? Common::swap32(partition.data_entries[1].first_sector) - + partition_first_sector + second_number_of_sectors : + Common::swap32(partition.data_entries[0].number_of_sectors); + + for (const PartitionDataEntry& partition_data : partition.data_entries) + { + if (size == 0) + return true; + + const u32 first_sector = Common::swap32(partition_data.first_sector); + const u32 number_of_sectors = Common::swap32(partition_data.number_of_sectors); + + const u64 data_offset = first_sector * VolumeWii::BLOCK_TOTAL_SIZE; + const u64 data_size = number_of_sectors * VolumeWii::BLOCK_TOTAL_SIZE; + + if (data_size == 0) + continue; + + if (data_offset + data_size <= offset) + continue; + + if (offset < data_offset) + return false; + + const u64 bytes_to_read = std::min(data_size - (offset - data_offset), size); + + if (!m_encryption_cache.EncryptGroups( + offset - partition_data_offset, bytes_to_read, out_ptr, partition_data_offset, + partition_total_sectors * VolumeWii::BLOCK_DATA_SIZE, partition.partition_key)) + { + return false; + } + + offset += bytes_to_read; + size -= bytes_to_read; + out_ptr += bytes_to_read; + } + } + else + { + const RawDataEntry& raw_data = m_raw_data_entries[data.index]; + if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_TOTAL_SIZE, + Common::swap64(raw_data.data_offset), Common::swap64(raw_data.data_size), + Common::swap32(raw_data.group_index), + Common::swap32(raw_data.number_of_groups), 0)) + { + return false; + } } } - return size == 0; + return true; } bool WIAFileReader::SupportsReadWiiDecrypted() const @@ -210,34 +279,35 @@ bool WIAFileReader::ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 part { const u64 chunk_size = Common::swap32(m_header_2.chunk_size) * VolumeWii::BLOCK_DATA_SIZE / VolumeWii::BLOCK_TOTAL_SIZE; - for (const PartitionEntry& partition : m_partition_entries) - { - const u32 partition_first_sector = Common::swap32(partition.data_entries[0].first_sector); - if (partition_data_offset != partition_first_sector * VolumeWii::BLOCK_TOTAL_SIZE) - continue; - for (const PartitionDataEntry& data : partition.data_entries) - { - if (size == 0) - return true; + const auto it = m_data_entries.upper_bound(partition_data_offset); + if (it == m_data_entries.end() || !it->second.is_partition) + return false; - const u64 data_offset = - (Common::swap32(data.first_sector) - partition_first_sector) * VolumeWii::BLOCK_DATA_SIZE; - const u64 data_size = Common::swap32(data.number_of_sectors) * VolumeWii::BLOCK_DATA_SIZE; + const PartitionEntry& partition = m_partition_entries[it->second.index]; + const u32 partition_first_sector = Common::swap32(partition.data_entries[0].first_sector); + if (partition_data_offset != partition_first_sector * VolumeWii::BLOCK_TOTAL_SIZE) + return false; - if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_DATA_SIZE, - data_offset, data_size, Common::swap32(data.group_index), - Common::swap32(data.number_of_groups), - chunk_size / VolumeWii::GROUP_DATA_SIZE)) - { - return false; - } - } + for (const PartitionDataEntry& data : partition.data_entries) + { + if (size == 0) + return true; + + const u64 data_offset = + (Common::swap32(data.first_sector) - partition_first_sector) * VolumeWii::BLOCK_DATA_SIZE; + const u64 data_size = Common::swap32(data.number_of_sectors) * VolumeWii::BLOCK_DATA_SIZE; - return size == 0; + if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_DATA_SIZE, + data_offset, data_size, Common::swap32(data.group_index), + Common::swap32(data.number_of_groups), + chunk_size / VolumeWii::GROUP_DATA_SIZE)) + { + return false; + } } - return false; + return size == 0; } bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index df0b841058dc..2360c7acf4fa 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -17,6 +18,7 @@ #include "Common/File.h" #include "Common/Swap.h" #include "DiscIO/Blob.h" +#include "DiscIO/WiiEncryptionCache.h" namespace DiscIO { @@ -274,6 +276,7 @@ class WIAFileReader : public BlobReader File::IOFile m_file; Chunk m_cached_chunk; u64 m_cached_chunk_offset = std::numeric_limits::max(); + WiiEncryptionCache m_encryption_cache; WIAHeader1 m_header_1; WIAHeader2 m_header_2; @@ -281,6 +284,22 @@ class WIAFileReader : public BlobReader std::vector m_raw_data_entries; std::vector m_group_entries; + struct DataEntry + { + u32 index; + bool is_partition; + u8 partition_data_index; + + DataEntry(size_t index_) : index(static_cast(index_)), is_partition(false) {} + DataEntry(size_t index_, size_t partition_data_index_) + : index(static_cast(index_)), is_partition(true), + partition_data_index(static_cast(partition_data_index_)) + { + } + }; + + std::map m_data_entries; + static constexpr u32 WIA_VERSION = 0x01000000; static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000; static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000; From 47067f661acc0b39a6ceb54227d52c1b0ced5600 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Mon, 27 Jan 2020 14:26:12 +0100 Subject: [PATCH 13/36] WIA: Properly check for overlapping data --- Source/Core/DiscIO/WIABlob.cpp | 43 ++++++++++++++++++++++++++++++---- Source/Core/DiscIO/WIABlob.h | 1 + 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index d92aa8f0f106..5d71f24a16bc 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -142,10 +142,7 @@ bool WIAFileReader::Initialize(const std::string& path) if (non_empty_entries > 1) { - const u32 first_end = - Common::swap32(entries[0].first_sector) + Common::swap32(entries[0].number_of_sectors); - const u32 second_start = Common::swap32(entries[1].first_sector); - if (first_end > second_start) + if (Common::swap32(entries[0].first_sector) > Common::swap32(entries[1].first_sector)) return false; } } @@ -175,9 +172,47 @@ bool WIAFileReader::Initialize(const std::string& path) if (!group_entries.ReadAll(&m_group_entries)) return false; + if (HasDataOverlap()) + return false; + return true; } +bool WIAFileReader::HasDataOverlap() const +{ + for (size_t i = 0; i < m_partition_entries.size(); ++i) + { + const std::array& entries = m_partition_entries[i].data_entries; + for (size_t j = 0; j < entries.size(); ++j) + { + if (Common::swap32(entries[j].number_of_sectors) == 0) + continue; + + const u64 data_offset = Common::swap32(entries[j].first_sector) * VolumeWii::BLOCK_TOTAL_SIZE; + const auto it = m_data_entries.upper_bound(data_offset); + if (it == m_data_entries.end()) + return true; // Not an overlap, but an error nonetheless + if (!it->second.is_partition || it->second.index != i || it->second.partition_data_index != j) + return true; // Overlap + } + } + + for (size_t i = 0; i < m_raw_data_entries.size(); ++i) + { + if (Common::swap64(m_raw_data_entries[i].data_size) == 0) + continue; + + const u64 data_offset = Common::swap64(m_raw_data_entries[i].data_offset); + const auto it = m_data_entries.upper_bound(data_offset); + if (it == m_data_entries.end()) + return true; // Not an overlap, but an error nonetheless + if (it->second.is_partition || it->second.index != i) + return true; // Overlap + } + + return false; +} + std::unique_ptr WIAFileReader::Create(File::IOFile file, const std::string& path) { std::unique_ptr blob(new WIAFileReader(std::move(file), path)); diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 2360c7acf4fa..5bd07707197d 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -261,6 +261,7 @@ class WIAFileReader : public BlobReader explicit WIAFileReader(File::IOFile file, const std::string& path); bool Initialize(const std::string& path); + bool HasDataOverlap() const; bool ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size, u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups, From 791e363c9a5cb3f1f4514f2b41d029c432344c50 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Mon, 27 Jan 2020 16:12:56 +0100 Subject: [PATCH 14/36] WIA: Make use of the exception lists --- Source/Core/DiscIO/VolumeWii.cpp | 12 +++-- Source/Core/DiscIO/VolumeWii.h | 5 ++- Source/Core/DiscIO/WIABlob.cpp | 53 ++++++++++++++++++++++- Source/Core/DiscIO/WIABlob.h | 4 ++ Source/Core/DiscIO/WiiEncryptionCache.cpp | 22 ++++++++-- Source/Core/DiscIO/WiiEncryptionCache.h | 12 ++--- 6 files changed, 93 insertions(+), 15 deletions(-) diff --git a/Source/Core/DiscIO/VolumeWii.cpp b/Source/Core/DiscIO/VolumeWii.cpp index 3bbe347211c2..6df1b56817c1 100644 --- a/Source/Core/DiscIO/VolumeWii.cpp +++ b/Source/Core/DiscIO/VolumeWii.cpp @@ -532,10 +532,11 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition) return CheckBlockIntegrity(block_index, cluster, partition); } -bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset, - u64 partition_data_decrypted_size, - const std::array& key, BlobReader* blob, - std::array* out) +bool VolumeWii::EncryptGroup( + u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size, + const std::array& key, BlobReader* blob, + std::array* out, + const std::function& hash_exception_callback) { std::vector> unencrypted_data(BLOCKS_PER_GROUP); std::vector unencrypted_hashes(BLOCKS_PER_GROUP); @@ -632,6 +633,9 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset, if (error_occurred) return false; + if (hash_exception_callback) + hash_exception_callback(unencrypted_hashes.data()); + const unsigned int threads = std::min(BLOCKS_PER_GROUP, std::max(1, std::thread::hardware_concurrency())); diff --git a/Source/Core/DiscIO/VolumeWii.h b/Source/Core/DiscIO/VolumeWii.h index 4def1a93934b..2722cc995ad4 100644 --- a/Source/Core/DiscIO/VolumeWii.h +++ b/Source/Core/DiscIO/VolumeWii.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include #include @@ -99,7 +100,9 @@ class VolumeWii : public VolumeDisc static bool EncryptGroup(u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size, const std::array& key, BlobReader* blob, - std::array* out); + std::array* out, + const std::function& + hash_exception_callback = {}); protected: u32 GetOffsetShift() const override { return 2; } diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 5d71f24a16bc..f97ceca75a65 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -277,12 +277,26 @@ bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) const u64 bytes_to_read = std::min(data_size - (offset - data_offset), size); + bool hash_exception_error = false; if (!m_encryption_cache.EncryptGroups( offset - partition_data_offset, bytes_to_read, out_ptr, partition_data_offset, - partition_total_sectors * VolumeWii::BLOCK_DATA_SIZE, partition.partition_key)) + partition_total_sectors * VolumeWii::BLOCK_DATA_SIZE, partition.partition_key, + [this, chunk_size, first_sector, partition_first_sector, &hash_exception_error]( + VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP], u64 offset) { + const u64 partition_part_offset = + (first_sector - partition_first_sector) * VolumeWii::BLOCK_TOTAL_SIZE; + const u64 index = + (offset - partition_part_offset) % chunk_size / VolumeWii::GROUP_TOTAL_SIZE; + + // EncryptGroups calls ReadWiiDecrypted, which populates m_cached_chunk + if (!m_cached_chunk.ApplyHashExceptions(hash_blocks, index)) + hash_exception_error = true; + })) { return false; } + if (hash_exception_error) + return false; offset += bytes_to_read; size -= bytes_to_read; @@ -836,4 +850,41 @@ bool WIAFileReader::Chunk::HandleExceptions(const u8* data, size_t bytes_allocat return true; } +bool WIAFileReader::Chunk::ApplyHashExceptions( + VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP], u64 exception_list_index) const +{ + if (m_exception_lists > 0) + return false; // We still have exception lists left to read + + const u8* data = m_compressed_exception_lists ? m_out.data.data() : m_in.data.data(); + + for (u64 i = exception_list_index; i > 0; --i) + data += Common::swap16(data) * sizeof(HashExceptionEntry) + sizeof(u16); + + const u16 exceptions = Common::swap16(data); + data += sizeof(u16); + + for (size_t i = 0; i < exceptions; ++i) + { + HashExceptionEntry exception; + std::memcpy(&exception, data, sizeof(HashExceptionEntry)); + data += sizeof(HashExceptionEntry); + + const u16 offset = Common::swap16(exception.offset); + + const size_t block_index = offset / VolumeWii::BLOCK_HEADER_SIZE; + if (block_index > VolumeWii::BLOCKS_PER_GROUP) + return false; + + const size_t offset_in_block = offset % VolumeWii::BLOCK_HEADER_SIZE; + if (offset_in_block + sizeof(SHA1) > VolumeWii::BLOCK_HEADER_SIZE) + return false; + + std::memcpy(reinterpret_cast(&hash_blocks[block_index]) + offset_in_block, &exception.hash, + sizeof(SHA1)); + } + + return true; +} + } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 5bd07707197d..63defd6055f9 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -234,6 +234,10 @@ class WIAFileReader : public BlobReader bool Read(u64 offset, u64 size, u8* out_ptr); + // This can only be called once at least one byte of data has been read + bool ApplyHashExceptions(VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP], + u64 exception_list_index) const; + template bool ReadAll(std::vector* vector) { diff --git a/Source/Core/DiscIO/WiiEncryptionCache.cpp b/Source/Core/DiscIO/WiiEncryptionCache.cpp index c5a2111daa71..0bb892da711e 100644 --- a/Source/Core/DiscIO/WiiEncryptionCache.cpp +++ b/Source/Core/DiscIO/WiiEncryptionCache.cpp @@ -24,7 +24,8 @@ WiiEncryptionCache::~WiiEncryptionCache() = default; const std::array* WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset, - u64 partition_data_decrypted_size, const Key& key) + u64 partition_data_decrypted_size, const Key& key, + const HashExceptionCallback& hash_exception_callback) { // Only allocate memory if this function actually ends up getting called if (!m_cache) @@ -40,8 +41,20 @@ WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset, if (m_cached_offset != group_offset_on_disc) { + std::function hash_exception_callback_2; + + if (hash_exception_callback) + { + hash_exception_callback_2 = + [offset, &hash_exception_callback]( + VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]) { + return hash_exception_callback(hash_blocks, offset); + }; + } + if (!VolumeWii::EncryptGroup(group_offset_in_partition, partition_data_offset, - partition_data_decrypted_size, key, m_blob, m_cache.get())) + partition_data_decrypted_size, key, m_blob, m_cache.get(), + hash_exception_callback_2)) { m_cached_offset = std::numeric_limits::max(); // Invalidate the cache return nullptr; @@ -54,13 +67,14 @@ WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset, } bool WiiEncryptionCache::EncryptGroups(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset, - u64 partition_data_decrypted_size, const Key& key) + u64 partition_data_decrypted_size, const Key& key, + const HashExceptionCallback& hash_exception_callback) { while (size > 0) { const std::array* group = EncryptGroup(Common::AlignDown(offset, VolumeWii::GROUP_TOTAL_SIZE), partition_data_offset, - partition_data_decrypted_size, key); + partition_data_decrypted_size, key, hash_exception_callback); if (!group) return false; diff --git a/Source/Core/DiscIO/WiiEncryptionCache.h b/Source/Core/DiscIO/WiiEncryptionCache.h index 0c8f4b489a9f..d0a48b054d44 100644 --- a/Source/Core/DiscIO/WiiEncryptionCache.h +++ b/Source/Core/DiscIO/WiiEncryptionCache.h @@ -19,6 +19,8 @@ class WiiEncryptionCache { public: using Key = std::array; + using HashExceptionCallback = std::function; // The blob pointer is kept around for the lifetime of this object. explicit WiiEncryptionCache(BlobReader* blob); @@ -28,15 +30,15 @@ class WiiEncryptionCache // If the returned pointer is nullptr, reading from the blob failed. // If the returned pointer is not nullptr, it is guaranteed to be valid until // the next call of this function or the destruction of this object. - const std::array* EncryptGroup(u64 offset, - u64 partition_data_offset, - u64 partition_data_decrypted_size, - const Key& key); + const std::array* + EncryptGroup(u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size, + const Key& key, const HashExceptionCallback& hash_exception_callback = {}); // Encrypts a variable number of groups, as determined by the offset and size parameters. // Supports reading groups partially. bool EncryptGroups(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset, - u64 partition_data_decrypted_size, const Key& key); + u64 partition_data_decrypted_size, const Key& key, + const HashExceptionCallback& hash_exception_callback = {}); private: BlobReader* m_blob; From 115edea34ea4ad0ce06e069e6eb9ee6d3dedf329 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Mon, 13 Apr 2020 22:04:16 +0200 Subject: [PATCH 15/36] WIA: Add early support for WIA writing --- Source/Core/DiscIO/Blob.h | 3 + Source/Core/DiscIO/WIABlob.cpp | 195 ++++++++++++++++++++++++ Source/Core/DiscIO/WIABlob.h | 14 ++ Source/Core/DolphinQt/ConvertDialog.cpp | 29 +++- 4 files changed, 239 insertions(+), 2 deletions(-) diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 5371478d6582..692ac5707260 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -173,5 +173,8 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path, bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, const std::string& outfile_path, CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, int chunk_size, CompressCB callback = nullptr, + void* arg = nullptr); } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index f97ceca75a65..4a523a5c257c 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -17,12 +17,16 @@ #include #include "Common/Align.h" +#include "Common/Assert.h" #include "Common/CommonTypes.h" #include "Common/File.h" +#include "Common/FileUtil.h" #include "Common/Logging/Log.h" +#include "Common/MsgHandler.h" #include "Common/StringUtil.h" #include "Common/Swap.h" +#include "DiscIO/Blob.h" #include "DiscIO/VolumeWii.h" #include "DiscIO/WiiEncryptionCache.h" @@ -887,4 +891,195 @@ bool WIAFileReader::Chunk::ApplyHashExceptions( return true; } +bool WIAFileReader::PadTo4(File::IOFile* file, u64* bytes_written) +{ + constexpr u32 ZEROES = 0; + const u64 bytes_to_write = Common::AlignUp(*bytes_written, 4) - *bytes_written; + if (bytes_to_write == 0) + return true; + + *bytes_written += bytes_to_write; + return file->WriteBytes(&ZEROES, bytes_to_write); +} + +WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, + File::IOFile* outfile, int chunk_size, + CompressCB callback, void* arg) +{ + ASSERT(infile->IsDataSizeAccurate()); + ASSERT(chunk_size > 0); + + const u64 iso_size = infile->GetDataSize(); + + u64 bytes_read = 0; + u64 bytes_written = 0; + + // These two headers will be filled in with proper values at the very end + WIAHeader1 header_1; + WIAHeader2 header_2; + if (!outfile->WriteArray(&header_1, 1) || !outfile->WriteArray(&header_2, 1)) + return ConversionResult::WriteFailed; + bytes_written += sizeof(WIAHeader1) + sizeof(WIAHeader2); + if (!PadTo4(outfile, &bytes_written)) + return ConversionResult::WriteFailed; + + std::vector group_entries; + group_entries.resize(Common::AlignUp(iso_size, chunk_size) / chunk_size); + + std::vector raw_data_entries; + raw_data_entries.emplace_back( + RawDataEntry{Common::swap64(header_2.disc_header.size()), + Common::swap64(iso_size - header_2.disc_header.size()), 0, + Common::swap32(static_cast(group_entries.size()))}); + + std::vector partition_entries; + + const auto run_callback = [&](size_t groups_written) { + int ratio = 0; + if (bytes_read != 0) + ratio = static_cast(100 * bytes_written / bytes_read); + + const std::string temp = + StringFromFormat(Common::GetStringT("%i of %i blocks. Compression ratio %i%%").c_str(), + groups_written, group_entries.size(), ratio); + return callback(temp, static_cast(groups_written) / group_entries.size(), arg); + }; + + if (!infile->Read(0, header_2.disc_header.size(), header_2.disc_header.data())) + return ConversionResult::ReadFailed; + // We intentially do not increment bytes_read here, since these bytes will be read again + + if (!run_callback(0)) + return ConversionResult::Canceled; + + std::vector buffer(chunk_size); + for (size_t i = 0; i < group_entries.size(); ++i) + { + const u64 bytes_to_read = std::min(chunk_size, iso_size - bytes_read); + + if (bytes_written >> 2 > std::numeric_limits::max()) + return ConversionResult::InternalError; + + ASSERT((bytes_written & 3) == 0); + group_entries[i] = GroupEntry{Common::swap32(static_cast(bytes_written >> 2)), + Common::swap32(static_cast(bytes_to_read))}; + + if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) + return ConversionResult::ReadFailed; + if (!outfile->WriteArray(buffer.data(), bytes_to_read)) + return ConversionResult::WriteFailed; + + bytes_read += bytes_to_read; + bytes_written += bytes_to_read; + if (!PadTo4(outfile, &bytes_written)) + return ConversionResult::WriteFailed; + + if (!run_callback(i)) + return ConversionResult::Canceled; + } + + const u64 partition_entries_offset = bytes_written; + const u64 partition_entries_size = partition_entries.size() * sizeof(PartitionEntry); + if (!outfile->WriteArray(partition_entries.data(), partition_entries.size())) + return ConversionResult::WriteFailed; + bytes_written += partition_entries_size; + if (!PadTo4(outfile, &bytes_written)) + return ConversionResult::WriteFailed; + + const u64 raw_data_entries_offset = bytes_written; + const u64 raw_data_entries_size = raw_data_entries.size() * sizeof(RawDataEntry); + if (!outfile->WriteArray(raw_data_entries.data(), raw_data_entries.size())) + return ConversionResult::WriteFailed; + bytes_written += raw_data_entries_size; + if (!PadTo4(outfile, &bytes_written)) + return ConversionResult::WriteFailed; + + const u64 group_entries_offset = bytes_written; + const u64 group_entries_size = group_entries.size() * sizeof(GroupEntry); + if (!outfile->WriteArray(group_entries.data(), group_entries.size())) + return ConversionResult::WriteFailed; + bytes_written += group_entries_size; + if (!PadTo4(outfile, &bytes_written)) + return ConversionResult::WriteFailed; + + header_2.disc_type = 0; // TODO + header_2.compression_type = Common::swap32(static_cast(CompressionType::None)); + header_2.compression_level = 0; + header_2.chunk_size = Common::swap32(static_cast(chunk_size)); + + header_2.number_of_partition_entries = Common::swap32(static_cast(partition_entries.size())); + header_2.partition_entry_size = Common::swap32(sizeof(PartitionEntry)); + header_2.partition_entries_offset = Common::swap64(partition_entries_offset); + + if (partition_entries.data() == nullptr) + partition_entries.reserve(1); // Avoid a crash in mbedtls_sha1_ret + mbedtls_sha1_ret(reinterpret_cast(partition_entries.data()), partition_entries_size, + header_2.partition_entries_hash.data()); + + header_2.number_of_raw_data_entries = Common::swap32(static_cast(raw_data_entries.size())); + header_2.raw_data_entries_offset = Common::swap64(raw_data_entries_offset); + header_2.raw_data_entries_size = Common::swap32(static_cast(raw_data_entries_size)); + + header_2.number_of_group_entries = Common::swap32(static_cast(group_entries.size())); + header_2.group_entries_offset = Common::swap64(group_entries_offset); + header_2.group_entries_size = Common::swap32(static_cast(group_entries_size)); + + header_2.compressor_data_size = 0; + std::fill(std::begin(header_2.compressor_data), std::end(header_2.compressor_data), 0); + + header_1.magic = WIA_MAGIC; + header_1.version = Common::swap32(WIA_VERSION); + header_1.version_compatible = Common::swap32(WIA_VERSION_WRITE_COMPATIBLE); + header_1.header_2_size = Common::swap32(sizeof(WIAHeader2)); + mbedtls_sha1_ret(reinterpret_cast(&header_2), sizeof(header_2), + header_1.header_2_hash.data()); + header_1.iso_file_size = Common::swap64(infile->GetDataSize()); + header_1.wia_file_size = Common::swap64(bytes_written); + mbedtls_sha1_ret(reinterpret_cast(&header_1), offsetof(WIAHeader1, header_1_hash), + header_1.header_1_hash.data()); + + if (!outfile->Seek(0, SEEK_SET)) + return ConversionResult::WriteFailed; + if (!outfile->WriteArray(&header_1, 1) || !outfile->WriteArray(&header_2, 1)) + return ConversionResult::WriteFailed; + + return ConversionResult::Success; +} + +bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, int chunk_size, CompressCB callback, void* arg) +{ + File::IOFile outfile(outfile_path, "wb"); + if (!outfile) + { + PanicAlertT("Failed to open the output file \"%s\".\n" + "Check that you have permissions to write the target folder and that the media can " + "be written.", + outfile_path.c_str()); + return false; + } + + WIAFileReader::ConversionResult result = + WIAFileReader::ConvertToWIA(infile, &outfile, chunk_size, callback, arg); + + if (result == WIAFileReader::ConversionResult::ReadFailed) + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + + if (result == WIAFileReader::ConversionResult::WriteFailed) + { + PanicAlertT("Failed to write the output file \"%s\".\n" + "Check that you have enough space available on the target drive.", + outfile_path.c_str()); + } + + if (result != WIAFileReader::ConversionResult::Success) + { + // Remove the incomplete output file + outfile.Close(); + File::Delete(outfile_path); + } + + return result == WIAFileReader::ConversionResult::Success; +} + } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 63defd6055f9..d42aa92fef21 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -44,6 +44,18 @@ class WIAFileReader : public BlobReader bool SupportsReadWiiDecrypted() const override; bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override; + enum class ConversionResult + { + Success, + Canceled, + ReadFailed, + WriteFailed, + InternalError, + }; + + static ConversionResult ConvertToWIA(BlobReader* infile, File::IOFile* outfile, int chunk_size, + CompressCB callback, void* arg); + private: using SHA1 = std::array; using WiiKey = std::array; @@ -275,6 +287,8 @@ class WIAFileReader : public BlobReader static std::string VersionToString(u32 version); + static bool PadTo4(File::IOFile* file, u64* bytes_written); + bool m_valid; CompressionType m_compression_type; diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index 0d01785dc400..831c0ec64147 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -57,6 +57,7 @@ ConvertDialog::ConvertDialog(QList> fi m_format = new QComboBox; m_format->addItem(QStringLiteral("ISO"), static_cast(DiscIO::BlobType::PLAIN)); m_format->addItem(QStringLiteral("GCZ"), static_cast(DiscIO::BlobType::GCZ)); + m_format->addItem(QStringLiteral("WIA"), static_cast(DiscIO::BlobType::WIA)); if (std::all_of(m_files.begin(), m_files.end(), [](const auto& file) { return file->GetBlobType() == DiscIO::BlobType::PLAIN; })) { @@ -88,7 +89,10 @@ ConvertDialog::ConvertDialog(QList> fi "It takes up more space than any other format.\n\n" "GCZ: A basic compressed format which is compatible with most versions of " "Dolphin and some other programs. It can't efficiently compress junk data " - "(unless removed) or encrypted Wii data.")); + "(unless removed) or encrypted Wii data.\n\n" + "WIA: An advanced compressed format which is compatible with recent versions " + "of Dolphin and a few other programs. It can efficiently compress encrypted " + "Wii data, but not junk data (unless removed).")); info_text->setWordWrap(true); QVBoxLayout* info_layout = new QVBoxLayout; @@ -166,6 +170,13 @@ void ConvertDialog::OnFormatChanged() break; } + case DiscIO::BlobType::WIA: + m_block_size->setEnabled(true); + + // This is the smallest block size supported by WIA. For performance, larger sizes are avoided. + AddToBlockSizeComboBox(0x200000); + + break; default: break; } @@ -224,7 +235,11 @@ void ConvertDialog::Convert() break; case DiscIO::BlobType::GCZ: extension = QStringLiteral(".gcz"); - filter = tr("Compressed GC/Wii images (*.gcz)"); + filter = tr("GCZ GC/Wii images (*.gcz)"); + break; + case DiscIO::BlobType::WIA: + extension = QStringLiteral(".wia"); + filter = tr("WIA GC/Wii images (*.wia)"); break; default: ASSERT(false); @@ -351,6 +366,16 @@ void ConvertDialog::Convert() return good; }); } + else if (format == DiscIO::BlobType::WIA) + { + good = std::async(std::launch::async, [&] { + const bool good = + DiscIO::ConvertToWIA(blob_reader.get(), original_path, dst_path.toStdString(), + block_size, &CompressCB, &progress_dialog); + progress_dialog.Reset(); + return good; + }); + } progress_dialog.GetRaw()->exec(); if (!good.get()) From 3b8c44fd0e986d81cd0d4f2c0a2107ca9e904f3b Mon Sep 17 00:00:00 2001 From: JosJuice Date: Tue, 14 Apr 2020 11:40:32 +0200 Subject: [PATCH 16/36] WIA: Decrypt Wii data when writing --- Source/Core/DiscIO/VolumeWii.cpp | 25 +-- Source/Core/DiscIO/VolumeWii.h | 2 + Source/Core/DiscIO/WIABlob.cpp | 339 ++++++++++++++++++++++++++++--- Source/Core/DiscIO/WIABlob.h | 47 +++-- 4 files changed, 350 insertions(+), 63 deletions(-) diff --git a/Source/Core/DiscIO/VolumeWii.cpp b/Source/Core/DiscIO/VolumeWii.cpp index 6df1b56817c1..227cedb5cf8c 100644 --- a/Source/Core/DiscIO/VolumeWii.cpp +++ b/Source/Core/DiscIO/VolumeWii.cpp @@ -201,18 +201,9 @@ bool VolumeWii::Read(u64 offset, u64 length, u8* buffer, const Partition& partit if (!m_reader->Read(block_offset_on_disc, BLOCK_TOTAL_SIZE, read_buffer.data())) return false; - // Decrypt the block's data. - // 0x3D0 - 0x3DF in read_buffer will be overwritten, - // but that won't affect anything, because we won't - // use the content of read_buffer anymore after this - mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_DATA_SIZE, &read_buffer[0x3D0], - &read_buffer[BLOCK_HEADER_SIZE], m_last_decrypted_block_data); + // Decrypt the block's data + DecryptBlockData(read_buffer.data(), m_last_decrypted_block_data, aes_context); m_last_decrypted_block = block_offset_on_disc; - - // The only thing we currently use from the 0x000 - 0x3FF part - // of the block is the IV (at 0x3D0), but it also contains SHA-1 - // hashes that IOS uses to check that discs aren't tampered with. - // http://wiibrew.org/wiki/Wii_Disc#Encrypted } // Copy the decrypted data @@ -487,9 +478,7 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const std::vector& encr encrypted_data.data(), reinterpret_cast(&hashes)); u8 cluster_data[BLOCK_DATA_SIZE]; - std::memcpy(iv, encrypted_data.data() + 0x3D0, 16); - mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(cluster_data), iv, - encrypted_data.data() + sizeof(HashBlock), cluster_data); + DecryptBlockData(encrypted_data.data(), cluster_data, aes_context); for (u32 hash_index = 0; hash_index < 31; ++hash_index) { @@ -671,4 +660,12 @@ bool VolumeWii::EncryptGroup( return true; } +void VolumeWii::DecryptBlockData(const u8* in, u8* out, mbedtls_aes_context* aes_context) +{ + std::array iv; + std::copy(&in[0x3d0], &in[0x3e0], iv.data()); + mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_DATA_SIZE, iv.data(), + &in[BLOCK_HEADER_SIZE], out); +} + } // namespace DiscIO diff --git a/Source/Core/DiscIO/VolumeWii.h b/Source/Core/DiscIO/VolumeWii.h index 2722cc995ad4..b0fdf713cbbc 100644 --- a/Source/Core/DiscIO/VolumeWii.h +++ b/Source/Core/DiscIO/VolumeWii.h @@ -104,6 +104,8 @@ class VolumeWii : public VolumeDisc const std::function& hash_exception_callback = {}); + static void DecryptBlockData(const u8* in, u8* out, mbedtls_aes_context* aes_context); + protected: u32 GetOffsetShift() const override { return 2; } diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 4a523a5c257c..af11451c6107 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -6,10 +6,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -27,6 +29,8 @@ #include "Common/Swap.h" #include "DiscIO/Blob.h" +#include "DiscIO/DiscExtractor.h" +#include "DiscIO/Volume.h" #include "DiscIO/VolumeWii.h" #include "DiscIO/WiiEncryptionCache.h" @@ -902,7 +906,161 @@ bool WIAFileReader::PadTo4(File::IOFile* file, u64* bytes_written) return file->WriteBytes(&ZEROES, bytes_to_write); } +void WIAFileReader::AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups, + std::vector* raw_data_entries, + std::vector* data_entries) +{ + constexpr size_t SKIP_SIZE = sizeof(WIAHeader2::disc_header); + const u64 skip = offset < SKIP_SIZE ? std::min(SKIP_SIZE - offset, size) : 0; + + offset += skip; + size -= skip; + + if (size == 0) + return; + + const u32 group_index = *total_groups; + const u32 groups = static_cast(Common::AlignUp(size, chunk_size) / chunk_size); + *total_groups += groups; + + data_entries->emplace_back(raw_data_entries->size()); + raw_data_entries->emplace_back(RawDataEntry{Common::swap64(offset), Common::swap64(size), + Common::swap32(group_index), Common::swap32(groups)}); +} + +WIAFileReader::PartitionDataEntry WIAFileReader::CreatePartitionDataEntry( + u64 offset, u64 size, u32 index, int chunk_size, u32* total_groups, + const std::vector& partition_entries, std::vector* data_entries) +{ + const u32 group_index = *total_groups; + const u64 rounded_size = Common::AlignDown(size, VolumeWii::BLOCK_TOTAL_SIZE); + const u32 groups = static_cast(Common::AlignUp(rounded_size, chunk_size) / chunk_size); + *total_groups += groups; + + data_entries->emplace_back(partition_entries.size(), index); + return PartitionDataEntry{Common::swap32(offset / VolumeWii::BLOCK_TOTAL_SIZE), + Common::swap32(size / VolumeWii::BLOCK_TOTAL_SIZE), + Common::swap32(group_index), Common::swap32(groups)}; +} + +WIAFileReader::ConversionResult WIAFileReader::SetUpDataEntriesForWriting( + const VolumeDisc* volume, int chunk_size, u64 iso_size, u32* total_groups, + std::vector* partition_entries, std::vector* raw_data_entries, + std::vector* data_entries) +{ + std::vector partitions; + if (volume && volume->IsEncryptedAndHashed()) + partitions = volume->GetPartitions(); + + std::sort(partitions.begin(), partitions.end(), + [](const Partition& a, const Partition& b) { return a.offset < b.offset; }); + + *total_groups = 0; + + u64 last_partition_end_offset = 0; + + const auto add_raw_data_entry = [&](u64 offset, u64 size) { + return AddRawDataEntry(offset, size, chunk_size, total_groups, raw_data_entries, data_entries); + }; + + const auto create_partition_data_entry = [&](u64 offset, u64 size, u32 index) { + return CreatePartitionDataEntry(offset, size, index, chunk_size, total_groups, + *partition_entries, data_entries); + }; + + for (const Partition& partition : partitions) + { + // If a partition is odd in some way that prevents us from encoding it as a partition, + // we encode it as raw data instead by skipping the current loop iteration. + // Partitions can always be encoded as raw data, but it is less space efficient. + + if (partition.offset < last_partition_end_offset) + { + WARN_LOG(DISCIO, "Overlapping partitions at %" PRIx64, partition.offset); + continue; + } + + if (volume->ReadSwapped(partition.offset, PARTITION_NONE) != u32(0x10001)) + { + // This looks more like garbage data than an actual partition. + // The values of data_offset and data_size will very likely also be garbage. + // Some WBFS writing programs scrub the SSBB Masterpiece partitions without + // removing them from the partition table, causing this problem. + WARN_LOG(DISCIO, "Invalid partition at %" PRIx64, partition.offset); + continue; + } + + std::optional data_offset = + volume->ReadSwappedAndShifted(partition.offset + 0x2b8, PARTITION_NONE); + std::optional data_size = + volume->ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE); + + if (!data_offset || !data_size) + return ConversionResult::ReadFailed; + + const u64 data_start = partition.offset + *data_offset; + const u64 data_end = data_start + *data_size; + + if (data_start % VolumeWii::BLOCK_TOTAL_SIZE != 0) + { + WARN_LOG(DISCIO, "Misaligned partition at %" PRIx64, partition.offset); + continue; + } + + if (*data_size < VolumeWii::BLOCK_TOTAL_SIZE) + { + WARN_LOG(DISCIO, "Very small partition at %" PRIx64, partition.offset); + continue; + } + + if (data_end > iso_size) + { + WARN_LOG(DISCIO, "Too large partition at %" PRIx64, partition.offset); + *data_size = iso_size - *data_offset - partition.offset; + } + + const std::optional fst_offset = GetFSTOffset(*volume, partition); + const std::optional fst_size = GetFSTSize(*volume, partition); + + if (!fst_offset || !fst_size) + return ConversionResult::ReadFailed; + + const IOS::ES::TicketReader& ticket = volume->GetTicket(partition); + if (!ticket.IsValid()) + return ConversionResult::ReadFailed; + + add_raw_data_entry(last_partition_end_offset, partition.offset - last_partition_end_offset); + + add_raw_data_entry(partition.offset, *data_offset); + + const u64 fst_end = volume->PartitionOffsetToRawOffset(*fst_offset + *fst_size, partition); + const u64 split_point = std::min( + data_end, Common::AlignUp(fst_end - data_start, VolumeWii::GROUP_TOTAL_SIZE) + data_start); + + PartitionEntry partition_entry; + partition_entry.partition_key = ticket.GetTitleKey(); + partition_entry.data_entries[0] = + create_partition_data_entry(data_start, split_point - data_start, 0); + partition_entry.data_entries[1] = + create_partition_data_entry(split_point, data_end - split_point, 1); + + // Note: We can't simply set last_partition_end_offset to data_end, + // because construct_partition_data_entry may have rounded it + last_partition_end_offset = + (Common::swap32(partition_entry.data_entries[1].first_sector) + + Common::swap32(partition_entry.data_entries[1].number_of_sectors)) * + VolumeWii::BLOCK_TOTAL_SIZE; + + partition_entries->emplace_back(std::move(partition_entry)); + } + + add_raw_data_entry(last_partition_end_offset, iso_size - last_partition_end_offset); + + return ConversionResult::Success; +} + WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, + const VolumeDisc* infile_volume, File::IOFile* outfile, int chunk_size, CompressCB callback, void* arg) { @@ -913,6 +1071,7 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, u64 bytes_read = 0; u64 bytes_written = 0; + size_t groups_written = 0; // These two headers will be filled in with proper values at the very end WIAHeader1 header_1; @@ -923,18 +1082,11 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, if (!PadTo4(outfile, &bytes_written)) return ConversionResult::WriteFailed; - std::vector group_entries; - group_entries.resize(Common::AlignUp(iso_size, chunk_size) / chunk_size); - - std::vector raw_data_entries; - raw_data_entries.emplace_back( - RawDataEntry{Common::swap64(header_2.disc_header.size()), - Common::swap64(iso_size - header_2.disc_header.size()), 0, - Common::swap32(static_cast(group_entries.size()))}); - std::vector partition_entries; + std::vector raw_data_entries; + std::vector group_entries; - const auto run_callback = [&](size_t groups_written) { + const auto run_callback = [&] { int ratio = 0; if (bytes_read != 0) ratio = static_cast(100 * bytes_written / bytes_read); @@ -942,42 +1094,152 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, const std::string temp = StringFromFormat(Common::GetStringT("%i of %i blocks. Compression ratio %i%%").c_str(), groups_written, group_entries.size(), ratio); - return callback(temp, static_cast(groups_written) / group_entries.size(), arg); + + float completion = 0.0f; + if (group_entries.size() != 0) + completion = static_cast(groups_written) / group_entries.size(); + + return callback(temp, completion, arg); }; + if (!run_callback()) + return ConversionResult::Canceled; + + u32 total_groups; + std::vector data_entries; + + const ConversionResult set_up_data_entries_result = + SetUpDataEntriesForWriting(infile_volume, chunk_size, iso_size, &total_groups, + &partition_entries, &raw_data_entries, &data_entries); + if (set_up_data_entries_result != ConversionResult::Success) + return set_up_data_entries_result; + + group_entries.resize(total_groups); + if (!infile->Read(0, header_2.disc_header.size(), header_2.disc_header.data())) return ConversionResult::ReadFailed; // We intentially do not increment bytes_read here, since these bytes will be read again - if (!run_callback(0)) - return ConversionResult::Canceled; - std::vector buffer(chunk_size); - for (size_t i = 0; i < group_entries.size(); ++i) + std::vector decryption_buffer(VolumeWii::BLOCK_DATA_SIZE); + for (const DataEntry& data_entry : data_entries) { - const u64 bytes_to_read = std::min(chunk_size, iso_size - bytes_read); + if (data_entry.is_partition) + { + const PartitionEntry& partition_entry = partition_entries[data_entry.index]; + const PartitionDataEntry& partition_data_entry = + partition_entry.data_entries[data_entry.partition_data_index]; - if (bytes_written >> 2 > std::numeric_limits::max()) - return ConversionResult::InternalError; + const u32 first_group = Common::swap32(partition_data_entry.group_index); + const u32 last_group = first_group + Common::swap32(partition_data_entry.number_of_groups); - ASSERT((bytes_written & 3) == 0); - group_entries[i] = GroupEntry{Common::swap32(static_cast(bytes_written >> 2)), - Common::swap32(static_cast(bytes_to_read))}; + const u64 data_offset = + Common::swap32(partition_data_entry.first_sector) * VolumeWii::BLOCK_TOTAL_SIZE; + const u64 data_size = + Common::swap32(partition_data_entry.number_of_sectors) * VolumeWii::BLOCK_TOTAL_SIZE; - if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) - return ConversionResult::ReadFailed; - if (!outfile->WriteArray(buffer.data(), bytes_to_read)) - return ConversionResult::WriteFailed; + ASSERT(groups_written == first_group); + ASSERT(bytes_read == data_offset); + + mbedtls_aes_context aes_context; + mbedtls_aes_setkey_dec(&aes_context, partition_entry.partition_key.data(), 128); + + for (u32 i = first_group; i < last_group; ++i) + { + const u64 bytes_to_read = std::min(chunk_size, data_offset + data_size - bytes_read); + + ASSERT(bytes_to_read % VolumeWii::BLOCK_TOTAL_SIZE == 0); + const u64 bytes_to_write = + bytes_to_read / VolumeWii::BLOCK_TOTAL_SIZE * VolumeWii::BLOCK_DATA_SIZE; + + if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) + return ConversionResult::ReadFailed; + + const u64 exception_lists = Common::AlignUp(bytes_to_read, VolumeWii::GROUP_TOTAL_SIZE) / + VolumeWii::GROUP_TOTAL_SIZE; + + const u64 exceptions_size = Common::AlignUp(exception_lists * sizeof(u16), 4); + const u64 total_size = exceptions_size + bytes_to_write; + ASSERT((bytes_written & 3) == 0); + group_entries[i].data_offset = Common::swap32(static_cast(bytes_written >> 2)); + group_entries[i].data_size = Common::swap32(static_cast(total_size)); + + for (u64 j = 0; j < exception_lists; ++j) + { + const u16 exceptions = 0; + if (!outfile->WriteArray(&exceptions, 1)) + return ConversionResult::WriteFailed; + bytes_written += sizeof(u16); + } + + if (!PadTo4(outfile, &bytes_written)) + return ConversionResult::WriteFailed; + + for (u64 j = 0; j < bytes_to_read; j += VolumeWii::BLOCK_TOTAL_SIZE) + { + VolumeWii::DecryptBlockData(buffer.data() + j, decryption_buffer.data(), &aes_context); + if (!outfile->WriteArray(decryption_buffer.data(), VolumeWii::BLOCK_DATA_SIZE)) + return ConversionResult::WriteFailed; + } + + bytes_read += bytes_to_read; + bytes_written += bytes_to_write; + ++groups_written; + if (!PadTo4(outfile, &bytes_written)) + return ConversionResult::WriteFailed; + + if (!run_callback()) + return ConversionResult::Canceled; + } + } + else + { + const RawDataEntry& raw_data_entry = raw_data_entries[data_entry.index]; + + const u32 first_group = Common::swap32(raw_data_entry.group_index); + const u32 last_group = first_group + Common::swap32(raw_data_entry.number_of_groups); + + u64 data_offset = Common::swap64(raw_data_entry.data_offset); + u64 data_size = Common::swap64(raw_data_entry.data_size); - bytes_read += bytes_to_read; - bytes_written += bytes_to_read; - if (!PadTo4(outfile, &bytes_written)) - return ConversionResult::WriteFailed; + const u64 skipped_data = data_offset % VolumeWii::BLOCK_TOTAL_SIZE; + data_offset -= skipped_data; + data_size += skipped_data; - if (!run_callback(i)) - return ConversionResult::Canceled; + ASSERT(groups_written == first_group); + ASSERT(bytes_read == data_offset); + + for (u32 i = first_group; i < last_group; ++i) + { + const u64 bytes_to_read = std::min(chunk_size, data_offset + data_size - bytes_read); + + if (bytes_written >> 2 > std::numeric_limits::max()) + return ConversionResult::InternalError; + + ASSERT((bytes_written & 3) == 0); + group_entries[i].data_offset = Common::swap32(static_cast(bytes_written >> 2)); + group_entries[i].data_size = Common::swap32(static_cast(bytes_to_read)); + + if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) + return ConversionResult::ReadFailed; + if (!outfile->WriteArray(buffer.data(), bytes_to_read)) + return ConversionResult::WriteFailed; + + bytes_read += bytes_to_read; + bytes_written += bytes_to_read; + ++groups_written; + if (!PadTo4(outfile, &bytes_written)) + return ConversionResult::WriteFailed; + + if (!run_callback()) + return ConversionResult::Canceled; + } + } } + ASSERT(groups_written == total_groups); + ASSERT(bytes_read == iso_size); + const u64 partition_entries_offset = bytes_written; const u64 partition_entries_size = partition_entries.size() * sizeof(PartitionEntry); if (!outfile->WriteArray(partition_entries.data(), partition_entries.size())) @@ -1002,7 +1264,16 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, if (!PadTo4(outfile, &bytes_written)) return ConversionResult::WriteFailed; - header_2.disc_type = 0; // TODO + u32 disc_type = 0; + if (infile_volume) + { + if (infile_volume->GetVolumeType() == Platform::GameCubeDisc) + disc_type = 1; + else if (infile_volume->GetVolumeType() == Platform::WiiDisc) + disc_type = 2; + } + + header_2.disc_type = Common::swap32(disc_type); header_2.compression_type = Common::swap32(static_cast(CompressionType::None)); header_2.compression_level = 0; header_2.chunk_size = Common::swap32(static_cast(chunk_size)); @@ -1059,8 +1330,10 @@ bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, return false; } + std::unique_ptr infile_volume = CreateDisc(infile_path); + WIAFileReader::ConversionResult result = - WIAFileReader::ConvertToWIA(infile, &outfile, chunk_size, callback, arg); + WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, chunk_size, callback, arg); if (result == WIAFileReader::ConversionResult::ReadFailed) PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index d42aa92fef21..c1f46cc5eae0 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -22,6 +22,8 @@ namespace DiscIO { +class VolumeDisc; + constexpr u32 WIA_MAGIC = 0x01414957; // "WIA\x1" (byteswapped to little endian) class WIAFileReader : public BlobReader @@ -53,8 +55,9 @@ class WIAFileReader : public BlobReader InternalError, }; - static ConversionResult ConvertToWIA(BlobReader* infile, File::IOFile* outfile, int chunk_size, - CompressCB callback, void* arg); + static ConversionResult ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, + File::IOFile* outfile, int chunk_size, CompressCB callback, + void* arg); private: using SHA1 = std::array; @@ -159,6 +162,20 @@ class WIAFileReader : public BlobReader static_assert(sizeof(PurgeSegment) == 0x08, "Wrong size for WIA purge segment"); #pragma pack(pop) + struct DataEntry + { + u32 index; + bool is_partition; + u8 partition_data_index; + + DataEntry(size_t index_) : index(static_cast(index_)), is_partition(false) {} + DataEntry(size_t index_, size_t partition_data_index_) + : index(static_cast(index_)), is_partition(true), + partition_data_index(static_cast(partition_data_index_)) + { + } + }; + struct DecompressionBuffer { std::vector data; @@ -288,6 +305,18 @@ class WIAFileReader : public BlobReader static std::string VersionToString(u32 version); static bool PadTo4(File::IOFile* file, u64* bytes_written); + static void AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups, + std::vector* raw_data_entries, + std::vector* data_entries); + static PartitionDataEntry + CreatePartitionDataEntry(u64 offset, u64 size, u32 index, int chunk_size, u32* total_groups, + const std::vector& partition_entries, + std::vector* data_entries); + static ConversionResult SetUpDataEntriesForWriting(const VolumeDisc* volume, int chunk_size, + u64 iso_size, u32* total_groups, + std::vector* partition_entries, + std::vector* raw_data_entries, + std::vector* data_entries); bool m_valid; CompressionType m_compression_type; @@ -303,20 +332,6 @@ class WIAFileReader : public BlobReader std::vector m_raw_data_entries; std::vector m_group_entries; - struct DataEntry - { - u32 index; - bool is_partition; - u8 partition_data_index; - - DataEntry(size_t index_) : index(static_cast(index_)), is_partition(false) {} - DataEntry(size_t index_, size_t partition_data_index_) - : index(static_cast(index_)), is_partition(true), - partition_data_index(static_cast(partition_data_index_)) - { - } - }; - std::map m_data_entries; static constexpr u32 WIA_VERSION = 0x01000000; From e936c4acd821a12f4d45d163df66cf062f64f902 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 15 Apr 2020 21:15:08 +0200 Subject: [PATCH 17/36] WIA: Write hash exceptions --- Source/Core/DiscIO/VolumeWii.cpp | 111 +++++++++++++++-------------- Source/Core/DiscIO/VolumeWii.h | 9 +++ Source/Core/DiscIO/WIABlob.cpp | 117 ++++++++++++++++++++++++++----- 3 files changed, 166 insertions(+), 71 deletions(-) diff --git a/Source/Core/DiscIO/VolumeWii.cpp b/Source/Core/DiscIO/VolumeWii.cpp index 227cedb5cf8c..6e2b40041fb4 100644 --- a/Source/Core/DiscIO/VolumeWii.cpp +++ b/Source/Core/DiscIO/VolumeWii.cpp @@ -473,9 +473,7 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const std::vector& encr return false; HashBlock hashes; - u8 iv[16] = {0}; - mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(HashBlock), iv, - encrypted_data.data(), reinterpret_cast(&hashes)); + DecryptBlockHashes(encrypted_data.data(), &hashes, aes_context); u8 cluster_data[BLOCK_DATA_SIZE]; DecryptBlockData(encrypted_data.data(), cluster_data, aes_context); @@ -521,55 +519,33 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition) return CheckBlockIntegrity(block_index, cluster, partition); } -bool VolumeWii::EncryptGroup( - u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size, - const std::array& key, BlobReader* blob, - std::array* out, - const std::function& hash_exception_callback) +bool VolumeWii::HashGroup(const std::array in[BLOCKS_PER_GROUP], + HashBlock out[BLOCKS_PER_GROUP], + const std::function& read_function) { - std::vector> unencrypted_data(BLOCKS_PER_GROUP); - std::vector unencrypted_hashes(BLOCKS_PER_GROUP); - std::array, BLOCKS_PER_GROUP> hash_futures; - bool error_occurred = false; + bool success = true; for (size_t i = 0; i < BLOCKS_PER_GROUP; ++i) { - if (!error_occurred) - { - if (offset + (i + 1) * BLOCK_DATA_SIZE <= partition_data_decrypted_size) - { - if (!blob->ReadWiiDecrypted(offset + i * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE, - unencrypted_data[i].data(), partition_data_offset)) - { - error_occurred = true; - } - } - else - { - unencrypted_data[i].fill(0); - } - } + if (read_function && success) + success = read_function(i); - hash_futures[i] = std::async(std::launch::async, [&unencrypted_data, &unencrypted_hashes, - &hash_futures, error_occurred, i]() { + hash_futures[i] = std::async(std::launch::async, [&in, &out, &hash_futures, success, i]() { const size_t h1_base = Common::AlignDown(i, 8); - if (!error_occurred) + if (success) { // H0 hashes for (size_t j = 0; j < 31; ++j) - { - mbedtls_sha1_ret(unencrypted_data[i].data() + j * 0x400, 0x400, - unencrypted_hashes[i].h0[j]); - } + mbedtls_sha1_ret(in[i].data() + j * 0x400, 0x400, out[i].h0[j]); // H0 padding - std::memset(unencrypted_hashes[i].padding_0, 0, sizeof(HashBlock::padding_0)); + std::memset(out[i].padding_0, 0, sizeof(HashBlock::padding_0)); // H1 hash - mbedtls_sha1_ret(reinterpret_cast(unencrypted_hashes[i].h0), sizeof(HashBlock::h0), - unencrypted_hashes[h1_base].h1[i - h1_base]); + mbedtls_sha1_ret(reinterpret_cast(out[i].h0), sizeof(HashBlock::h0), + out[h1_base].h1[i - h1_base]); } if (i % 8 == 7) @@ -577,21 +553,18 @@ bool VolumeWii::EncryptGroup( for (size_t j = 0; j < 7; ++j) hash_futures[h1_base + j].get(); - if (!error_occurred) + if (success) { // H1 padding - std::memset(unencrypted_hashes[h1_base].padding_1, 0, sizeof(HashBlock::padding_1)); + std::memset(out[h1_base].padding_1, 0, sizeof(HashBlock::padding_1)); // H1 copies for (size_t j = 1; j < 8; ++j) - { - std::memcpy(unencrypted_hashes[h1_base + j].h1, unencrypted_hashes[h1_base].h1, - sizeof(HashBlock::h1)); - } + std::memcpy(out[h1_base + j].h1, out[h1_base].h1, sizeof(HashBlock::h1)); // H2 hash - mbedtls_sha1_ret(reinterpret_cast(unencrypted_hashes[i].h1), sizeof(HashBlock::h1), - unencrypted_hashes[0].h2[h1_base / 8]); + mbedtls_sha1_ret(reinterpret_cast(out[i].h1), sizeof(HashBlock::h1), + out[0].h2[h1_base / 8]); } if (i == BLOCKS_PER_GROUP - 1) @@ -599,17 +572,14 @@ bool VolumeWii::EncryptGroup( for (size_t j = 0; j < 7; ++j) hash_futures[j * 8 + 7].get(); - if (!error_occurred) + if (success) { // H2 padding - std::memset(unencrypted_hashes[0].padding_2, 0, sizeof(HashBlock::padding_2)); + std::memset(out[0].padding_2, 0, sizeof(HashBlock::padding_2)); // H2 copies for (size_t j = 1; j < BLOCKS_PER_GROUP; ++j) - { - std::memcpy(unencrypted_hashes[j].h2, unencrypted_hashes[0].h2, - sizeof(HashBlock::h2)); - } + std::memcpy(out[j].h2, out[0].h2, sizeof(HashBlock::h2)); } } } @@ -619,7 +589,36 @@ bool VolumeWii::EncryptGroup( // Wait for all the async tasks to finish hash_futures.back().get(); - if (error_occurred) + return success; +} + +bool VolumeWii::EncryptGroup( + u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size, + const std::array& key, BlobReader* blob, + std::array* out, + const std::function& hash_exception_callback) +{ + std::vector> unencrypted_data(BLOCKS_PER_GROUP); + std::vector unencrypted_hashes(BLOCKS_PER_GROUP); + + const bool success = + HashGroup(unencrypted_data.data(), unencrypted_hashes.data(), [&](size_t block) { + if (offset + (block + 1) * BLOCK_DATA_SIZE <= partition_data_decrypted_size) + { + if (!blob->ReadWiiDecrypted(offset + block * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE, + unencrypted_data[block].data(), partition_data_offset)) + { + return false; + } + } + else + { + unencrypted_data[block].fill(0); + } + return true; + }); + + if (!success) return false; if (hash_exception_callback) @@ -660,6 +659,14 @@ bool VolumeWii::EncryptGroup( return true; } +void VolumeWii::DecryptBlockHashes(const u8* in, HashBlock* out, mbedtls_aes_context* aes_context) +{ + std::array iv; + iv.fill(0); + mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(HashBlock), iv.data(), in, + reinterpret_cast(out)); +} + void VolumeWii::DecryptBlockData(const u8* in, u8* out, mbedtls_aes_context* aes_context) { std::array iv; diff --git a/Source/Core/DiscIO/VolumeWii.h b/Source/Core/DiscIO/VolumeWii.h index b0fdf713cbbc..0f2bb43d0174 100644 --- a/Source/Core/DiscIO/VolumeWii.h +++ b/Source/Core/DiscIO/VolumeWii.h @@ -98,12 +98,21 @@ class VolumeWii : public VolumeDisc u64 GetRawSize() const override; const BlobReader& GetBlobReader() const; + // The in parameter can either contain all the data to begin with, + // or read_function can write data into the in parameter when called. + // The latter lets reading run in parallel with hashing. + // This function returns false iff read_function returns false. + static bool HashGroup(const std::array in[BLOCKS_PER_GROUP], + HashBlock out[BLOCKS_PER_GROUP], + const std::function& read_function = {}); + static bool EncryptGroup(u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size, const std::array& key, BlobReader* blob, std::array* out, const std::function& hash_exception_callback = {}); + static void DecryptBlockHashes(const u8* in, HashBlock* out, mbedtls_aes_context* aes_context); static void DecryptBlockData(const u8* in, u8* out, mbedtls_aes_context* aes_context); protected: diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index af11451c6107..157a59f01ba3 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -1068,6 +1068,7 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, ASSERT(chunk_size > 0); const u64 iso_size = infile->GetDataSize(); + const u64 exception_lists_per_chunk = chunk_size / VolumeWii::GROUP_TOTAL_SIZE; u64 bytes_read = 0; u64 bytes_written = 0; @@ -1120,8 +1121,12 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, return ConversionResult::ReadFailed; // We intentially do not increment bytes_read here, since these bytes will be read again + using WiiBlockData = std::array; + std::vector buffer(chunk_size); - std::vector decryption_buffer(VolumeWii::BLOCK_DATA_SIZE); + std::vector decryption_buffer(VolumeWii::BLOCKS_PER_GROUP); + std::vector hash_buffer(VolumeWii::BLOCKS_PER_GROUP); + for (const DataEntry& data_entry : data_entries) { if (data_entry.is_partition) @@ -1148,39 +1153,109 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, { const u64 bytes_to_read = std::min(chunk_size, data_offset + data_size - bytes_read); + const u64 groups = Common::AlignUp(bytes_to_read, VolumeWii::GROUP_TOTAL_SIZE) / + VolumeWii::GROUP_TOTAL_SIZE; + ASSERT(bytes_to_read % VolumeWii::BLOCK_TOTAL_SIZE == 0); - const u64 bytes_to_write = - bytes_to_read / VolumeWii::BLOCK_TOTAL_SIZE * VolumeWii::BLOCK_DATA_SIZE; + const u64 blocks = bytes_to_read / VolumeWii::BLOCK_TOTAL_SIZE; + const u64 bytes_to_write = blocks * VolumeWii::BLOCK_DATA_SIZE; if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) return ConversionResult::ReadFailed; - const u64 exception_lists = Common::AlignUp(bytes_to_read, VolumeWii::GROUP_TOTAL_SIZE) / - VolumeWii::GROUP_TOTAL_SIZE; + std::vector> exception_lists(exception_lists_per_chunk); - const u64 exceptions_size = Common::AlignUp(exception_lists * sizeof(u16), 4); - const u64 total_size = exceptions_size + bytes_to_write; - ASSERT((bytes_written & 3) == 0); - group_entries[i].data_offset = Common::swap32(static_cast(bytes_written >> 2)); - group_entries[i].data_size = Common::swap32(static_cast(total_size)); + for (u64 j = 0; j < groups; ++j) + { + const u64 offset_of_group = j * VolumeWii::GROUP_TOTAL_SIZE; + const u64 write_offset_of_group = j * VolumeWii::GROUP_DATA_SIZE; + + const u64 blocks_in_this_group = + std::min(VolumeWii::BLOCKS_PER_GROUP, blocks - j * VolumeWii::BLOCKS_PER_GROUP); + + for (u32 k = 0; k < VolumeWii::BLOCKS_PER_GROUP; ++k) + { + if (k < blocks_in_this_group) + { + const u64 offset_of_block = offset_of_group + k * VolumeWii::BLOCK_TOTAL_SIZE; + VolumeWii::DecryptBlockData(buffer.data() + offset_of_block, + decryption_buffer[k].data(), &aes_context); + } + else + { + decryption_buffer[k].fill(0); + } + } + + VolumeWii::HashGroup(decryption_buffer.data(), hash_buffer.data()); + + for (u64 k = 0; k < blocks_in_this_group; ++k) + { + const u64 offset_of_block = offset_of_group + k * VolumeWii::BLOCK_TOTAL_SIZE; + const u64 hash_offset_of_block = k * VolumeWii::BLOCK_HEADER_SIZE; + + VolumeWii::HashBlock hashes; + VolumeWii::DecryptBlockHashes(buffer.data() + offset_of_block, &hashes, &aes_context); + + const auto compare_hash = [&](size_t offset_in_block) { + ASSERT(offset_in_block + sizeof(SHA1) <= VolumeWii::BLOCK_HEADER_SIZE); + + const u8* desired_hash = reinterpret_cast(&hashes) + offset_in_block; + const u8* computed_hash = reinterpret_cast(&hash_buffer[k]) + offset_in_block; + + if (!std::equal(desired_hash, desired_hash + sizeof(SHA1), computed_hash)) + { + const u64 hash_offset = hash_offset_of_block + offset_in_block; + ASSERT(hash_offset <= std::numeric_limits::max()); + + HashExceptionEntry& exception = exception_lists[j].emplace_back(); + exception.offset = static_cast(Common::swap16(hash_offset)); + std::memcpy(exception.hash.data(), desired_hash, sizeof(SHA1)); + } + }; + + const auto compare_hashes = [&compare_hash](size_t offset, size_t size) { + for (size_t l = 0; l < size; l += sizeof(SHA1)) + // The std::min is to ensure that we don't go beyond the end of HashBlock with + // padding_2, which is 32 bytes long (not divisible by sizeof(SHA1), which is 20). + compare_hash(offset + std::min(l, size - sizeof(SHA1))); + }; + + using HashBlock = VolumeWii::HashBlock; + compare_hashes(offsetof(HashBlock, h0), sizeof(HashBlock::h0)); + compare_hashes(offsetof(HashBlock, padding_0), sizeof(HashBlock::padding_0)); + compare_hashes(offsetof(HashBlock, h1), sizeof(HashBlock::h1)); + compare_hashes(offsetof(HashBlock, padding_1), sizeof(HashBlock::padding_1)); + compare_hashes(offsetof(HashBlock, h2), sizeof(HashBlock::h2)); + compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); + } + + for (u64 k = 0; k < blocks_in_this_group; ++k) + { + std::memcpy(buffer.data() + write_offset_of_group + k * VolumeWii::BLOCK_DATA_SIZE, + decryption_buffer[k].data(), VolumeWii::BLOCK_DATA_SIZE); + } + } + + const u64 write_offset = bytes_written; - for (u64 j = 0; j < exception_lists; ++j) + for (const std::vector& exception_list : exception_lists) { - const u16 exceptions = 0; + const u16 exceptions = Common::swap16(static_cast(exception_list.size())); if (!outfile->WriteArray(&exceptions, 1)) return ConversionResult::WriteFailed; - bytes_written += sizeof(u16); + + if (!outfile->WriteArray(exception_list.data(), exception_list.size())) + return ConversionResult::WriteFailed; + + bytes_written += sizeof(u16) + exception_list.size() * sizeof(HashExceptionEntry); } if (!PadTo4(outfile, &bytes_written)) return ConversionResult::WriteFailed; - for (u64 j = 0; j < bytes_to_read; j += VolumeWii::BLOCK_TOTAL_SIZE) - { - VolumeWii::DecryptBlockData(buffer.data() + j, decryption_buffer.data(), &aes_context); - if (!outfile->WriteArray(decryption_buffer.data(), VolumeWii::BLOCK_DATA_SIZE)) - return ConversionResult::WriteFailed; - } + if (!outfile->WriteArray(buffer.data(), bytes_to_write)) + return ConversionResult::WriteFailed; bytes_read += bytes_to_read; bytes_written += bytes_to_write; @@ -1188,6 +1263,10 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, if (!PadTo4(outfile, &bytes_written)) return ConversionResult::WriteFailed; + ASSERT((write_offset & 3) == 0); + group_entries[i].data_offset = Common::swap32(static_cast(write_offset >> 2)); + group_entries[i].data_size = Common::swap32(static_cast(bytes_written - write_offset)); + if (!run_callback()) return ConversionResult::Canceled; } From e8b019ac29a254689c745681e47c0a5f282428d1 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 17 Apr 2020 20:38:33 +0200 Subject: [PATCH 18/36] WIA: Implement compression --- Source/Core/DiscIO/Blob.h | 5 +- Source/Core/DiscIO/WIABlob.cpp | 571 +++++++++++++++++++++--- Source/Core/DiscIO/WIABlob.h | 125 +++++- Source/Core/DolphinQt/ConvertDialog.cpp | 88 +++- Source/Core/DolphinQt/ConvertDialog.h | 10 + 5 files changed, 708 insertions(+), 91 deletions(-) diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 692ac5707260..49e42e67415f 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -25,6 +25,8 @@ namespace DiscIO { +enum class WIACompressionType : u32; + // Increment CACHE_REVISION (GameFileCache.cpp) if the enum below is modified enum class BlobType { @@ -174,7 +176,8 @@ bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, const std::string& outfile_path, CompressCB callback = nullptr, void* arg = nullptr); bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, - const std::string& outfile_path, int chunk_size, CompressCB callback = nullptr, + const std::string& outfile_path, WIACompressionType compression_type, + int compression_level, int chunk_size, CompressCB callback = nullptr, void* arg = nullptr); } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 157a59f01ba3..b482f4538c10 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -100,8 +100,8 @@ bool WIAFileReader::Initialize(const std::string& path) return false; const u32 compression_type = Common::swap32(m_header_2.compression_type); - m_compression_type = static_cast(compression_type); - if (m_compression_type > CompressionType::LZMA2) + m_compression_type = static_cast(compression_type); + if (m_compression_type > WIACompressionType::LZMA2) { ERROR_LOG(DISCIO, "Unsupported WIA compression type %u in %s", compression_type, path.c_str()); return false; @@ -429,26 +429,26 @@ WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 std::unique_ptr decompressor; switch (m_compression_type) { - case CompressionType::None: + case WIACompressionType::None: decompressor = std::make_unique(); break; - case CompressionType::Purge: + case WIACompressionType::Purge: decompressor = std::make_unique(decompressed_size); break; - case CompressionType::Bzip2: + case WIACompressionType::Bzip2: decompressor = std::make_unique(); break; - case CompressionType::LZMA: + case WIACompressionType::LZMA: decompressor = std::make_unique(false, m_header_2.compressor_data, m_header_2.compressor_data_size); break; - case CompressionType::LZMA2: + case WIACompressionType::LZMA2: decompressor = std::make_unique(true, m_header_2.compressor_data, m_header_2.compressor_data_size); break; } - const bool compressed_exception_lists = m_compression_type > CompressionType::Purge; + const bool compressed_exception_lists = m_compression_type > WIACompressionType::Purge; m_cached_chunk = Chunk(&m_file, offset_in_file, compressed_size, decompressed_size, exception_lists, compressed_exception_lists, std::move(decompressor)); @@ -469,6 +469,11 @@ std::string WIAFileReader::VersionToString(u32 version) return StringFromFormat("%u.%02x.%02x.beta%u", a, b, c, d); } +u32 WIAFileReader::LZMA2DictionarySize(u8 p) +{ + return (static_cast(2) | (p & 1)) << (p / 2 + 11); +} + WIAFileReader::Decompressor::~Decompressor() = default; bool WIAFileReader::NoneDecompressor::Decompress(const DecompressionBuffer& in, @@ -659,7 +664,7 @@ WIAFileReader::LZMADecompressor::LZMADecompressor(bool lzma2, const u8* filter_o if (d > 40) m_error_occurred = true; else - m_options.dict_size = d == 40 ? 0xFFFFFFFF : (static_cast(2) | (d & 1)) << (d / 2 + 11); + m_options.dict_size = d == 40 ? 0xFFFFFFFF : LZMA2DictionarySize(d); } else { @@ -706,6 +711,303 @@ bool WIAFileReader::LZMADecompressor::Decompress(const DecompressionBuffer& in, return result == LZMA_OK || result == LZMA_STREAM_END; } +WIAFileReader::Compressor::~Compressor() = default; + +WIAFileReader::PurgeCompressor::PurgeCompressor() +{ + mbedtls_sha1_init(&m_sha1_context); +} + +WIAFileReader::PurgeCompressor::~PurgeCompressor() = default; + +bool WIAFileReader::PurgeCompressor::Start() +{ + m_buffer.clear(); + m_bytes_written = 0; + + mbedtls_sha1_starts_ret(&m_sha1_context); + + return true; +} + +bool WIAFileReader::PurgeCompressor::AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) +{ + mbedtls_sha1_update_ret(&m_sha1_context, data, size); + return true; +} + +bool WIAFileReader::PurgeCompressor::Compress(const u8* data, size_t size) +{ + // We could add support for calling this twice if we're fine with + // making the code more complicated, but there's no need to support it + ASSERT_MSG(DISCIO, m_bytes_written == 0, + "Calling PurgeCompressor::Compress() twice is not supported"); + + m_buffer.resize(size + sizeof(PurgeSegment) + sizeof(SHA1)); + + size_t bytes_read = 0; + + while (true) + { + const auto first_non_zero = + std::find_if(data + bytes_read, data + size, [](u8 x) { return x != 0; }); + + const u32 non_zero_data_start = static_cast(first_non_zero - data); + if (non_zero_data_start == size) + break; + + size_t non_zero_data_end = non_zero_data_start; + size_t sequence_length = 0; + for (size_t i = non_zero_data_start; i < size; ++i) + { + if (data[i] == 0) + { + ++sequence_length; + } + else + { + sequence_length = 0; + non_zero_data_end = i + 1; + } + + // To avoid wasting space, only count runs of zeroes that are of a certain length + // (unless there is nothing after the run of zeroes, then we might as well always count it) + if (sequence_length > sizeof(PurgeSegment)) + break; + } + + const u32 non_zero_data_length = static_cast(non_zero_data_end - non_zero_data_start); + + const PurgeSegment segment{Common::swap32(non_zero_data_start), + Common::swap32(non_zero_data_length)}; + std::memcpy(m_buffer.data() + m_bytes_written, &segment, sizeof(segment)); + m_bytes_written += sizeof(segment); + + std::memcpy(m_buffer.data() + m_bytes_written, data + non_zero_data_start, + non_zero_data_length); + m_bytes_written += non_zero_data_length; + + bytes_read = non_zero_data_end; + } + + return true; +} + +bool WIAFileReader::PurgeCompressor::End() +{ + mbedtls_sha1_update_ret(&m_sha1_context, m_buffer.data(), m_bytes_written); + + mbedtls_sha1_finish_ret(&m_sha1_context, m_buffer.data() + m_bytes_written); + m_bytes_written += sizeof(SHA1); + + ASSERT(m_bytes_written <= m_buffer.size()); + + return true; +} + +const u8* WIAFileReader::PurgeCompressor::GetData() const +{ + return m_buffer.data(); +} + +size_t WIAFileReader::PurgeCompressor::GetSize() const +{ + return m_bytes_written; +} + +WIAFileReader::Bzip2Compressor::Bzip2Compressor(int compression_level) + : m_compression_level(compression_level) +{ +} + +WIAFileReader::Bzip2Compressor::~Bzip2Compressor() +{ + BZ2_bzCompressEnd(&m_stream); +} + +bool WIAFileReader::Bzip2Compressor::Start() +{ + ASSERT_MSG(DISCIO, m_stream.state == nullptr, + "Called Bzip2Compressor::Start() twice without calling Bzip2Compressor::End()"); + + m_buffer.clear(); + m_stream.next_out = reinterpret_cast(m_buffer.data()); + + return BZ2_bzCompressInit(&m_stream, m_compression_level, 0, 0) == BZ_OK; +} + +bool WIAFileReader::Bzip2Compressor::Compress(const u8* data, size_t size) +{ + m_stream.next_in = reinterpret_cast(const_cast(data)); + m_stream.avail_in = static_cast(size); + + ExpandBuffer(size); + + while (m_stream.avail_in != 0) + { + if (m_stream.avail_out == 0) + ExpandBuffer(0x100); + + if (BZ2_bzCompress(&m_stream, BZ_RUN) != BZ_RUN_OK) + return false; + } + + return true; +} + +bool WIAFileReader::Bzip2Compressor::End() +{ + bool success = true; + + while (true) + { + if (m_stream.avail_out == 0) + ExpandBuffer(0x100); + + const int result = BZ2_bzCompress(&m_stream, BZ_FINISH); + if (result != BZ_FINISH_OK && result != BZ_STREAM_END) + success = false; + if (result != BZ_FINISH_OK) + break; + } + + if (BZ2_bzCompressEnd(&m_stream) != BZ_OK) + success = false; + + return success; +} + +void WIAFileReader::Bzip2Compressor::ExpandBuffer(size_t bytes_to_add) +{ + const size_t bytes_written = GetSize(); + m_buffer.resize(m_buffer.size() + bytes_to_add); + m_stream.next_out = reinterpret_cast(m_buffer.data()) + bytes_written; + m_stream.avail_out = static_cast(m_buffer.size() - bytes_written); +} + +const u8* WIAFileReader::Bzip2Compressor::GetData() const +{ + return m_buffer.data(); +} + +size_t WIAFileReader::Bzip2Compressor::GetSize() const +{ + return static_cast(reinterpret_cast(m_stream.next_out) - m_buffer.data()); +} + +WIAFileReader::LZMACompressor::LZMACompressor(bool lzma2, int compression_level, + u8 compressor_data_out[7], + u8* compressor_data_size_out) +{ + // lzma_lzma_preset returns false on success for some reason + if (lzma_lzma_preset(&m_options, static_cast(compression_level))) + { + m_initialization_failed = true; + return; + } + + if (!lzma2) + { + *compressor_data_size_out = 5; + + ASSERT(m_options.lc < 9); + ASSERT(m_options.lp < 5); + ASSERT(m_options.pb < 5); + compressor_data_out[0] = static_cast((m_options.pb * 5 + m_options.lp) * 9 + m_options.lc); + + // The dictionary size is stored as a 32-bit little endian unsigned integer + static_assert(sizeof(m_options.dict_size) == sizeof(u32)); + std::memcpy(compressor_data_out + 1, &m_options.dict_size, sizeof(u32)); + } + else + { + *compressor_data_size_out = 1; + + u8 encoded_dict_size = 0; + while (encoded_dict_size < 40 && m_options.dict_size > LZMA2DictionarySize(encoded_dict_size)) + ++encoded_dict_size; + + compressor_data_out[0] = encoded_dict_size; + } + + m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1; + m_filters[0].options = &m_options; + m_filters[1].id = LZMA_VLI_UNKNOWN; + m_filters[1].options = nullptr; +} + +WIAFileReader::LZMACompressor::~LZMACompressor() +{ + lzma_end(&m_stream); +} + +bool WIAFileReader::LZMACompressor::Start() +{ + if (m_initialization_failed) + return false; + + m_buffer.clear(); + m_stream.next_out = m_buffer.data(); + + return lzma_raw_encoder(&m_stream, m_filters) == LZMA_OK; +} + +bool WIAFileReader::LZMACompressor::Compress(const u8* data, size_t size) +{ + m_stream.next_in = data; + m_stream.avail_in = size; + + ExpandBuffer(size); + + while (m_stream.avail_in != 0) + { + if (m_stream.avail_out == 0) + ExpandBuffer(0x100); + + if (lzma_code(&m_stream, LZMA_RUN) != LZMA_OK) + return false; + } + + return true; +} + +bool WIAFileReader::LZMACompressor::End() +{ + while (true) + { + if (m_stream.avail_out == 0) + ExpandBuffer(0x100); + + switch (lzma_code(&m_stream, LZMA_FINISH)) + { + case LZMA_OK: + break; + case LZMA_STREAM_END: + return true; + default: + return false; + } + } +} + +void WIAFileReader::LZMACompressor::ExpandBuffer(size_t bytes_to_add) +{ + const size_t bytes_written = GetSize(); + m_buffer.resize(m_buffer.size() + bytes_to_add); + m_stream.next_out = m_buffer.data() + bytes_written; + m_stream.avail_out = m_buffer.size() - bytes_written; +} + +const u8* WIAFileReader::LZMACompressor::GetData() const +{ + return m_buffer.data(); +} + +size_t WIAFileReader::LZMACompressor::GetSize() const +{ + return static_cast(m_stream.next_out - m_buffer.data()); +} + WIAFileReader::Chunk::Chunk() = default; WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, @@ -1059,30 +1361,162 @@ WIAFileReader::ConversionResult WIAFileReader::SetUpDataEntriesForWriting( return ConversionResult::Success; } -WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, - const VolumeDisc* infile_volume, - File::IOFile* outfile, int chunk_size, - CompressCB callback, void* arg) +WIAFileReader::ConversionResult WIAFileReader::CompressAndWriteGroup( + File::IOFile* file, u64* bytes_written, std::vector* group_entries, + size_t* groups_written, Compressor* compressor, bool compressed_exception_lists, + const std::vector& exception_lists, const std::vector& main_data) +{ + const u64 data_offset = *bytes_written; + + if (compressor) + { + if (!compressor->Start()) + return ConversionResult::InternalError; + } + + if (!exception_lists.empty()) + { + if (compressed_exception_lists && compressor) + { + if (!compressor->Compress(exception_lists.data(), exception_lists.size())) + return ConversionResult::InternalError; + } + else + { + *bytes_written += exception_lists.size(); + if (!file->WriteArray(exception_lists.data(), exception_lists.size())) + return ConversionResult::WriteFailed; + + const u64 offset_of_padding = *bytes_written; + if (!compressed_exception_lists) + { + if (!PadTo4(file, bytes_written)) + return ConversionResult::WriteFailed; + } + const u64 padding_written = *bytes_written - offset_of_padding; + + // Some extra stuff we have to do because Purge for some reason is supposed to hash + // the exception lists and the following padding but not actually compress them... + if (compressor) + { + if (!compressor->AddPrecedingDataOnlyForPurgeHashing(exception_lists.data(), + exception_lists.size())) + { + return ConversionResult::InternalError; + } + + constexpr u32 ZEROES = 0; + if (!compressor->AddPrecedingDataOnlyForPurgeHashing(reinterpret_cast(ZEROES), + padding_written)) + { + return ConversionResult::InternalError; + } + } + } + } + + if (compressor) + { + if (!compressor->Compress(main_data.data(), main_data.size())) + return ConversionResult::InternalError; + if (!compressor->End()) + return ConversionResult::InternalError; + } + + const u8* data = compressor ? compressor->GetData() : main_data.data(); + const size_t size = compressor ? compressor->GetSize() : main_data.size(); + + *bytes_written += size; + if (!file->WriteArray(data, size)) + return ConversionResult::WriteFailed; + + if (*bytes_written >> 2 > std::numeric_limits::max()) + return ConversionResult::InternalError; + + ASSERT((data_offset & 3) == 0); + GroupEntry& group_entry = (*group_entries)[*groups_written]; + group_entry.data_offset = Common::swap32(static_cast(data_offset >> 2)); + group_entry.data_size = Common::swap32(static_cast(*bytes_written - data_offset)); + ++*groups_written; + + if (!PadTo4(file, bytes_written)) + return ConversionResult::WriteFailed; + + return ConversionResult::Success; +} + +WIAFileReader::ConversionResult +WIAFileReader::CompressAndWrite(File::IOFile* file, u64* bytes_written, Compressor* compressor, + const u8* data, size_t size, size_t* size_out) +{ + if (compressor) + { + if (!compressor->Start() || !compressor->Compress(data, size) || !compressor->End()) + return ConversionResult::InternalError; + + data = compressor->GetData(); + size = compressor->GetSize(); + } + + *size_out = size; + + *bytes_written += size; + if (!file->WriteArray(data, size)) + return ConversionResult::WriteFailed; + + if (!PadTo4(file, bytes_written)) + return ConversionResult::WriteFailed; + + return ConversionResult::Success; +} + +WIAFileReader::ConversionResult +WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, + File::IOFile* outfile, WIACompressionType compression_type, + int compression_level, int chunk_size, CompressCB callback, void* arg) { ASSERT(infile->IsDataSizeAccurate()); ASSERT(chunk_size > 0); const u64 iso_size = infile->GetDataSize(); const u64 exception_lists_per_chunk = chunk_size / VolumeWii::GROUP_TOTAL_SIZE; + const bool compressed_exception_lists = compression_type > WIACompressionType::Purge; u64 bytes_read = 0; u64 bytes_written = 0; size_t groups_written = 0; - // These two headers will be filled in with proper values at the very end - WIAHeader1 header_1; - WIAHeader2 header_2; + // These two headers will be filled in with proper values later + WIAHeader1 header_1{}; + WIAHeader2 header_2{}; if (!outfile->WriteArray(&header_1, 1) || !outfile->WriteArray(&header_2, 1)) return ConversionResult::WriteFailed; bytes_written += sizeof(WIAHeader1) + sizeof(WIAHeader2); if (!PadTo4(outfile, &bytes_written)) return ConversionResult::WriteFailed; + std::unique_ptr compressor; + switch (compression_type) + { + case WIACompressionType::None: + compressor = nullptr; + break; + case WIACompressionType::Purge: + compressor = std::make_unique(); + break; + case WIACompressionType::Bzip2: + compressor = std::make_unique(compression_level); + break; + case WIACompressionType::LZMA: + compressor = std::make_unique( + false, compression_level, header_2.compressor_data, &header_2.compressor_data_size); + break; + case WIACompressionType::LZMA2: + compressor = std::make_unique(true, compression_level, header_2.compressor_data, + &header_2.compressor_data_size); + break; + } + std::vector partition_entries; std::vector raw_data_entries; std::vector group_entries; @@ -1123,9 +1557,16 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, using WiiBlockData = std::array; - std::vector buffer(chunk_size); - std::vector decryption_buffer(VolumeWii::BLOCKS_PER_GROUP); - std::vector hash_buffer(VolumeWii::BLOCKS_PER_GROUP); + std::vector buffer; + std::vector exceptions_buffer; + std::vector decryption_buffer; + std::vector hash_buffer; + + if (!partition_entries.empty()) + { + decryption_buffer.resize(VolumeWii::BLOCKS_PER_GROUP); + hash_buffer.resize(VolumeWii::BLOCKS_PER_GROUP); + } for (const DataEntry& data_entry : data_entries) { @@ -1160,8 +1601,10 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, const u64 blocks = bytes_to_read / VolumeWii::BLOCK_TOTAL_SIZE; const u64 bytes_to_write = blocks * VolumeWii::BLOCK_DATA_SIZE; + buffer.resize(bytes_to_read); if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) return ConversionResult::ReadFailed; + bytes_read += bytes_to_read; std::vector> exception_lists(exception_lists_per_chunk); @@ -1237,35 +1680,23 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, } } - const u64 write_offset = bytes_written; - + exceptions_buffer.clear(); for (const std::vector& exception_list : exception_lists) { const u16 exceptions = Common::swap16(static_cast(exception_list.size())); - if (!outfile->WriteArray(&exceptions, 1)) - return ConversionResult::WriteFailed; - - if (!outfile->WriteArray(exception_list.data(), exception_list.size())) - return ConversionResult::WriteFailed; - - bytes_written += sizeof(u16) + exception_list.size() * sizeof(HashExceptionEntry); + PushBack(&exceptions_buffer, exceptions); + for (const HashExceptionEntry& exception : exception_list) + PushBack(&exceptions_buffer, exception); } - if (!PadTo4(outfile, &bytes_written)) - return ConversionResult::WriteFailed; + buffer.resize(bytes_to_write); - if (!outfile->WriteArray(buffer.data(), bytes_to_write)) - return ConversionResult::WriteFailed; - - bytes_read += bytes_to_read; - bytes_written += bytes_to_write; - ++groups_written; - if (!PadTo4(outfile, &bytes_written)) - return ConversionResult::WriteFailed; + const ConversionResult write_result = CompressAndWriteGroup( + outfile, &bytes_written, &group_entries, &groups_written, compressor.get(), + compressed_exception_lists, exceptions_buffer, buffer); - ASSERT((write_offset & 3) == 0); - group_entries[i].data_offset = Common::swap32(static_cast(write_offset >> 2)); - group_entries[i].data_size = Common::swap32(static_cast(bytes_written - write_offset)); + if (write_result != ConversionResult::Success) + return write_result; if (!run_callback()) return ConversionResult::Canceled; @@ -1288,27 +1719,24 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, ASSERT(groups_written == first_group); ASSERT(bytes_read == data_offset); + exceptions_buffer.clear(); + for (u32 i = first_group; i < last_group; ++i) { const u64 bytes_to_read = std::min(chunk_size, data_offset + data_size - bytes_read); - if (bytes_written >> 2 > std::numeric_limits::max()) - return ConversionResult::InternalError; - - ASSERT((bytes_written & 3) == 0); - group_entries[i].data_offset = Common::swap32(static_cast(bytes_written >> 2)); - group_entries[i].data_size = Common::swap32(static_cast(bytes_to_read)); + buffer.resize(bytes_to_read); if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) return ConversionResult::ReadFailed; - if (!outfile->WriteArray(buffer.data(), bytes_to_read)) - return ConversionResult::WriteFailed; - bytes_read += bytes_to_read; - bytes_written += bytes_to_read; - ++groups_written; - if (!PadTo4(outfile, &bytes_written)) - return ConversionResult::WriteFailed; + + const ConversionResult write_result = CompressAndWriteGroup( + outfile, &bytes_written, &group_entries, &groups_written, compressor.get(), + compressed_exception_lists, exceptions_buffer, buffer); + + if (write_result != ConversionResult::Success) + return write_result; if (!run_callback()) return ConversionResult::Canceled; @@ -1328,20 +1756,16 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, return ConversionResult::WriteFailed; const u64 raw_data_entries_offset = bytes_written; - const u64 raw_data_entries_size = raw_data_entries.size() * sizeof(RawDataEntry); - if (!outfile->WriteArray(raw_data_entries.data(), raw_data_entries.size())) - return ConversionResult::WriteFailed; - bytes_written += raw_data_entries_size; - if (!PadTo4(outfile, &bytes_written)) - return ConversionResult::WriteFailed; + size_t raw_data_entries_size = raw_data_entries.size() * sizeof(RawDataEntry); + const ConversionResult raw_data_result = CompressAndWrite( + outfile, &bytes_written, compressor.get(), reinterpret_cast(raw_data_entries.data()), + raw_data_entries_size, &raw_data_entries_size); const u64 group_entries_offset = bytes_written; - const u64 group_entries_size = group_entries.size() * sizeof(GroupEntry); - if (!outfile->WriteArray(group_entries.data(), group_entries.size())) - return ConversionResult::WriteFailed; - bytes_written += group_entries_size; - if (!PadTo4(outfile, &bytes_written)) - return ConversionResult::WriteFailed; + size_t group_entries_size = group_entries.size() * sizeof(GroupEntry); + const ConversionResult groups_result = CompressAndWrite( + outfile, &bytes_written, compressor.get(), reinterpret_cast(group_entries.data()), + group_entries_size, &group_entries_size); u32 disc_type = 0; if (infile_volume) @@ -1353,8 +1777,8 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, } header_2.disc_type = Common::swap32(disc_type); - header_2.compression_type = Common::swap32(static_cast(CompressionType::None)); - header_2.compression_level = 0; + header_2.compression_type = Common::swap32(static_cast(compression_type)); + header_2.compression_level = Common::swap32(static_cast(compression_level)); header_2.chunk_size = Common::swap32(static_cast(chunk_size)); header_2.number_of_partition_entries = Common::swap32(static_cast(partition_entries.size())); @@ -1374,9 +1798,6 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, header_2.group_entries_offset = Common::swap64(group_entries_offset); header_2.group_entries_size = Common::swap32(static_cast(group_entries_size)); - header_2.compressor_data_size = 0; - std::fill(std::begin(header_2.compressor_data), std::end(header_2.compressor_data), 0); - header_1.magic = WIA_MAGIC; header_1.version = Common::swap32(WIA_VERSION); header_1.version_compatible = Common::swap32(WIA_VERSION_WRITE_COMPATIBLE); @@ -1397,7 +1818,8 @@ WIAFileReader::ConversionResult WIAFileReader::ConvertToWIA(BlobReader* infile, } bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, - const std::string& outfile_path, int chunk_size, CompressCB callback, void* arg) + const std::string& outfile_path, WIACompressionType compression_type, + int compression_level, int chunk_size, CompressCB callback, void* arg) { File::IOFile outfile(outfile_path, "wb"); if (!outfile) @@ -1411,8 +1833,9 @@ bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, std::unique_ptr infile_volume = CreateDisc(infile_path); - WIAFileReader::ConversionResult result = - WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, chunk_size, callback, arg); + const WIAFileReader::ConversionResult result = + WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, compression_type, + compression_level, chunk_size, callback, arg); if (result == WIAFileReader::ConversionResult::ReadFailed) PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index c1f46cc5eae0..dafaa13f33a0 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -24,6 +24,15 @@ namespace DiscIO { class VolumeDisc; +enum class WIACompressionType : u32 +{ + None = 0, + Purge = 1, + Bzip2 = 2, + LZMA = 3, + LZMA2 = 4, +}; + constexpr u32 WIA_MAGIC = 0x01414957; // "WIA\x1" (byteswapped to little endian) class WIAFileReader : public BlobReader @@ -56,22 +65,14 @@ class WIAFileReader : public BlobReader }; static ConversionResult ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, int chunk_size, CompressCB callback, + File::IOFile* outfile, WIACompressionType compression_type, + int compression_level, int chunk_size, CompressCB callback, void* arg); private: using SHA1 = std::array; using WiiKey = std::array; - enum class CompressionType : u32 - { - None = 0, - Purge = 1, - Bzip2 = 2, - LZMA = 3, - LZMA2 = 4, - }; - // See docs/WIA.md for details about the format #pragma pack(push, 1) @@ -253,6 +254,88 @@ class WIAFileReader : public BlobReader bool m_error_occurred = false; }; + class Compressor + { + public: + virtual ~Compressor(); + + // First call Start, then AddDataOnlyForPurgeHashing/Compress any number of times, + // then End, then GetData/GetSize any number of times. + + virtual bool Start() = 0; + virtual bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) { return true; } + virtual bool Compress(const u8* data, size_t size) = 0; + virtual bool End() = 0; + + virtual const u8* GetData() const = 0; + virtual size_t GetSize() const = 0; + }; + + class PurgeCompressor final : public Compressor + { + public: + PurgeCompressor(); + ~PurgeCompressor(); + + bool Start() override; + bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) override; + bool Compress(const u8* data, size_t size) override; + bool End() override; + + const u8* GetData() const override; + size_t GetSize() const override; + + private: + std::vector m_buffer; + size_t m_bytes_written; + mbedtls_sha1_context m_sha1_context; + }; + + class Bzip2Compressor final : public Compressor + { + public: + Bzip2Compressor(int compression_level); + ~Bzip2Compressor(); + + bool Start() override; + bool Compress(const u8* data, size_t size) override; + bool End() override; + + const u8* GetData() const override; + size_t GetSize() const override; + + private: + void ExpandBuffer(size_t bytes_to_add); + + bz_stream m_stream = {}; + std::vector m_buffer; + int m_compression_level; + }; + + class LZMACompressor final : public Compressor + { + public: + LZMACompressor(bool lzma2, int compression_level, u8 compressor_data_out[7], + u8* compressor_data_size_out); + ~LZMACompressor(); + + bool Start() override; + bool Compress(const u8* data, size_t size) override; + bool End() override; + + const u8* GetData() const override; + size_t GetSize() const override; + + private: + void ExpandBuffer(size_t bytes_to_add); + + lzma_stream m_stream = LZMA_STREAM_INIT; + lzma_options_lzma m_options = {}; + lzma_filter m_filters[2]; + std::vector m_buffer; + bool m_initialization_failed = false; + }; + class Chunk { public: @@ -304,6 +387,8 @@ class WIAFileReader : public BlobReader static std::string VersionToString(u32 version); + static u32 LZMA2DictionarySize(u8 p); + static bool PadTo4(File::IOFile* file, u64* bytes_written); static void AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups, std::vector* raw_data_entries, @@ -317,9 +402,27 @@ class WIAFileReader : public BlobReader std::vector* partition_entries, std::vector* raw_data_entries, std::vector* data_entries); + static ConversionResult CompressAndWriteGroup(File::IOFile* file, u64* bytes_written, + std::vector* group_entries, + size_t* groups_written, Compressor* compressor, + bool compressed_exception_lists, + const std::vector& exception_lists, + const std::vector& main_data); + static ConversionResult CompressAndWrite(File::IOFile* file, u64* bytes_written, + Compressor* compressor, const u8* data, size_t size, + size_t* size_out); + + template + static void PushBack(std::vector* vector, const T& x) + { + const size_t offset_in_vector = vector->size(); + vector->resize(offset_in_vector + sizeof(T)); + const u8* x_ptr = reinterpret_cast(&x); + std::copy(x_ptr, x_ptr + sizeof(T), vector->data() + offset_in_vector); + } bool m_valid; - CompressionType m_compression_type; + WIACompressionType m_compression_type; File::IOFile m_file; Chunk m_cached_chunk; diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index 831c0ec64147..e52868039bd9 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -26,6 +26,7 @@ #include "Common/Logging/Log.h" #include "DiscIO/Blob.h" #include "DiscIO/ScrubbedBlob.h" +#include "DiscIO/WIABlob.h" #include "DolphinQt/QtUtils/ModalMessageBox.h" #include "DolphinQt/QtUtils/ParallelProgressDialog.h" #include "UICommon/GameFile.h" @@ -70,9 +71,17 @@ ConvertDialog::ConvertDialog(QList> fi grid_layout->addWidget(new QLabel(tr("Block Size:")), 1, 0); grid_layout->addWidget(m_block_size, 1, 1); + m_compression = new QComboBox; + grid_layout->addWidget(new QLabel(tr("Compression:")), 2, 0); + grid_layout->addWidget(m_compression, 2, 1); + + m_compression_level = new QComboBox; + grid_layout->addWidget(new QLabel(tr("Compression Level:")), 3, 0); + grid_layout->addWidget(m_compression_level, 3, 1); + m_scrub = new QCheckBox; - grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 2, 0); - grid_layout->addWidget(m_scrub, 2, 1); + grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 4, 0); + grid_layout->addWidget(m_scrub, 4, 1); m_scrub->setEnabled( std::none_of(m_files.begin(), m_files.end(), std::mem_fn(&UICommon::GameFile::IsDatelDisc))); @@ -108,9 +117,12 @@ ConvertDialog::ConvertDialog(QList> fi connect(m_format, QOverload::of(&QComboBox::currentIndexChanged), this, &ConvertDialog::OnFormatChanged); + connect(m_compression, QOverload::of(&QComboBox::currentIndexChanged), this, + &ConvertDialog::OnCompressionChanged); connect(convert_button, &QPushButton::clicked, this, &ConvertDialog::Convert); OnFormatChanged(); + OnCompressionChanged(); } void ConvertDialog::AddToBlockSizeComboBox(int size) @@ -118,6 +130,16 @@ void ConvertDialog::AddToBlockSizeComboBox(int size) m_block_size->addItem(QString::fromStdString(UICommon::FormatSize(size, 0)), size); } +void ConvertDialog::AddToCompressionComboBox(const QString& name, DiscIO::WIACompressionType type) +{ + m_compression->addItem(name, static_cast(type)); +} + +void ConvertDialog::AddToCompressionLevelComboBox(int level) +{ + m_compression_level->addItem(QString::number(level), level); +} + void ConvertDialog::OnFormatChanged() { // Because DVD timings are emulated as if we can't read less than an entire ECC block at once @@ -131,6 +153,9 @@ void ConvertDialog::OnFormatChanged() const DiscIO::BlobType format = static_cast(m_format->currentData().toInt()); m_block_size->clear(); + m_compression->clear(); + + // Populate m_block_size switch (format) { case DiscIO::BlobType::GCZ: @@ -181,7 +206,57 @@ void ConvertDialog::OnFormatChanged() break; } + // Populate m_compression + switch (format) + { + case DiscIO::BlobType::GCZ: + m_compression->setEnabled(true); + AddToCompressionComboBox(QStringLiteral("Deflate"), DiscIO::WIACompressionType::None); + break; + case DiscIO::BlobType::WIA: + { + m_compression->setEnabled(true); + + // i18n: %1 is the name of a compression method (e.g. LZMA) + const QString slow = tr("%1 (slow)"); + + AddToCompressionComboBox(tr("No Compression"), DiscIO::WIACompressionType::None); + AddToCompressionComboBox(QStringLiteral("Purge"), DiscIO::WIACompressionType::Purge); + AddToCompressionComboBox(slow.arg(QStringLiteral("bzip2")), DiscIO::WIACompressionType::Bzip2); + AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA")), DiscIO::WIACompressionType::LZMA); + AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA2")), DiscIO::WIACompressionType::LZMA2); + + break; + } + default: + m_compression->setEnabled(false); + break; + } + m_block_size->setEnabled(m_block_size->count() > 1); + m_compression->setEnabled(m_compression->count() > 1); +} + +void ConvertDialog::OnCompressionChanged() +{ + m_compression_level->clear(); + + switch (static_cast(m_compression->currentData().toInt())) + { + case DiscIO::WIACompressionType::Bzip2: + case DiscIO::WIACompressionType::LZMA: + case DiscIO::WIACompressionType::LZMA2: + for (int i = 1; i <= 9; ++i) + AddToCompressionLevelComboBox(i); + + m_compression_level->setCurrentIndex(4); + + break; + default: + break; + } + + m_compression_level->setEnabled(m_compression_level->count() > 1); } bool ConvertDialog::ShowAreYouSureDialog(const QString& text) @@ -200,6 +275,9 @@ void ConvertDialog::Convert() { const DiscIO::BlobType format = static_cast(m_format->currentData().toInt()); const int block_size = m_block_size->currentData().toInt(); + const DiscIO::WIACompressionType compression = + static_cast(m_compression->currentData().toInt()); + const int compression_level = m_compression_level->currentData().toInt(); const bool scrub = m_scrub->isChecked(); if (scrub && format == DiscIO::BlobType::PLAIN) @@ -369,9 +447,9 @@ void ConvertDialog::Convert() else if (format == DiscIO::BlobType::WIA) { good = std::async(std::launch::async, [&] { - const bool good = - DiscIO::ConvertToWIA(blob_reader.get(), original_path, dst_path.toStdString(), - block_size, &CompressCB, &progress_dialog); + const bool good = DiscIO::ConvertToWIA( + blob_reader.get(), original_path, dst_path.toStdString(), compression, + compression_level, block_size, &CompressCB, &progress_dialog); progress_dialog.Reset(); return good; }); diff --git a/Source/Core/DolphinQt/ConvertDialog.h b/Source/Core/DolphinQt/ConvertDialog.h index 1a3a9bbf4a6b..b14ecb6c527e 100644 --- a/Source/Core/DolphinQt/ConvertDialog.h +++ b/Source/Core/DolphinQt/ConvertDialog.h @@ -14,6 +14,11 @@ class QCheckBox; class QComboBox; +namespace DiscIO +{ +enum class WIACompressionType : u32; +} + namespace UICommon { class GameFile; @@ -29,15 +34,20 @@ class ConvertDialog final : public QDialog private slots: void OnFormatChanged(); + void OnCompressionChanged(); void Convert(); private: void AddToBlockSizeComboBox(int size); + void AddToCompressionComboBox(const QString& name, DiscIO::WIACompressionType type); + void AddToCompressionLevelComboBox(int level); bool ShowAreYouSureDialog(const QString& text); QComboBox* m_format; QComboBox* m_block_size; + QComboBox* m_compression; + QComboBox* m_compression_level; QCheckBox* m_scrub; QList> m_files; }; From 40e46aee57f5ac589c538f0e740effab50275cbf Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 19 Apr 2020 13:39:10 +0200 Subject: [PATCH 19/36] WIA: Store all-zero data efficiently --- Source/Core/DiscIO/WIABlob.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index b482f4538c10..4941bcb8e241 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -1366,6 +1366,17 @@ WIAFileReader::ConversionResult WIAFileReader::CompressAndWriteGroup( size_t* groups_written, Compressor* compressor, bool compressed_exception_lists, const std::vector& exception_lists, const std::vector& main_data) { + const auto all_zero = [](const std::vector& data) { + return std::all_of(data.begin(), data.end(), [](u8 x) { return x == 0; }); + }; + + if (all_zero(exception_lists) && all_zero(main_data)) + { + (*group_entries)[*groups_written] = GroupEntry{0, 0}; + ++*groups_written; + return ConversionResult::Success; + } + const u64 data_offset = *bytes_written; if (compressor) From e5b9e1ba1f7a8c7a48671f947cf096e0f4cb0fe9 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 19 Apr 2020 15:34:02 +0200 Subject: [PATCH 20/36] WIA: Reuse groups when writing This is useful for the way Dolphin scrubs Wii discs. The encrypted data is what gets zeroed out, but this zeroed out data then gets decrypted before being stored, and the resulting data does not compress well. However, each block of decrypted scrubbed data is identical given the same encryption key, and there's nothing stopping us from making multiple group entries point to the same offset in the file, so we only have to store one copy of this data per partition. For reference, wit zeroes out the decrypted data, but Dolphin's WIA writer can't do this because it currently doesn't know which parts of the disc are scrubbed. This is also useful for things such as storing Datel discs full of 0x55 blocks (repesenting unreadable blocks) without compression enabled. --- Source/Core/DiscIO/WIABlob.cpp | 209 +++++++++++++++++++++------------ Source/Core/DiscIO/WIABlob.h | 42 ++++++- 2 files changed, 169 insertions(+), 82 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 4941bcb8e241..f468ecaee319 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -730,7 +730,8 @@ bool WIAFileReader::PurgeCompressor::Start() return true; } -bool WIAFileReader::PurgeCompressor::AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) +bool WIAFileReader::PurgeCompressor::AddPrecedingDataOnlyForPurgeHashing(const u8* data, + size_t size) { mbedtls_sha1_update_ret(&m_sha1_context, data, size); return true; @@ -1361,10 +1362,28 @@ WIAFileReader::ConversionResult WIAFileReader::SetUpDataEntriesForWriting( return ConversionResult::Success; } +bool WIAFileReader::TryReuseGroup(std::vector* group_entries, size_t* groups_written, + std::map* reusable_groups, + std::optional reuse_id) +{ + if (!reuse_id) + return false; + + const auto it = reusable_groups->find(*reuse_id); + if (it != reusable_groups->end()) + { + (*group_entries)[*groups_written] = it->second; + ++*groups_written; + } + + return it != reusable_groups->end(); +} + WIAFileReader::ConversionResult WIAFileReader::CompressAndWriteGroup( File::IOFile* file, u64* bytes_written, std::vector* group_entries, size_t* groups_written, Compressor* compressor, bool compressed_exception_lists, - const std::vector& exception_lists, const std::vector& main_data) + const std::vector& exception_lists, const std::vector& main_data, + std::map* reusable_groups, std::optional reuse_id) { const auto all_zero = [](const std::vector& data) { return std::all_of(data.begin(), data.end(), [](u8 x) { return x == 0; }); @@ -1377,6 +1396,9 @@ WIAFileReader::ConversionResult WIAFileReader::CompressAndWriteGroup( return ConversionResult::Success; } + if (TryReuseGroup(group_entries, groups_written, reusable_groups, reuse_id)) + return ConversionResult::Success; + const u64 data_offset = *bytes_written; if (compressor) @@ -1450,6 +1472,9 @@ WIAFileReader::ConversionResult WIAFileReader::CompressAndWriteGroup( group_entry.data_size = Common::swap32(static_cast(*bytes_written - data_offset)); ++*groups_written; + if (reuse_id) + reusable_groups->emplace(*reuse_id, group_entry); + if (!PadTo4(file, bytes_written)) return ConversionResult::WriteFailed; @@ -1566,6 +1591,11 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, return ConversionResult::ReadFailed; // We intentially do not increment bytes_read here, since these bytes will be read again + const auto all_same = [](const std::vector& data) { + const u8 first_byte = data.front(); + return std::all_of(data.begin(), data.end(), [first_byte](u8 x) { return x == first_byte; }); + }; + using WiiBlockData = std::array; std::vector buffer; @@ -1573,6 +1603,8 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, std::vector decryption_buffer; std::vector hash_buffer; + std::map reusable_groups; + if (!partition_entries.empty()) { decryption_buffer.resize(VolumeWii::BLOCKS_PER_GROUP); @@ -1617,97 +1649,118 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, return ConversionResult::ReadFailed; bytes_read += bytes_to_read; - std::vector> exception_lists(exception_lists_per_chunk); + const auto create_reuse_id = [&partition_entry, bytes_to_write](u8 value, bool decrypted) { + return ReuseID{&partition_entry.partition_key, bytes_to_write, decrypted, value}; + }; - for (u64 j = 0; j < groups; ++j) - { - const u64 offset_of_group = j * VolumeWii::GROUP_TOTAL_SIZE; - const u64 write_offset_of_group = j * VolumeWii::GROUP_DATA_SIZE; - - const u64 blocks_in_this_group = - std::min(VolumeWii::BLOCKS_PER_GROUP, blocks - j * VolumeWii::BLOCKS_PER_GROUP); + std::optional reuse_id; - for (u32 k = 0; k < VolumeWii::BLOCKS_PER_GROUP; ++k) - { - if (k < blocks_in_this_group) - { - const u64 offset_of_block = offset_of_group + k * VolumeWii::BLOCK_TOTAL_SIZE; - VolumeWii::DecryptBlockData(buffer.data() + offset_of_block, - decryption_buffer[k].data(), &aes_context); - } - else - { - decryption_buffer[k].fill(0); - } - } + // Set this group as reusable if the encrypted data is all_same + if (all_same(buffer)) + reuse_id = create_reuse_id(buffer.front(), false); - VolumeWii::HashGroup(decryption_buffer.data(), hash_buffer.data()); + if (!TryReuseGroup(&group_entries, &groups_written, &reusable_groups, reuse_id)) + { + std::vector> exception_lists(exception_lists_per_chunk); - for (u64 k = 0; k < blocks_in_this_group; ++k) + for (u64 j = 0; j < groups; ++j) { - const u64 offset_of_block = offset_of_group + k * VolumeWii::BLOCK_TOTAL_SIZE; - const u64 hash_offset_of_block = k * VolumeWii::BLOCK_HEADER_SIZE; + const u64 offset_of_group = j * VolumeWii::GROUP_TOTAL_SIZE; + const u64 write_offset_of_group = j * VolumeWii::GROUP_DATA_SIZE; - VolumeWii::HashBlock hashes; - VolumeWii::DecryptBlockHashes(buffer.data() + offset_of_block, &hashes, &aes_context); + const u64 blocks_in_this_group = std::min( + VolumeWii::BLOCKS_PER_GROUP, blocks - j * VolumeWii::BLOCKS_PER_GROUP); - const auto compare_hash = [&](size_t offset_in_block) { - ASSERT(offset_in_block + sizeof(SHA1) <= VolumeWii::BLOCK_HEADER_SIZE); + for (u32 k = 0; k < VolumeWii::BLOCKS_PER_GROUP; ++k) + { + if (k < blocks_in_this_group) + { + const u64 offset_of_block = offset_of_group + k * VolumeWii::BLOCK_TOTAL_SIZE; + VolumeWii::DecryptBlockData(buffer.data() + offset_of_block, + decryption_buffer[k].data(), &aes_context); + } + else + { + decryption_buffer[k].fill(0); + } + } - const u8* desired_hash = reinterpret_cast(&hashes) + offset_in_block; - const u8* computed_hash = reinterpret_cast(&hash_buffer[k]) + offset_in_block; + VolumeWii::HashGroup(decryption_buffer.data(), hash_buffer.data()); - if (!std::equal(desired_hash, desired_hash + sizeof(SHA1), computed_hash)) - { - const u64 hash_offset = hash_offset_of_block + offset_in_block; - ASSERT(hash_offset <= std::numeric_limits::max()); + for (u64 k = 0; k < blocks_in_this_group; ++k) + { + const u64 offset_of_block = offset_of_group + k * VolumeWii::BLOCK_TOTAL_SIZE; + const u64 hash_offset_of_block = k * VolumeWii::BLOCK_HEADER_SIZE; + + VolumeWii::HashBlock hashes; + VolumeWii::DecryptBlockHashes(buffer.data() + offset_of_block, &hashes, &aes_context); + + const auto compare_hash = [&](size_t offset_in_block) { + ASSERT(offset_in_block + sizeof(SHA1) <= VolumeWii::BLOCK_HEADER_SIZE); + + const u8* desired_hash = reinterpret_cast(&hashes) + offset_in_block; + const u8* computed_hash = reinterpret_cast(&hash_buffer[k]) + offset_in_block; + + if (!std::equal(desired_hash, desired_hash + sizeof(SHA1), computed_hash)) + { + const u64 hash_offset = hash_offset_of_block + offset_in_block; + ASSERT(hash_offset <= std::numeric_limits::max()); + + HashExceptionEntry& exception = exception_lists[j].emplace_back(); + exception.offset = static_cast(Common::swap16(hash_offset)); + std::memcpy(exception.hash.data(), desired_hash, sizeof(SHA1)); + } + }; + + const auto compare_hashes = [&compare_hash](size_t offset, size_t size) { + for (size_t l = 0; l < size; l += sizeof(SHA1)) + // The std::min is to ensure that we don't go beyond the end of HashBlock with + // padding_2, which is 32 bytes long (not divisible by sizeof(SHA1), which is 20). + compare_hash(offset + std::min(l, size - sizeof(SHA1))); + }; + + using HashBlock = VolumeWii::HashBlock; + compare_hashes(offsetof(HashBlock, h0), sizeof(HashBlock::h0)); + compare_hashes(offsetof(HashBlock, padding_0), sizeof(HashBlock::padding_0)); + compare_hashes(offsetof(HashBlock, h1), sizeof(HashBlock::h1)); + compare_hashes(offsetof(HashBlock, padding_1), sizeof(HashBlock::padding_1)); + compare_hashes(offsetof(HashBlock, h2), sizeof(HashBlock::h2)); + compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); + } - HashExceptionEntry& exception = exception_lists[j].emplace_back(); - exception.offset = static_cast(Common::swap16(hash_offset)); - std::memcpy(exception.hash.data(), desired_hash, sizeof(SHA1)); - } - }; - - const auto compare_hashes = [&compare_hash](size_t offset, size_t size) { - for (size_t l = 0; l < size; l += sizeof(SHA1)) - // The std::min is to ensure that we don't go beyond the end of HashBlock with - // padding_2, which is 32 bytes long (not divisible by sizeof(SHA1), which is 20). - compare_hash(offset + std::min(l, size - sizeof(SHA1))); - }; - - using HashBlock = VolumeWii::HashBlock; - compare_hashes(offsetof(HashBlock, h0), sizeof(HashBlock::h0)); - compare_hashes(offsetof(HashBlock, padding_0), sizeof(HashBlock::padding_0)); - compare_hashes(offsetof(HashBlock, h1), sizeof(HashBlock::h1)); - compare_hashes(offsetof(HashBlock, padding_1), sizeof(HashBlock::padding_1)); - compare_hashes(offsetof(HashBlock, h2), sizeof(HashBlock::h2)); - compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); + for (u64 k = 0; k < blocks_in_this_group; ++k) + { + std::memcpy(buffer.data() + write_offset_of_group + k * VolumeWii::BLOCK_DATA_SIZE, + decryption_buffer[k].data(), VolumeWii::BLOCK_DATA_SIZE); + } } - for (u64 k = 0; k < blocks_in_this_group; ++k) + bool have_exceptions = false; + + exceptions_buffer.clear(); + for (const std::vector& exception_list : exception_lists) { - std::memcpy(buffer.data() + write_offset_of_group + k * VolumeWii::BLOCK_DATA_SIZE, - decryption_buffer[k].data(), VolumeWii::BLOCK_DATA_SIZE); + const u16 exceptions = Common::swap16(static_cast(exception_list.size())); + PushBack(&exceptions_buffer, exceptions); + for (const HashExceptionEntry& exception : exception_list) + PushBack(&exceptions_buffer, exception); + if (!exception_list.empty()) + have_exceptions = true; } - } - exceptions_buffer.clear(); - for (const std::vector& exception_list : exception_lists) - { - const u16 exceptions = Common::swap16(static_cast(exception_list.size())); - PushBack(&exceptions_buffer, exceptions); - for (const HashExceptionEntry& exception : exception_list) - PushBack(&exceptions_buffer, exception); - } + buffer.resize(bytes_to_write); - buffer.resize(bytes_to_write); + // Set this group as reusable if it lacks exceptions and the decrypted data is all_same + if (!reuse_id && !have_exceptions && all_same(buffer)) + reuse_id = create_reuse_id(buffer.front(), true); - const ConversionResult write_result = CompressAndWriteGroup( - outfile, &bytes_written, &group_entries, &groups_written, compressor.get(), - compressed_exception_lists, exceptions_buffer, buffer); + const ConversionResult write_result = CompressAndWriteGroup( + outfile, &bytes_written, &group_entries, &groups_written, compressor.get(), + compressed_exception_lists, exceptions_buffer, buffer, &reusable_groups, reuse_id); - if (write_result != ConversionResult::Success) - return write_result; + if (write_result != ConversionResult::Success) + return write_result; + } if (!run_callback()) return ConversionResult::Canceled; @@ -1742,9 +1795,13 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, return ConversionResult::ReadFailed; bytes_read += bytes_to_read; + std::optional reuse_id; + if (all_same(buffer)) + reuse_id = ReuseID{nullptr, bytes_to_read, false, buffer.front()}; + const ConversionResult write_result = CompressAndWriteGroup( outfile, &bytes_written, &group_entries, &groups_written, compressor.get(), - compressed_exception_lists, exceptions_buffer, buffer); + compressed_exception_lists, exceptions_buffer, buffer, &reusable_groups, reuse_id); if (write_result != ConversionResult::Success) return write_result; diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index dafaa13f33a0..75d2918919bc 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -389,6 +390,33 @@ class WIAFileReader : public BlobReader static u32 LZMA2DictionarySize(u8 p); + struct ReuseID + { + bool operator==(const ReuseID& other) const + { + return std::tie(partition_key, data_size, decrypted, value) == + std::tie(other.partition_key, other.data_size, other.decrypted, other.value); + } + bool operator<(const ReuseID& other) const + { + return std::tie(partition_key, data_size, decrypted, value) < + std::tie(other.partition_key, other.data_size, other.decrypted, other.value); + } + bool operator>(const ReuseID& other) const + { + return std::tie(partition_key, data_size, decrypted, value) > + std::tie(other.partition_key, other.data_size, other.decrypted, other.value); + } + bool operator!=(const ReuseID& other) const { return !operator==(other); } + bool operator>=(const ReuseID& other) const { return !operator<(other); } + bool operator<=(const ReuseID& other) const { return !operator>(other); } + + const WiiKey* partition_key; + u64 data_size; + bool decrypted; + u8 value; + }; + static bool PadTo4(File::IOFile* file, u64* bytes_written); static void AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups, std::vector* raw_data_entries, @@ -402,12 +430,14 @@ class WIAFileReader : public BlobReader std::vector* partition_entries, std::vector* raw_data_entries, std::vector* data_entries); - static ConversionResult CompressAndWriteGroup(File::IOFile* file, u64* bytes_written, - std::vector* group_entries, - size_t* groups_written, Compressor* compressor, - bool compressed_exception_lists, - const std::vector& exception_lists, - const std::vector& main_data); + static bool TryReuseGroup(std::vector* group_entries, size_t* groups_written, + std::map* reusable_groups, + std::optional reuse_id); + static ConversionResult CompressAndWriteGroup( + File::IOFile* file, u64* bytes_written, std::vector* group_entries, + size_t* groups_written, Compressor* compressor, bool compressed_exception_lists, + const std::vector& exception_lists, const std::vector& main_data, + std::map* reusable_groups, std::optional reuse_id); static ConversionResult CompressAndWrite(File::IOFile* file, u64* bytes_written, Compressor* compressor, const u8* data, size_t size, size_t* size_out); From 9dea8169e09af9a9e4c15db04c36db1e3259351c Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 19 Apr 2020 23:13:51 +0200 Subject: [PATCH 21/36] WIA: Write all headers at the start of the file Gets rid of the need to seek to the end of the file when opening a file. The downside of this is that we waste a little space, since we can't know in advance exactly how much space the compressed parts of the headers will need. --- Source/Core/DiscIO/WIABlob.cpp | 117 ++++++++++++++++++++++----------- Source/Core/DiscIO/WIABlob.h | 7 +- 2 files changed, 83 insertions(+), 41 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index f468ecaee319..79c979fa7d66 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -1481,29 +1481,39 @@ WIAFileReader::ConversionResult WIAFileReader::CompressAndWriteGroup( return ConversionResult::Success; } -WIAFileReader::ConversionResult -WIAFileReader::CompressAndWrite(File::IOFile* file, u64* bytes_written, Compressor* compressor, - const u8* data, size_t size, size_t* size_out) +std::optional> WIAFileReader::Compress(Compressor* compressor, const u8* data, + size_t size) { if (compressor) { if (!compressor->Start() || !compressor->Compress(data, size) || !compressor->End()) - return ConversionResult::InternalError; + return std::nullopt; data = compressor->GetData(); size = compressor->GetSize(); } - *size_out = size; - - *bytes_written += size; - if (!file->WriteArray(data, size)) - return ConversionResult::WriteFailed; + return std::vector(data, data + size); +} - if (!PadTo4(file, bytes_written)) - return ConversionResult::WriteFailed; +bool WIAFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound, + u64* bytes_written, u64* offset_out) +{ + // The first part of the check is to prevent this from running more than once. If *bytes_written + // is past the upper bound, we are already at the end of the file, so we don't need to do anything + if (*bytes_written <= upper_bound && *bytes_written + size > upper_bound) + { + WARN_LOG(DISCIO, "Headers did not fit in the allocated space. Writing to end of file instead"); + if (!file->Seek(0, SEEK_END)) + return false; + *bytes_written = file->Tell(); + } - return ConversionResult::Success; + *offset_out = *bytes_written; + if (!file->WriteArray(data, size)) + return false; + *bytes_written += size; + return PadTo4(file, bytes_written); } WIAFileReader::ConversionResult @@ -1522,14 +1532,8 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, u64 bytes_written = 0; size_t groups_written = 0; - // These two headers will be filled in with proper values later WIAHeader1 header_1{}; WIAHeader2 header_2{}; - if (!outfile->WriteArray(&header_1, 1) || !outfile->WriteArray(&header_2, 1)) - return ConversionResult::WriteFailed; - bytes_written += sizeof(WIAHeader1) + sizeof(WIAHeader2); - if (!PadTo4(outfile, &bytes_written)) - return ConversionResult::WriteFailed; std::unique_ptr compressor; switch (compression_type) @@ -1587,6 +1591,24 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, group_entries.resize(total_groups); + const size_t partition_entries_size = partition_entries.size() * sizeof(PartitionEntry); + const size_t raw_data_entries_size = raw_data_entries.size() * sizeof(RawDataEntry); + const size_t group_entries_size = group_entries.size() * sizeof(GroupEntry); + + // Conservative estimate for how much space will be taken up by headers. + // The compression methods None and Purge have very predictable overhead, + // and the other methods are able to compress group entries well + const u64 headers_size_upper_bound = + Common::AlignUp(sizeof(WIAHeader1) + sizeof(WIAHeader2) + partition_entries_size + + raw_data_entries_size + group_entries_size + 0x100, + VolumeWii::BLOCK_TOTAL_SIZE); + + std::vector buffer; + + buffer.resize(headers_size_upper_bound); + outfile->WriteBytes(buffer.data(), buffer.size()); + bytes_written = headers_size_upper_bound; + if (!infile->Read(0, header_2.disc_header.size(), header_2.disc_header.data())) return ConversionResult::ReadFailed; // We intentially do not increment bytes_read here, since these bytes will be read again @@ -1598,7 +1620,6 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, using WiiBlockData = std::array; - std::vector buffer; std::vector exceptions_buffer; std::vector decryption_buffer; std::vector hash_buffer; @@ -1815,25 +1836,41 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, ASSERT(groups_written == total_groups); ASSERT(bytes_read == iso_size); - const u64 partition_entries_offset = bytes_written; - const u64 partition_entries_size = partition_entries.size() * sizeof(PartitionEntry); - if (!outfile->WriteArray(partition_entries.data(), partition_entries.size())) + const std::optional> compressed_raw_data_entries = Compress( + compressor.get(), reinterpret_cast(raw_data_entries.data()), raw_data_entries_size); + if (!compressed_raw_data_entries) + return ConversionResult::InternalError; + + const std::optional> compressed_group_entries = + Compress(compressor.get(), reinterpret_cast(group_entries.data()), group_entries_size); + if (!compressed_group_entries) + return ConversionResult::InternalError; + + bytes_written = sizeof(WIAHeader1) + sizeof(WIAHeader2); + if (!outfile->Seek(sizeof(WIAHeader1) + sizeof(WIAHeader2), SEEK_SET)) return ConversionResult::WriteFailed; - bytes_written += partition_entries_size; - if (!PadTo4(outfile, &bytes_written)) + + u64 partition_entries_offset; + if (!WriteHeader(outfile, reinterpret_cast(partition_entries.data()), partition_entries_size, + headers_size_upper_bound, &bytes_written, &partition_entries_offset)) + { return ConversionResult::WriteFailed; + } - const u64 raw_data_entries_offset = bytes_written; - size_t raw_data_entries_size = raw_data_entries.size() * sizeof(RawDataEntry); - const ConversionResult raw_data_result = CompressAndWrite( - outfile, &bytes_written, compressor.get(), reinterpret_cast(raw_data_entries.data()), - raw_data_entries_size, &raw_data_entries_size); + u64 raw_data_entries_offset; + if (!WriteHeader(outfile, compressed_raw_data_entries->data(), + compressed_raw_data_entries->size(), headers_size_upper_bound, &bytes_written, + &raw_data_entries_offset)) + { + return ConversionResult::WriteFailed; + } - const u64 group_entries_offset = bytes_written; - size_t group_entries_size = group_entries.size() * sizeof(GroupEntry); - const ConversionResult groups_result = CompressAndWrite( - outfile, &bytes_written, compressor.get(), reinterpret_cast(group_entries.data()), - group_entries_size, &group_entries_size); + u64 group_entries_offset; + if (!WriteHeader(outfile, compressed_group_entries->data(), compressed_group_entries->size(), + headers_size_upper_bound, &bytes_written, &group_entries_offset)) + { + return ConversionResult::WriteFailed; + } u32 disc_type = 0; if (infile_volume) @@ -1860,11 +1897,12 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, header_2.number_of_raw_data_entries = Common::swap32(static_cast(raw_data_entries.size())); header_2.raw_data_entries_offset = Common::swap64(raw_data_entries_offset); - header_2.raw_data_entries_size = Common::swap32(static_cast(raw_data_entries_size)); + header_2.raw_data_entries_size = + Common::swap32(static_cast(compressed_raw_data_entries->size())); header_2.number_of_group_entries = Common::swap32(static_cast(group_entries.size())); header_2.group_entries_offset = Common::swap64(group_entries_offset); - header_2.group_entries_size = Common::swap32(static_cast(group_entries_size)); + header_2.group_entries_size = Common::swap32(static_cast(compressed_group_entries->size())); header_1.magic = WIA_MAGIC; header_1.version = Common::swap32(WIA_VERSION); @@ -1873,13 +1911,16 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, mbedtls_sha1_ret(reinterpret_cast(&header_2), sizeof(header_2), header_1.header_2_hash.data()); header_1.iso_file_size = Common::swap64(infile->GetDataSize()); - header_1.wia_file_size = Common::swap64(bytes_written); + header_1.wia_file_size = Common::swap64(outfile->GetSize()); mbedtls_sha1_ret(reinterpret_cast(&header_1), offsetof(WIAHeader1, header_1_hash), header_1.header_1_hash.data()); if (!outfile->Seek(0, SEEK_SET)) return ConversionResult::WriteFailed; - if (!outfile->WriteArray(&header_1, 1) || !outfile->WriteArray(&header_2, 1)) + + if (!outfile->WriteArray(&header_1, 1)) + return ConversionResult::WriteFailed; + if (!outfile->WriteArray(&header_2, 1)) return ConversionResult::WriteFailed; return ConversionResult::Success; diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 75d2918919bc..5aad1e727628 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -438,9 +438,10 @@ class WIAFileReader : public BlobReader size_t* groups_written, Compressor* compressor, bool compressed_exception_lists, const std::vector& exception_lists, const std::vector& main_data, std::map* reusable_groups, std::optional reuse_id); - static ConversionResult CompressAndWrite(File::IOFile* file, u64* bytes_written, - Compressor* compressor, const u8* data, size_t size, - size_t* size_out); + static std::optional> Compress(Compressor* compressor, const u8* data, + size_t size); + static bool WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound, + u64* bytes_written, u64* offset_out); template static void PushBack(std::vector* vector, const T& x) From f21a254042d4f92b8ff81684a29e8393ac6231e0 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 22 Apr 2020 09:50:27 +0200 Subject: [PATCH 22/36] WIA: Implement multithreaded compression --- Source/Core/DiscIO/WIABlob.cpp | 748 ++++++++++++++++++--------------- Source/Core/DiscIO/WIABlob.h | 88 ++-- 2 files changed, 460 insertions(+), 376 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 79c979fa7d66..1924c7d08d26 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,7 @@ #include "DiscIO/Blob.h" #include "DiscIO/DiscExtractor.h" +#include "DiscIO/MultithreadedCompressor.h" #include "DiscIO/Volume.h" #include "DiscIO/VolumeWii.h" #include "DiscIO/WiiEncryptionCache.h" @@ -909,26 +911,35 @@ WIAFileReader::LZMACompressor::LZMACompressor(bool lzma2, int compression_level, if (!lzma2) { - *compressor_data_size_out = 5; + if (compressor_data_size_out) + *compressor_data_size_out = 5; - ASSERT(m_options.lc < 9); - ASSERT(m_options.lp < 5); - ASSERT(m_options.pb < 5); - compressor_data_out[0] = static_cast((m_options.pb * 5 + m_options.lp) * 9 + m_options.lc); - - // The dictionary size is stored as a 32-bit little endian unsigned integer - static_assert(sizeof(m_options.dict_size) == sizeof(u32)); - std::memcpy(compressor_data_out + 1, &m_options.dict_size, sizeof(u32)); + if (compressor_data_out) + { + ASSERT(m_options.lc < 9); + ASSERT(m_options.lp < 5); + ASSERT(m_options.pb < 5); + compressor_data_out[0] = + static_cast((m_options.pb * 5 + m_options.lp) * 9 + m_options.lc); + + // The dictionary size is stored as a 32-bit little endian unsigned integer + static_assert(sizeof(m_options.dict_size) == sizeof(u32)); + std::memcpy(compressor_data_out + 1, &m_options.dict_size, sizeof(u32)); + } } else { - *compressor_data_size_out = 1; + if (compressor_data_size_out) + *compressor_data_size_out = 1; - u8 encoded_dict_size = 0; - while (encoded_dict_size < 40 && m_options.dict_size > LZMA2DictionarySize(encoded_dict_size)) - ++encoded_dict_size; + if (compressor_data_out) + { + u8 encoded_dict_size = 0; + while (encoded_dict_size < 40 && m_options.dict_size > LZMA2DictionarySize(encoded_dict_size)) + ++encoded_dict_size; - compressor_data_out[0] = encoded_dict_size; + compressor_data_out[0] = encoded_dict_size; + } } m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1; @@ -1246,7 +1257,7 @@ WIAFileReader::PartitionDataEntry WIAFileReader::CreatePartitionDataEntry( Common::swap32(group_index), Common::swap32(groups)}; } -WIAFileReader::ConversionResult WIAFileReader::SetUpDataEntriesForWriting( +ConversionResultCode WIAFileReader::SetUpDataEntriesForWriting( const VolumeDisc* volume, int chunk_size, u64 iso_size, u32* total_groups, std::vector* partition_entries, std::vector* raw_data_entries, std::vector* data_entries) @@ -1299,7 +1310,7 @@ WIAFileReader::ConversionResult WIAFileReader::SetUpDataEntriesForWriting( volume->ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE); if (!data_offset || !data_size) - return ConversionResult::ReadFailed; + return ConversionResultCode::ReadFailed; const u64 data_start = partition.offset + *data_offset; const u64 data_end = data_start + *data_size; @@ -1326,11 +1337,11 @@ WIAFileReader::ConversionResult WIAFileReader::SetUpDataEntriesForWriting( const std::optional fst_size = GetFSTSize(*volume, partition); if (!fst_offset || !fst_size) - return ConversionResult::ReadFailed; + return ConversionResultCode::ReadFailed; const IOS::ES::TicketReader& ticket = volume->GetTicket(partition); if (!ticket.IsValid()) - return ConversionResult::ReadFailed; + return ConversionResultCode::ReadFailed; add_raw_data_entry(last_partition_end_offset, partition.offset - last_partition_end_offset); @@ -1359,141 +1370,342 @@ WIAFileReader::ConversionResult WIAFileReader::SetUpDataEntriesForWriting( add_raw_data_entry(last_partition_end_offset, iso_size - last_partition_end_offset); - return ConversionResult::Success; + return ConversionResultCode::Success; } -bool WIAFileReader::TryReuseGroup(std::vector* group_entries, size_t* groups_written, - std::map* reusable_groups, - std::optional reuse_id) +std::optional> WIAFileReader::Compress(Compressor* compressor, const u8* data, + size_t size) { - if (!reuse_id) - return false; - - const auto it = reusable_groups->find(*reuse_id); - if (it != reusable_groups->end()) + if (compressor) { - (*group_entries)[*groups_written] = it->second; - ++*groups_written; + if (!compressor->Start() || !compressor->Compress(data, size) || !compressor->End()) + return std::nullopt; + + data = compressor->GetData(); + size = compressor->GetSize(); } - return it != reusable_groups->end(); + return std::vector(data, data + size); } -WIAFileReader::ConversionResult WIAFileReader::CompressAndWriteGroup( - File::IOFile* file, u64* bytes_written, std::vector* group_entries, - size_t* groups_written, Compressor* compressor, bool compressed_exception_lists, - const std::vector& exception_lists, const std::vector& main_data, - std::map* reusable_groups, std::optional reuse_id) +void WIAFileReader::SetUpCompressor(std::unique_ptr* compressor, + WIACompressionType compression_type, int compression_level, + WIAHeader2* header_2) { - const auto all_zero = [](const std::vector& data) { - return std::all_of(data.begin(), data.end(), [](u8 x) { return x == 0; }); + switch (compression_type) + { + case WIACompressionType::None: + *compressor = nullptr; + break; + case WIACompressionType::Purge: + *compressor = std::make_unique(); + break; + case WIACompressionType::Bzip2: + *compressor = std::make_unique(compression_level); + break; + case WIACompressionType::LZMA: + case WIACompressionType::LZMA2: + { + u8* compressor_data = nullptr; + u8* compressor_data_size = nullptr; + + if (header_2) + { + compressor_data = header_2->compressor_data; + compressor_data_size = &header_2->compressor_data_size; + } + + const bool lzma2 = compression_type == WIACompressionType::LZMA2; + *compressor = std::make_unique(lzma2, compression_level, compressor_data, + compressor_data_size); + break; + } + } +} + +ConversionResult +WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, + const std::vector& partition_entries, + const std::vector& data_entries, + std::map* reusable_groups, + std::mutex* reusable_groups_mutex, u64 exception_lists_per_chunk, + bool compressed_exception_lists) +{ + const auto all_are = [](const std::vector& data, u8 x) { + return std::all_of(data.begin(), data.end(), [x](u8 y) { return x == y; }); + }; + + const auto all_zero = [&all_are](const std::vector& data) { return all_are(data, 0); }; + + const auto all_same = [&all_are](const std::vector& data) { + return all_are(data, data.front()); + }; + + const auto reuse_id_exists = [reusable_groups, + reusable_groups_mutex](const std::optional& reuse_id) { + if (!reuse_id) + return false; + + std::lock_guard guard(*reusable_groups_mutex); + const auto it = reusable_groups->find(*reuse_id); + return it != reusable_groups->end(); }; - if (all_zero(exception_lists) && all_zero(main_data)) + std::optional reuse_id; + + state->exceptions_buffer.clear(); + + if (!parameters.data_entry->is_partition) { - (*group_entries)[*groups_written] = GroupEntry{0, 0}; - ++*groups_written; - return ConversionResult::Success; + if (all_same(parameters.data)) + reuse_id = ReuseID{nullptr, parameters.data.size(), false, parameters.data.front()}; } + else + { + const PartitionEntry& partition_entry = partition_entries[parameters.data_entry->index]; - if (TryReuseGroup(group_entries, groups_written, reusable_groups, reuse_id)) - return ConversionResult::Success; + mbedtls_aes_context aes_context; + mbedtls_aes_setkey_dec(&aes_context, partition_entry.partition_key.data(), 128); - const u64 data_offset = *bytes_written; + const u64 groups = Common::AlignUp(parameters.data.size(), VolumeWii::GROUP_TOTAL_SIZE) / + VolumeWii::GROUP_TOTAL_SIZE; - if (compressor) + ASSERT(parameters.data.size() % VolumeWii::BLOCK_TOTAL_SIZE == 0); + const u64 blocks = parameters.data.size() / VolumeWii::BLOCK_TOTAL_SIZE; + const u64 bytes_to_write = blocks * VolumeWii::BLOCK_DATA_SIZE; + + const auto create_reuse_id = [&partition_entry, bytes_to_write](u8 value, bool decrypted) { + return ReuseID{&partition_entry.partition_key, bytes_to_write, decrypted, value}; + }; + + // Set this group as reusable if the encrypted data is all_same + if (all_same(parameters.data)) + reuse_id = create_reuse_id(parameters.data.front(), false); + + if (reuse_id_exists(reuse_id)) + return OutputParameters{{}, {}, reuse_id, parameters.bytes_read, parameters.group_index}; + + std::vector> exception_lists(exception_lists_per_chunk); + + for (u64 i = 0; i < groups; ++i) + { + const u64 offset_of_group = i * VolumeWii::GROUP_TOTAL_SIZE; + const u64 write_offset_of_group = i * VolumeWii::GROUP_DATA_SIZE; + + const u64 blocks_in_this_group = + std::min(VolumeWii::BLOCKS_PER_GROUP, blocks - i * VolumeWii::BLOCKS_PER_GROUP); + + for (u32 j = 0; j < VolumeWii::BLOCKS_PER_GROUP; ++j) + { + if (j < blocks_in_this_group) + { + const u64 offset_of_block = offset_of_group + j * VolumeWii::BLOCK_TOTAL_SIZE; + VolumeWii::DecryptBlockData(parameters.data.data() + offset_of_block, + state->decryption_buffer[j].data(), &aes_context); + } + else + { + state->decryption_buffer[j].fill(0); + } + } + + VolumeWii::HashGroup(state->decryption_buffer.data(), state->hash_buffer.data()); + + for (u64 j = 0; j < blocks_in_this_group; ++j) + { + const u64 offset_of_block = offset_of_group + j * VolumeWii::BLOCK_TOTAL_SIZE; + const u64 hash_offset_of_block = j * VolumeWii::BLOCK_HEADER_SIZE; + + VolumeWii::HashBlock hashes; + VolumeWii::DecryptBlockHashes(parameters.data.data() + offset_of_block, &hashes, + &aes_context); + + const auto compare_hash = [&](size_t offset_in_block) { + ASSERT(offset_in_block + sizeof(SHA1) <= VolumeWii::BLOCK_HEADER_SIZE); + + const u8* desired_hash = reinterpret_cast(&hashes) + offset_in_block; + const u8* computed_hash = reinterpret_cast(&state->hash_buffer[j]) + offset_in_block; + + if (!std::equal(desired_hash, desired_hash + sizeof(SHA1), computed_hash)) + { + const u64 hash_offset = hash_offset_of_block + offset_in_block; + ASSERT(hash_offset <= std::numeric_limits::max()); + + HashExceptionEntry& exception = exception_lists[i].emplace_back(); + exception.offset = static_cast(Common::swap16(hash_offset)); + std::memcpy(exception.hash.data(), desired_hash, sizeof(SHA1)); + } + }; + + const auto compare_hashes = [&compare_hash](size_t offset, size_t size) { + for (size_t l = 0; l < size; l += sizeof(SHA1)) + // The std::min is to ensure that we don't go beyond the end of HashBlock with + // padding_2, which is 32 bytes long (not divisible by sizeof(SHA1), which is 20). + compare_hash(offset + std::min(l, size - sizeof(SHA1))); + }; + + using HashBlock = VolumeWii::HashBlock; + compare_hashes(offsetof(HashBlock, h0), sizeof(HashBlock::h0)); + compare_hashes(offsetof(HashBlock, padding_0), sizeof(HashBlock::padding_0)); + compare_hashes(offsetof(HashBlock, h1), sizeof(HashBlock::h1)); + compare_hashes(offsetof(HashBlock, padding_1), sizeof(HashBlock::padding_1)); + compare_hashes(offsetof(HashBlock, h2), sizeof(HashBlock::h2)); + compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); + } + + for (u64 j = 0; j < blocks_in_this_group; ++j) + { + const u64 write_offset_of_block = write_offset_of_group + j * VolumeWii::BLOCK_DATA_SIZE; + std::memcpy(parameters.data.data() + write_offset_of_block, + state->decryption_buffer[j].data(), VolumeWii::BLOCK_DATA_SIZE); + } + } + + bool have_exceptions = false; + + for (const std::vector& exception_list : exception_lists) + { + const u16 exceptions = Common::swap16(static_cast(exception_list.size())); + PushBack(&state->exceptions_buffer, exceptions); + for (const HashExceptionEntry& exception : exception_list) + PushBack(&state->exceptions_buffer, exception); + if (!exception_list.empty()) + have_exceptions = true; + } + + parameters.data.resize(bytes_to_write); + + // Set this group as reusable if it lacks exceptions and the decrypted data is all_same + if (!reuse_id && !have_exceptions && all_same(parameters.data)) + reuse_id = create_reuse_id(parameters.data.front(), true); + } + + // Special case - a compressed size of zero is treated by WIA as meaning the data is all zeroes + if (all_zero(state->exceptions_buffer) && all_zero(parameters.data)) + return OutputParameters{{}, {}, reuse_id, parameters.bytes_read, parameters.group_index}; + + if (reuse_id_exists(reuse_id)) + return OutputParameters{{}, {}, reuse_id, parameters.bytes_read, parameters.group_index}; + + if (state->compressor) { - if (!compressor->Start()) - return ConversionResult::InternalError; + if (!state->compressor->Start()) + return ConversionResultCode::InternalError; } - if (!exception_lists.empty()) + if (!state->exceptions_buffer.empty()) { - if (compressed_exception_lists && compressor) + if (compressed_exception_lists && state->compressor) { - if (!compressor->Compress(exception_lists.data(), exception_lists.size())) - return ConversionResult::InternalError; + if (!state->compressor->Compress(state->exceptions_buffer.data(), + state->exceptions_buffer.size())) + { + return ConversionResultCode::InternalError; + } + + state->exceptions_buffer.clear(); } else { - *bytes_written += exception_lists.size(); - if (!file->WriteArray(exception_lists.data(), exception_lists.size())) - return ConversionResult::WriteFailed; - - const u64 offset_of_padding = *bytes_written; if (!compressed_exception_lists) { - if (!PadTo4(file, bytes_written)) - return ConversionResult::WriteFailed; + while (state->exceptions_buffer.size() % 4 != 0) + state->exceptions_buffer.push_back(0); } - const u64 padding_written = *bytes_written - offset_of_padding; - // Some extra stuff we have to do because Purge for some reason is supposed to hash - // the exception lists and the following padding but not actually compress them... - if (compressor) + if (state->compressor) { - if (!compressor->AddPrecedingDataOnlyForPurgeHashing(exception_lists.data(), - exception_lists.size())) + if (!state->compressor->AddPrecedingDataOnlyForPurgeHashing( + state->exceptions_buffer.data(), state->exceptions_buffer.size())) { - return ConversionResult::InternalError; - } - - constexpr u32 ZEROES = 0; - if (!compressor->AddPrecedingDataOnlyForPurgeHashing(reinterpret_cast(ZEROES), - padding_written)) - { - return ConversionResult::InternalError; + return ConversionResultCode::InternalError; } } } } - if (compressor) + if (state->compressor) { - if (!compressor->Compress(main_data.data(), main_data.size())) - return ConversionResult::InternalError; - if (!compressor->End()) - return ConversionResult::InternalError; + if (!state->compressor->Compress(parameters.data.data(), parameters.data.size())) + return ConversionResultCode::InternalError; + if (!state->compressor->End()) + return ConversionResultCode::InternalError; } - const u8* data = compressor ? compressor->GetData() : main_data.data(); - const size_t size = compressor ? compressor->GetSize() : main_data.size(); + if (state->compressor) + { + const u8* data = state->compressor->GetData(); + const size_t size = state->compressor->GetSize(); - *bytes_written += size; - if (!file->WriteArray(data, size)) - return ConversionResult::WriteFailed; + parameters.data.resize(size); + std::copy(data, data + size, parameters.data.data()); + } + + return OutputParameters{state->exceptions_buffer, std::move(parameters.data), reuse_id, + parameters.bytes_read, parameters.group_index}; +} + +ConversionResultCode WIAFileReader::Output(const OutputParameters& parameters, + File::IOFile* outfile, + std::map* reusable_groups, + std::mutex* reusable_groups_mutex, + GroupEntry* group_entry, u64* bytes_written) +{ + if (parameters.reuse_id) + { + std::lock_guard guard(*reusable_groups_mutex); + const auto it = reusable_groups->find(*parameters.reuse_id); + if (it != reusable_groups->end()) + { + *group_entry = it->second; + return ConversionResultCode::Success; + } + } + + const size_t data_size = parameters.exception_lists.size() + parameters.main_data.size(); if (*bytes_written >> 2 > std::numeric_limits::max()) - return ConversionResult::InternalError; + return ConversionResultCode::InternalError; - ASSERT((data_offset & 3) == 0); - GroupEntry& group_entry = (*group_entries)[*groups_written]; - group_entry.data_offset = Common::swap32(static_cast(data_offset >> 2)); - group_entry.data_size = Common::swap32(static_cast(*bytes_written - data_offset)); - ++*groups_written; + ASSERT((*bytes_written & 3) == 0); + group_entry->data_offset = Common::swap32(static_cast(*bytes_written >> 2)); + group_entry->data_size = Common::swap32(static_cast(data_size)); - if (reuse_id) - reusable_groups->emplace(*reuse_id, group_entry); + if (!outfile->WriteArray(parameters.exception_lists.data(), parameters.exception_lists.size())) + return ConversionResultCode::WriteFailed; + if (!outfile->WriteArray(parameters.main_data.data(), parameters.main_data.size())) + return ConversionResultCode::WriteFailed; + + *bytes_written += data_size; + + if (parameters.reuse_id) + { + std::lock_guard guard(*reusable_groups_mutex); + reusable_groups->emplace(*parameters.reuse_id, *group_entry); + } - if (!PadTo4(file, bytes_written)) - return ConversionResult::WriteFailed; + if (!PadTo4(outfile, bytes_written)) + return ConversionResultCode::WriteFailed; - return ConversionResult::Success; + return ConversionResultCode::Success; } -std::optional> WIAFileReader::Compress(Compressor* compressor, const u8* data, - size_t size) +ConversionResultCode WIAFileReader::RunCallback(size_t groups_written, u64 bytes_read, + u64 bytes_written, u32 total_groups, u64 iso_size, + CompressCB callback, void* arg) { - if (compressor) - { - if (!compressor->Start() || !compressor->Compress(data, size) || !compressor->End()) - return std::nullopt; + int ratio = 0; + if (bytes_read != 0) + ratio = static_cast(100 * bytes_written / bytes_read); - data = compressor->GetData(); - size = compressor->GetSize(); - } + const std::string text = + StringFromFormat(Common::GetStringT("%i of %i blocks. Compression ratio %i%%").c_str(), + groups_written, total_groups, ratio); - return std::vector(data, data + size); + const float completion = static_cast(bytes_read) / iso_size; + + return callback(text, completion, arg) ? ConversionResultCode::Success : + ConversionResultCode::Canceled; } bool WIAFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound, @@ -1516,7 +1728,7 @@ bool WIAFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, return PadTo4(file, bytes_written); } -WIAFileReader::ConversionResult +ConversionResultCode WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, File::IOFile* outfile, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback, void* arg) @@ -1530,63 +1742,22 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, u64 bytes_read = 0; u64 bytes_written = 0; - size_t groups_written = 0; + size_t groups_processed = 0; WIAHeader1 header_1{}; WIAHeader2 header_2{}; - std::unique_ptr compressor; - switch (compression_type) - { - case WIACompressionType::None: - compressor = nullptr; - break; - case WIACompressionType::Purge: - compressor = std::make_unique(); - break; - case WIACompressionType::Bzip2: - compressor = std::make_unique(compression_level); - break; - case WIACompressionType::LZMA: - compressor = std::make_unique( - false, compression_level, header_2.compressor_data, &header_2.compressor_data_size); - break; - case WIACompressionType::LZMA2: - compressor = std::make_unique(true, compression_level, header_2.compressor_data, - &header_2.compressor_data_size); - break; - } - std::vector partition_entries; std::vector raw_data_entries; std::vector group_entries; - const auto run_callback = [&] { - int ratio = 0; - if (bytes_read != 0) - ratio = static_cast(100 * bytes_written / bytes_read); - - const std::string temp = - StringFromFormat(Common::GetStringT("%i of %i blocks. Compression ratio %i%%").c_str(), - groups_written, group_entries.size(), ratio); - - float completion = 0.0f; - if (group_entries.size() != 0) - completion = static_cast(groups_written) / group_entries.size(); - - return callback(temp, completion, arg); - }; - - if (!run_callback()) - return ConversionResult::Canceled; - u32 total_groups; std::vector data_entries; - const ConversionResult set_up_data_entries_result = + const ConversionResultCode set_up_data_entries_result = SetUpDataEntriesForWriting(infile_volume, chunk_size, iso_size, &total_groups, &partition_entries, &raw_data_entries, &data_entries); - if (set_up_data_entries_result != ConversionResult::Success) + if (set_up_data_entries_result != ConversionResultCode::Success) return set_up_data_entries_result; group_entries.resize(total_groups); @@ -1610,251 +1781,128 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, bytes_written = headers_size_upper_bound; if (!infile->Read(0, header_2.disc_header.size(), header_2.disc_header.data())) - return ConversionResult::ReadFailed; + return ConversionResultCode::ReadFailed; // We intentially do not increment bytes_read here, since these bytes will be read again - const auto all_same = [](const std::vector& data) { - const u8 first_byte = data.front(); - return std::all_of(data.begin(), data.end(), [first_byte](u8 x) { return x == first_byte; }); + std::map reusable_groups; + std::mutex reusable_groups_mutex; + + const auto set_up_compress_thread_state = [&](CompressThreadState* state) { + SetUpCompressor(&state->compressor, compression_type, compression_level, nullptr); + return ConversionResultCode::Success; }; - using WiiBlockData = std::array; + const auto process_and_compress = [&](CompressThreadState* state, CompressParameters parameters) { + return ProcessAndCompress(state, std::move(parameters), partition_entries, data_entries, + &reusable_groups, &reusable_groups_mutex, exception_lists_per_chunk, + compressed_exception_lists); + }; - std::vector exceptions_buffer; - std::vector decryption_buffer; - std::vector hash_buffer; + const auto output = [&](OutputParameters parameters) { + const ConversionResultCode result = + Output(parameters, outfile, &reusable_groups, &reusable_groups_mutex, + &group_entries[parameters.group_index], &bytes_written); - std::map reusable_groups; + if (result != ConversionResultCode::Success) + return result; - if (!partition_entries.empty()) - { - decryption_buffer.resize(VolumeWii::BLOCKS_PER_GROUP); - hash_buffer.resize(VolumeWii::BLOCKS_PER_GROUP); - } + return RunCallback(parameters.group_index + 1, parameters.bytes_read, bytes_written, + total_groups, iso_size, callback, arg); + }; + + MultithreadedCompressor mt_compressor( + set_up_compress_thread_state, process_and_compress, output); for (const DataEntry& data_entry : data_entries) { + u32 first_group; + u32 last_group; + + u64 data_offset; + u64 data_size; + if (data_entry.is_partition) { const PartitionEntry& partition_entry = partition_entries[data_entry.index]; const PartitionDataEntry& partition_data_entry = partition_entry.data_entries[data_entry.partition_data_index]; - const u32 first_group = Common::swap32(partition_data_entry.group_index); - const u32 last_group = first_group + Common::swap32(partition_data_entry.number_of_groups); + first_group = Common::swap32(partition_data_entry.group_index); + last_group = first_group + Common::swap32(partition_data_entry.number_of_groups); - const u64 data_offset = - Common::swap32(partition_data_entry.first_sector) * VolumeWii::BLOCK_TOTAL_SIZE; - const u64 data_size = + data_offset = Common::swap32(partition_data_entry.first_sector) * VolumeWii::BLOCK_TOTAL_SIZE; + data_size = Common::swap32(partition_data_entry.number_of_sectors) * VolumeWii::BLOCK_TOTAL_SIZE; - - ASSERT(groups_written == first_group); - ASSERT(bytes_read == data_offset); - - mbedtls_aes_context aes_context; - mbedtls_aes_setkey_dec(&aes_context, partition_entry.partition_key.data(), 128); - - for (u32 i = first_group; i < last_group; ++i) - { - const u64 bytes_to_read = std::min(chunk_size, data_offset + data_size - bytes_read); - - const u64 groups = Common::AlignUp(bytes_to_read, VolumeWii::GROUP_TOTAL_SIZE) / - VolumeWii::GROUP_TOTAL_SIZE; - - ASSERT(bytes_to_read % VolumeWii::BLOCK_TOTAL_SIZE == 0); - const u64 blocks = bytes_to_read / VolumeWii::BLOCK_TOTAL_SIZE; - const u64 bytes_to_write = blocks * VolumeWii::BLOCK_DATA_SIZE; - - buffer.resize(bytes_to_read); - if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) - return ConversionResult::ReadFailed; - bytes_read += bytes_to_read; - - const auto create_reuse_id = [&partition_entry, bytes_to_write](u8 value, bool decrypted) { - return ReuseID{&partition_entry.partition_key, bytes_to_write, decrypted, value}; - }; - - std::optional reuse_id; - - // Set this group as reusable if the encrypted data is all_same - if (all_same(buffer)) - reuse_id = create_reuse_id(buffer.front(), false); - - if (!TryReuseGroup(&group_entries, &groups_written, &reusable_groups, reuse_id)) - { - std::vector> exception_lists(exception_lists_per_chunk); - - for (u64 j = 0; j < groups; ++j) - { - const u64 offset_of_group = j * VolumeWii::GROUP_TOTAL_SIZE; - const u64 write_offset_of_group = j * VolumeWii::GROUP_DATA_SIZE; - - const u64 blocks_in_this_group = std::min( - VolumeWii::BLOCKS_PER_GROUP, blocks - j * VolumeWii::BLOCKS_PER_GROUP); - - for (u32 k = 0; k < VolumeWii::BLOCKS_PER_GROUP; ++k) - { - if (k < blocks_in_this_group) - { - const u64 offset_of_block = offset_of_group + k * VolumeWii::BLOCK_TOTAL_SIZE; - VolumeWii::DecryptBlockData(buffer.data() + offset_of_block, - decryption_buffer[k].data(), &aes_context); - } - else - { - decryption_buffer[k].fill(0); - } - } - - VolumeWii::HashGroup(decryption_buffer.data(), hash_buffer.data()); - - for (u64 k = 0; k < blocks_in_this_group; ++k) - { - const u64 offset_of_block = offset_of_group + k * VolumeWii::BLOCK_TOTAL_SIZE; - const u64 hash_offset_of_block = k * VolumeWii::BLOCK_HEADER_SIZE; - - VolumeWii::HashBlock hashes; - VolumeWii::DecryptBlockHashes(buffer.data() + offset_of_block, &hashes, &aes_context); - - const auto compare_hash = [&](size_t offset_in_block) { - ASSERT(offset_in_block + sizeof(SHA1) <= VolumeWii::BLOCK_HEADER_SIZE); - - const u8* desired_hash = reinterpret_cast(&hashes) + offset_in_block; - const u8* computed_hash = reinterpret_cast(&hash_buffer[k]) + offset_in_block; - - if (!std::equal(desired_hash, desired_hash + sizeof(SHA1), computed_hash)) - { - const u64 hash_offset = hash_offset_of_block + offset_in_block; - ASSERT(hash_offset <= std::numeric_limits::max()); - - HashExceptionEntry& exception = exception_lists[j].emplace_back(); - exception.offset = static_cast(Common::swap16(hash_offset)); - std::memcpy(exception.hash.data(), desired_hash, sizeof(SHA1)); - } - }; - - const auto compare_hashes = [&compare_hash](size_t offset, size_t size) { - for (size_t l = 0; l < size; l += sizeof(SHA1)) - // The std::min is to ensure that we don't go beyond the end of HashBlock with - // padding_2, which is 32 bytes long (not divisible by sizeof(SHA1), which is 20). - compare_hash(offset + std::min(l, size - sizeof(SHA1))); - }; - - using HashBlock = VolumeWii::HashBlock; - compare_hashes(offsetof(HashBlock, h0), sizeof(HashBlock::h0)); - compare_hashes(offsetof(HashBlock, padding_0), sizeof(HashBlock::padding_0)); - compare_hashes(offsetof(HashBlock, h1), sizeof(HashBlock::h1)); - compare_hashes(offsetof(HashBlock, padding_1), sizeof(HashBlock::padding_1)); - compare_hashes(offsetof(HashBlock, h2), sizeof(HashBlock::h2)); - compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); - } - - for (u64 k = 0; k < blocks_in_this_group; ++k) - { - std::memcpy(buffer.data() + write_offset_of_group + k * VolumeWii::BLOCK_DATA_SIZE, - decryption_buffer[k].data(), VolumeWii::BLOCK_DATA_SIZE); - } - } - - bool have_exceptions = false; - - exceptions_buffer.clear(); - for (const std::vector& exception_list : exception_lists) - { - const u16 exceptions = Common::swap16(static_cast(exception_list.size())); - PushBack(&exceptions_buffer, exceptions); - for (const HashExceptionEntry& exception : exception_list) - PushBack(&exceptions_buffer, exception); - if (!exception_list.empty()) - have_exceptions = true; - } - - buffer.resize(bytes_to_write); - - // Set this group as reusable if it lacks exceptions and the decrypted data is all_same - if (!reuse_id && !have_exceptions && all_same(buffer)) - reuse_id = create_reuse_id(buffer.front(), true); - - const ConversionResult write_result = CompressAndWriteGroup( - outfile, &bytes_written, &group_entries, &groups_written, compressor.get(), - compressed_exception_lists, exceptions_buffer, buffer, &reusable_groups, reuse_id); - - if (write_result != ConversionResult::Success) - return write_result; - } - - if (!run_callback()) - return ConversionResult::Canceled; - } } else { const RawDataEntry& raw_data_entry = raw_data_entries[data_entry.index]; - const u32 first_group = Common::swap32(raw_data_entry.group_index); - const u32 last_group = first_group + Common::swap32(raw_data_entry.number_of_groups); + first_group = Common::swap32(raw_data_entry.group_index); + last_group = first_group + Common::swap32(raw_data_entry.number_of_groups); - u64 data_offset = Common::swap64(raw_data_entry.data_offset); - u64 data_size = Common::swap64(raw_data_entry.data_size); + data_offset = Common::swap64(raw_data_entry.data_offset); + data_size = Common::swap64(raw_data_entry.data_size); const u64 skipped_data = data_offset % VolumeWii::BLOCK_TOTAL_SIZE; data_offset -= skipped_data; data_size += skipped_data; + } - ASSERT(groups_written == first_group); - ASSERT(bytes_read == data_offset); - - exceptions_buffer.clear(); - - for (u32 i = first_group; i < last_group; ++i) - { - const u64 bytes_to_read = std::min(chunk_size, data_offset + data_size - bytes_read); - - buffer.resize(bytes_to_read); + ASSERT(groups_processed == first_group); + ASSERT(bytes_read == data_offset); - if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) - return ConversionResult::ReadFailed; - bytes_read += bytes_to_read; + for (u32 i = first_group; i < last_group; ++i) + { + const ConversionResultCode status = mt_compressor.GetStatus(); + if (status != ConversionResultCode::Success) + return status; - std::optional reuse_id; - if (all_same(buffer)) - reuse_id = ReuseID{nullptr, bytes_to_read, false, buffer.front()}; + const u64 bytes_to_read = std::min(chunk_size, data_offset + data_size - bytes_read); - const ConversionResult write_result = CompressAndWriteGroup( - outfile, &bytes_written, &group_entries, &groups_written, compressor.get(), - compressed_exception_lists, exceptions_buffer, buffer, &reusable_groups, reuse_id); + buffer.resize(bytes_to_read); + if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) + return ConversionResultCode::ReadFailed; + bytes_read += bytes_to_read; - if (write_result != ConversionResult::Success) - return write_result; + mt_compressor.CompressAndWrite( + CompressParameters{buffer, &data_entry, bytes_read, groups_processed}); - if (!run_callback()) - return ConversionResult::Canceled; - } + ++groups_processed; } } - ASSERT(groups_written == total_groups); + ASSERT(groups_processed == total_groups); ASSERT(bytes_read == iso_size); + mt_compressor.Shutdown(); + + const ConversionResultCode status = mt_compressor.GetStatus(); + if (status != ConversionResultCode::Success) + return status; + + std::unique_ptr compressor; + SetUpCompressor(&compressor, compression_type, compression_level, &header_2); + const std::optional> compressed_raw_data_entries = Compress( compressor.get(), reinterpret_cast(raw_data_entries.data()), raw_data_entries_size); if (!compressed_raw_data_entries) - return ConversionResult::InternalError; + return ConversionResultCode::InternalError; const std::optional> compressed_group_entries = Compress(compressor.get(), reinterpret_cast(group_entries.data()), group_entries_size); if (!compressed_group_entries) - return ConversionResult::InternalError; + return ConversionResultCode::InternalError; bytes_written = sizeof(WIAHeader1) + sizeof(WIAHeader2); if (!outfile->Seek(sizeof(WIAHeader1) + sizeof(WIAHeader2), SEEK_SET)) - return ConversionResult::WriteFailed; + return ConversionResultCode::WriteFailed; u64 partition_entries_offset; if (!WriteHeader(outfile, reinterpret_cast(partition_entries.data()), partition_entries_size, headers_size_upper_bound, &bytes_written, &partition_entries_offset)) { - return ConversionResult::WriteFailed; + return ConversionResultCode::WriteFailed; } u64 raw_data_entries_offset; @@ -1862,14 +1910,14 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, compressed_raw_data_entries->size(), headers_size_upper_bound, &bytes_written, &raw_data_entries_offset)) { - return ConversionResult::WriteFailed; + return ConversionResultCode::WriteFailed; } u64 group_entries_offset; if (!WriteHeader(outfile, compressed_group_entries->data(), compressed_group_entries->size(), headers_size_upper_bound, &bytes_written, &group_entries_offset)) { - return ConversionResult::WriteFailed; + return ConversionResultCode::WriteFailed; } u32 disc_type = 0; @@ -1916,14 +1964,14 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, header_1.header_1_hash.data()); if (!outfile->Seek(0, SEEK_SET)) - return ConversionResult::WriteFailed; + return ConversionResultCode::WriteFailed; if (!outfile->WriteArray(&header_1, 1)) - return ConversionResult::WriteFailed; + return ConversionResultCode::WriteFailed; if (!outfile->WriteArray(&header_2, 1)) - return ConversionResult::WriteFailed; + return ConversionResultCode::WriteFailed; - return ConversionResult::Success; + return ConversionResultCode::Success; } bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, @@ -1942,28 +1990,28 @@ bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, std::unique_ptr infile_volume = CreateDisc(infile_path); - const WIAFileReader::ConversionResult result = + const ConversionResultCode result = WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, compression_type, compression_level, chunk_size, callback, arg); - if (result == WIAFileReader::ConversionResult::ReadFailed) + if (result == ConversionResultCode::ReadFailed) PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); - if (result == WIAFileReader::ConversionResult::WriteFailed) + if (result == ConversionResultCode::WriteFailed) { PanicAlertT("Failed to write the output file \"%s\".\n" "Check that you have enough space available on the target drive.", outfile_path.c_str()); } - if (result != WIAFileReader::ConversionResult::Success) + if (result != ConversionResultCode::Success) { // Remove the incomplete output file outfile.Close(); File::Delete(outfile_path); } - return result == WIAFileReader::ConversionResult::Success; + return result == ConversionResultCode::Success; } } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 5aad1e727628..653f7312f372 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ #include "Common/File.h" #include "Common/Swap.h" #include "DiscIO/Blob.h" +#include "DiscIO/MultithreadedCompressor.h" #include "DiscIO/WiiEncryptionCache.h" namespace DiscIO @@ -56,19 +58,11 @@ class WIAFileReader : public BlobReader bool SupportsReadWiiDecrypted() const override; bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override; - enum class ConversionResult - { - Success, - Canceled, - ReadFailed, - WriteFailed, - InternalError, - }; - - static ConversionResult ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, WIACompressionType compression_type, - int compression_level, int chunk_size, CompressCB callback, - void* arg); + static ConversionResultCode ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, + File::IOFile* outfile, + WIACompressionType compression_type, + int compression_level, int chunk_size, + CompressCB callback, void* arg); private: using SHA1 = std::array; @@ -417,6 +411,38 @@ class WIAFileReader : public BlobReader u8 value; }; + struct CompressThreadState + { + using WiiBlockData = std::array; + + std::unique_ptr compressor; + + std::vector decryption_buffer = + std::vector(VolumeWii::BLOCKS_PER_GROUP); + + std::vector hash_buffer = + std::vector(VolumeWii::BLOCKS_PER_GROUP); + + std::vector exceptions_buffer; + }; + + struct CompressParameters + { + std::vector data; + const DataEntry* data_entry; + u64 bytes_read; + size_t group_index; + }; + + struct OutputParameters + { + std::vector exception_lists; + std::vector main_data; + std::optional reuse_id; + u64 bytes_read; + size_t group_index; + }; + static bool PadTo4(File::IOFile* file, u64* bytes_written); static void AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups, std::vector* raw_data_entries, @@ -425,24 +451,34 @@ class WIAFileReader : public BlobReader CreatePartitionDataEntry(u64 offset, u64 size, u32 index, int chunk_size, u32* total_groups, const std::vector& partition_entries, std::vector* data_entries); - static ConversionResult SetUpDataEntriesForWriting(const VolumeDisc* volume, int chunk_size, - u64 iso_size, u32* total_groups, - std::vector* partition_entries, - std::vector* raw_data_entries, - std::vector* data_entries); - static bool TryReuseGroup(std::vector* group_entries, size_t* groups_written, - std::map* reusable_groups, - std::optional reuse_id); - static ConversionResult CompressAndWriteGroup( - File::IOFile* file, u64* bytes_written, std::vector* group_entries, - size_t* groups_written, Compressor* compressor, bool compressed_exception_lists, - const std::vector& exception_lists, const std::vector& main_data, - std::map* reusable_groups, std::optional reuse_id); + static ConversionResultCode + SetUpDataEntriesForWriting(const VolumeDisc* volume, int chunk_size, u64 iso_size, + u32* total_groups, std::vector* partition_entries, + std::vector* raw_data_entries, + std::vector* data_entries); static std::optional> Compress(Compressor* compressor, const u8* data, size_t size); static bool WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound, u64* bytes_written, u64* offset_out); + static void SetUpCompressor(std::unique_ptr* compressor, + WIACompressionType compression_type, int compression_level, + WIAHeader2* header_2); + static ConversionResult + ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, + const std::vector& partition_entries, + const std::vector& data_entries, + std::map* reusable_groups, + std::mutex* reusable_groups_mutex, u64 exception_lists_per_chunk, + bool compressed_exception_lists); + static ConversionResultCode Output(const OutputParameters& parameters, File::IOFile* outfile, + std::map* reusable_groups, + std::mutex* reusable_groups_mutex, GroupEntry* group_entry, + u64* bytes_written); + static ConversionResultCode RunCallback(size_t groups_written, u64 bytes_read, u64 bytes_written, + u32 total_groups, u64 iso_size, CompressCB callback, + void* arg); + template static void PushBack(std::vector* vector, const T& x) { From e2ae2b3b0b11cc869468bed8a850e2012a68c2d9 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Mon, 4 May 2020 12:41:14 +0200 Subject: [PATCH 23/36] Add new file format RVZ based on WIA --- .../dolphinemu/utils/FileBrowserHelper.java | 2 +- Source/Core/Core/Boot/Boot.cpp | 2 +- Source/Core/DiscIO/Blob.cpp | 1 + Source/Core/DiscIO/Blob.h | 5 +- Source/Core/DiscIO/WIABlob.cpp | 35 +++++++++---- Source/Core/DiscIO/WIABlob.h | 16 ++++-- Source/Core/DolphinQt/ConvertDialog.cpp | 51 ++++++++++++------- .../Core/DolphinQt/GameList/GameTracker.cpp | 5 +- Source/Core/DolphinQt/Info.plist.in | 1 + Source/Core/DolphinQt/MainWindow.cpp | 4 +- Source/Core/DolphinQt/Settings/PathPane.cpp | 8 +-- Source/Core/UICommon/GameFileCache.cpp | 2 +- 12 files changed, 84 insertions(+), 48 deletions(-) diff --git a/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java b/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java index a27b2f2e0563..4c6803d1afab 100644 --- a/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java +++ b/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java @@ -22,7 +22,7 @@ public final class FileBrowserHelper { public static final HashSet GAME_EXTENSIONS = new HashSet<>(Arrays.asList( - "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "wad", "dol", "elf", "dff")); + "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "rvz", "wad", "dol", "elf", "dff")); public static final HashSet RAW_EXTENSION = new HashSet<>(Collections.singletonList( "raw")); diff --git a/Source/Core/Core/Boot/Boot.cpp b/Source/Core/Core/Boot/Boot.cpp index f06c1077f92f..a1ab46ed3c6c 100644 --- a/Source/Core/Core/Boot/Boot.cpp +++ b/Source/Core/Core/Boot/Boot.cpp @@ -159,7 +159,7 @@ BootParameters::GenerateFromFile(std::vector paths, paths.clear(); static const std::unordered_set disc_image_extensions = { - {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".dol", ".elf"}}; + {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".rvz", ".dol", ".elf"}}; if (disc_image_extensions.find(extension) != disc_image_extensions.end() || is_drive) { std::unique_ptr disc = DiscIO::CreateDisc(path); diff --git a/Source/Core/DiscIO/Blob.cpp b/Source/Core/DiscIO/Blob.cpp index 97e48eb047c5..c2c44caa60c9 100644 --- a/Source/Core/DiscIO/Blob.cpp +++ b/Source/Core/DiscIO/Blob.cpp @@ -207,6 +207,7 @@ std::unique_ptr CreateBlobReader(const std::string& filename) case WBFS_MAGIC: return WbfsFileReader::Create(std::move(file), filename); case WIA_MAGIC: + case RVZ_MAGIC: return WIAFileReader::Create(std::move(file), filename); default: if (auto directory_blob = DirectoryBlobReader::Create(filename)) diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 49e42e67415f..149df2afc339 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -37,7 +37,8 @@ enum class BlobType CISO, WBFS, TGC, - WIA + WIA, + RVZ, }; class BlobReader @@ -176,7 +177,7 @@ bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, const std::string& outfile_path, CompressCB callback = nullptr, void* arg = nullptr); bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, - const std::string& outfile_path, WIACompressionType compression_type, + const std::string& outfile_path, bool rvz, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback = nullptr, void* arg = nullptr); diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 1924c7d08d26..7414c4822194 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -51,14 +51,21 @@ bool WIAFileReader::Initialize(const std::string& path) if (!m_file.Seek(0, SEEK_SET) || !m_file.ReadArray(&m_header_1, 1)) return false; - if (m_header_1.magic != WIA_MAGIC) + if (m_header_1.magic != WIA_MAGIC && m_header_1.magic != RVZ_MAGIC) return false; - const u32 version = Common::swap32(m_header_1.version); - const u32 version_compatible = Common::swap32(m_header_1.version_compatible); - if (WIA_VERSION < version_compatible || WIA_VERSION_READ_COMPATIBLE > version) + m_rvz = m_header_1.magic == RVZ_MAGIC; + + const u32 version = m_rvz ? RVZ_VERSION : WIA_VERSION; + const u32 version_read_compatible = + m_rvz ? RVZ_VERSION_READ_COMPATIBLE : WIA_VERSION_READ_COMPATIBLE; + + const u32 file_version = Common::swap32(m_header_1.version); + const u32 file_version_compatible = Common::swap32(m_header_1.version_compatible); + + if (version < file_version_compatible || version_read_compatible > file_version) { - ERROR_LOG(DISCIO, "Unsupported WIA version %s in %s", VersionToString(version).c_str(), + ERROR_LOG(DISCIO, "Unsupported version %s in %s", VersionToString(file_version).c_str(), path.c_str()); return false; } @@ -229,6 +236,11 @@ std::unique_ptr WIAFileReader::Create(File::IOFile file, const st return blob->m_valid ? std::move(blob) : nullptr; } +BlobType WIAFileReader::GetBlobType() const +{ + return m_rvz ? BlobType::RVZ : BlobType::WIA; +} + bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) { if (offset + size > Common::swap64(m_header_1.iso_file_size)) @@ -1730,7 +1742,7 @@ bool WIAFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, ConversionResultCode WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, WIACompressionType compression_type, + File::IOFile* outfile, bool rvz, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback, void* arg) { ASSERT(infile->IsDataSizeAccurate()); @@ -1952,9 +1964,10 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, header_2.group_entries_offset = Common::swap64(group_entries_offset); header_2.group_entries_size = Common::swap32(static_cast(compressed_group_entries->size())); - header_1.magic = WIA_MAGIC; - header_1.version = Common::swap32(WIA_VERSION); - header_1.version_compatible = Common::swap32(WIA_VERSION_WRITE_COMPATIBLE); + header_1.magic = rvz ? RVZ_MAGIC : WIA_MAGIC; + header_1.version = Common::swap32(rvz ? RVZ_VERSION : WIA_VERSION); + header_1.version_compatible = + Common::swap32(rvz ? RVZ_VERSION_WRITE_COMPATIBLE : WIA_VERSION_WRITE_COMPATIBLE); header_1.header_2_size = Common::swap32(sizeof(WIAHeader2)); mbedtls_sha1_ret(reinterpret_cast(&header_2), sizeof(header_2), header_1.header_2_hash.data()); @@ -1975,7 +1988,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, } bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, - const std::string& outfile_path, WIACompressionType compression_type, + const std::string& outfile_path, bool rvz, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback, void* arg) { File::IOFile outfile(outfile_path, "wb"); @@ -1991,7 +2004,7 @@ bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, std::unique_ptr infile_volume = CreateDisc(infile_path); const ConversionResultCode result = - WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, compression_type, + WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, rvz, compression_type, compression_level, chunk_size, callback, arg); if (result == ConversionResultCode::ReadFailed) diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 653f7312f372..356d8a3edba7 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -37,6 +37,7 @@ enum class WIACompressionType : u32 }; constexpr u32 WIA_MAGIC = 0x01414957; // "WIA\x1" (byteswapped to little endian) +constexpr u32 RVZ_MAGIC = 0x015A5652; // "RVZ\x1" (byteswapped to little endian) class WIAFileReader : public BlobReader { @@ -45,7 +46,7 @@ class WIAFileReader : public BlobReader static std::unique_ptr Create(File::IOFile file, const std::string& path); - BlobType GetBlobType() const override { return BlobType::WIA; } + BlobType GetBlobType() const override; u64 GetRawSize() const override { return Common::swap64(m_header_1.wia_file_size); } u64 GetDataSize() const override { return Common::swap64(m_header_1.iso_file_size); } @@ -59,7 +60,7 @@ class WIAFileReader : public BlobReader bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override; static ConversionResultCode ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, + File::IOFile* outfile, bool rvz, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback, void* arg); @@ -489,6 +490,7 @@ class WIAFileReader : public BlobReader } bool m_valid; + bool m_rvz; WIACompressionType m_compression_type; File::IOFile m_file; @@ -504,13 +506,17 @@ class WIAFileReader : public BlobReader std::map m_data_entries; + // Perhaps we could set WIA_VERSION_WRITE_COMPATIBLE to 0.9, but WIA version 0.9 was never in + // any official release of wit, and interim versions (either source or binaries) are hard to find. + // Since we've been unable to check if we're write compatible with 0.9, we set it 1.0 to be safe. + static constexpr u32 WIA_VERSION = 0x01000000; static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000; static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000; - // Perhaps we could set WIA_VERSION_WRITE_COMPATIBLE to 0.9, but WIA version 0.9 was never in - // any official release of wit, and interim versions (either source or binaries) are hard to find. - // Since we've been unable to check if we're write compatible with 0.9, we set it 1.0 to be safe. + static constexpr u32 RVZ_VERSION = 0x00010000; + static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00010000; + static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00010000; }; } // namespace DiscIO diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index e52868039bd9..120b557c6b03 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -59,6 +59,7 @@ ConvertDialog::ConvertDialog(QList> fi m_format->addItem(QStringLiteral("ISO"), static_cast(DiscIO::BlobType::PLAIN)); m_format->addItem(QStringLiteral("GCZ"), static_cast(DiscIO::BlobType::GCZ)); m_format->addItem(QStringLiteral("WIA"), static_cast(DiscIO::BlobType::WIA)); + m_format->addItem(QStringLiteral("RVZ"), static_cast(DiscIO::BlobType::RVZ)); if (std::all_of(m_files.begin(), m_files.end(), [](const auto& file) { return file->GetBlobType() == DiscIO::BlobType::PLAIN; })) { @@ -93,15 +94,17 @@ ConvertDialog::ConvertDialog(QList> fi QGroupBox* options_group = new QGroupBox(tr("Options")); options_group->setLayout(options_layout); - QLabel* info_text = - new QLabel(tr("ISO: A simple and robust format which is supported by many programs. " - "It takes up more space than any other format.\n\n" - "GCZ: A basic compressed format which is compatible with most versions of " - "Dolphin and some other programs. It can't efficiently compress junk data " - "(unless removed) or encrypted Wii data.\n\n" - "WIA: An advanced compressed format which is compatible with recent versions " - "of Dolphin and a few other programs. It can efficiently compress encrypted " - "Wii data, but not junk data (unless removed).")); + QLabel* info_text = new QLabel( + tr("ISO: A simple and robust format which is supported by many programs. It takes up more " + "space than any other format.\n\n" + "GCZ: A basic compressed format which is compatible with most versions of Dolphin and " + "some other programs. It can't efficiently compress junk data (unless removed) or " + "encrypted Wii data.\n\n" + "WIA: An advanced compressed format which is compatible with recent versions of Dolphin " + "and a few other programs. It can efficiently compress encrypted Wii data, but not junk " + "data (unless removed).\n\n" + "RVZ: An advanced compressed format which is compatible with recent versions of Dolphin. " + "It can efficiently compress both junk data and encrypted Wii data.")); info_text->setWordWrap(true); QVBoxLayout* info_layout = new QVBoxLayout; @@ -196,6 +199,7 @@ void ConvertDialog::OnFormatChanged() break; } case DiscIO::BlobType::WIA: + case DiscIO::BlobType::RVZ: m_block_size->setEnabled(true); // This is the smallest block size supported by WIA. For performance, larger sizes are avoided. @@ -214,6 +218,7 @@ void ConvertDialog::OnFormatChanged() AddToCompressionComboBox(QStringLiteral("Deflate"), DiscIO::WIACompressionType::None); break; case DiscIO::BlobType::WIA: + case DiscIO::BlobType::RVZ: { m_compression->setEnabled(true); @@ -319,6 +324,10 @@ void ConvertDialog::Convert() extension = QStringLiteral(".wia"); filter = tr("WIA GC/Wii images (*.wia)"); break; + case DiscIO::BlobType::RVZ: + extension = QStringLiteral(".rvz"); + filter = tr("RVZ GC/Wii images (*.rvz)"); + break; default: ASSERT(false); return; @@ -423,8 +432,9 @@ void ConvertDialog::Convert() { std::future good; - if (format == DiscIO::BlobType::PLAIN) + switch (format) { + case DiscIO::BlobType::PLAIN: good = std::async(std::launch::async, [&] { const bool good = DiscIO::ConvertToPlain(blob_reader.get(), original_path, dst_path.toStdString(), @@ -432,9 +442,9 @@ void ConvertDialog::Convert() progress_dialog.Reset(); return good; }); - } - else if (format == DiscIO::BlobType::GCZ) - { + break; + + case DiscIO::BlobType::GCZ: good = std::async(std::launch::async, [&] { const bool good = DiscIO::ConvertToGCZ(blob_reader.get(), original_path, dst_path.toStdString(), @@ -443,16 +453,19 @@ void ConvertDialog::Convert() progress_dialog.Reset(); return good; }); - } - else if (format == DiscIO::BlobType::WIA) - { + break; + + case DiscIO::BlobType::WIA: + case DiscIO::BlobType::RVZ: good = std::async(std::launch::async, [&] { - const bool good = DiscIO::ConvertToWIA( - blob_reader.get(), original_path, dst_path.toStdString(), compression, - compression_level, block_size, &CompressCB, &progress_dialog); + const bool good = + DiscIO::ConvertToWIA(blob_reader.get(), original_path, dst_path.toStdString(), + format == DiscIO::BlobType::RVZ, compression, compression_level, + block_size, &CompressCB, &progress_dialog); progress_dialog.Reset(); return good; }); + break; } progress_dialog.GetRaw()->exec(); diff --git a/Source/Core/DolphinQt/GameList/GameTracker.cpp b/Source/Core/DolphinQt/GameList/GameTracker.cpp index 4aa930140c93..1f2b7383dd34 100644 --- a/Source/Core/DolphinQt/GameList/GameTracker.cpp +++ b/Source/Core/DolphinQt/GameList/GameTracker.cpp @@ -24,8 +24,9 @@ static const QStringList game_filters{ QStringLiteral("*.[gG][cC][mM]"), QStringLiteral("*.[iI][sS][oO]"), QStringLiteral("*.[tT][gG][cC]"), QStringLiteral("*.[cC][iI][sS][oO]"), QStringLiteral("*.[gG][cC][zZ]"), QStringLiteral("*.[wW][bB][fF][sS]"), - QStringLiteral("*.[wW][iI][aA]"), QStringLiteral("*.[wW][aA][dD]"), - QStringLiteral("*.[eE][lL][fF]"), QStringLiteral("*.[dD][oO][lL]")}; + QStringLiteral("*.[wW][iI][aA]"), QStringLiteral("*.[rR][vV][zZ]"), + QStringLiteral("*.[wW][aA][dD]"), QStringLiteral("*.[eE][lL][fF]"), + QStringLiteral("*.[dD][oO][lL]")}; GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent) { diff --git a/Source/Core/DolphinQt/Info.plist.in b/Source/Core/DolphinQt/Info.plist.in index 979f5f734882..b5f3a3f44baa 100644 --- a/Source/Core/DolphinQt/Info.plist.in +++ b/Source/Core/DolphinQt/Info.plist.in @@ -14,6 +14,7 @@ gcz iso m3u + rvz tgc wad wia diff --git a/Source/Core/DolphinQt/MainWindow.cpp b/Source/Core/DolphinQt/MainWindow.cpp index 508b8224271d..1ecb875986d9 100644 --- a/Source/Core/DolphinQt/MainWindow.cpp +++ b/Source/Core/DolphinQt/MainWindow.cpp @@ -686,8 +686,8 @@ QStringList MainWindow::PromptFileNames() QStringList paths = QFileDialog::getOpenFileNames( this, tr("Select a File"), settings.value(QStringLiteral("mainwindow/lastdir"), QString{}).toString(), - tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.wad *.dff " - "*.m3u);;All Files (*)")); + tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.rvz *.wad " + "*.dff *.m3u);;All Files (*)")); if (!paths.isEmpty()) { diff --git a/Source/Core/DolphinQt/Settings/PathPane.cpp b/Source/Core/DolphinQt/Settings/PathPane.cpp index e77f4d09c33e..fd85d42a229c 100644 --- a/Source/Core/DolphinQt/Settings/PathPane.cpp +++ b/Source/Core/DolphinQt/Settings/PathPane.cpp @@ -42,10 +42,10 @@ void PathPane::Browse() void PathPane::BrowseDefaultGame() { - QString file = QDir::toNativeSeparators(QFileDialog::getOpenFileName( - this, tr("Select a Game"), Settings::Instance().GetDefaultGame(), - tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.wad *.m3u);;" - "All Files (*)"))); + QString file = QDir::toNativeSeparators( + QFileDialog::getOpenFileName(this, tr("Select a Game"), Settings::Instance().GetDefaultGame(), + tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs " + "*.ciso *.gcz *.wia *.rvz *.wad *.m3u);;All Files (*)"))); if (!file.isEmpty()) Settings::Instance().SetDefaultGame(file); diff --git a/Source/Core/UICommon/GameFileCache.cpp b/Source/Core/UICommon/GameFileCache.cpp index bb12eb3ccd50..1af33ae44bd3 100644 --- a/Source/Core/UICommon/GameFileCache.cpp +++ b/Source/Core/UICommon/GameFileCache.cpp @@ -33,7 +33,7 @@ std::vector FindAllGamePaths(const std::vector& direct bool recursive_scan) { static const std::vector search_extensions = { - ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wia", ".wad", ".dol", ".elf"}; + ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wia", ".rvz", ".wad", ".dol", ".elf"}; // TODO: We could process paths iteratively as they are found return Common::DoFileSearch(directories_to_scan, search_extensions, recursive_scan); From 1f7c0b636fd0fffbf63892a7e3c2ab0109c33185 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Mon, 4 May 2020 16:01:56 +0200 Subject: [PATCH 24/36] RVZ: Add Zstandard as a compression method --- Source/Core/DiscIO/CMakeLists.txt | 1 + Source/Core/DiscIO/DiscIO.vcxproj | 3 + Source/Core/DiscIO/WIABlob.cpp | 130 +++++++++++++++++++++++- Source/Core/DiscIO/WIABlob.h | 38 +++++++ Source/Core/DolphinQt/ConvertDialog.cpp | 24 +++-- 5 files changed, 182 insertions(+), 14 deletions(-) diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index 46d66d217dc3..dcbab6cce478 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -54,6 +54,7 @@ target_link_libraries(discio PUBLIC BZip2::BZip2 LibLZMA::LibLZMA + zstd PRIVATE minizip diff --git a/Source/Core/DiscIO/DiscIO.vcxproj b/Source/Core/DiscIO/DiscIO.vcxproj index 7d94fcff278b..2a60bfacee42 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj +++ b/Source/Core/DiscIO/DiscIO.vcxproj @@ -118,6 +118,9 @@ {1d8c51d2-ffa4-418e-b183-9f42b6a6717e} + + {1bea10f3-80ce-4bc4-9331-5769372cdf99} + diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 7414c4822194..b3c9553d4920 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include "Common/Align.h" #include "Common/Assert.h" @@ -38,6 +39,24 @@ namespace DiscIO { +std::pair GetAllowedCompressionLevels(WIACompressionType compression_type) +{ + switch (compression_type) + { + case WIACompressionType::Bzip2: + case WIACompressionType::LZMA: + case WIACompressionType::LZMA2: + return {1, 9}; + case WIACompressionType::Zstd: + // The actual minimum level can be gotten by calling ZSTD_minCLevel(). However, returning that + // would make the UI rather weird, because it is a negative number with very large magnitude. + // Note: Level 0 is a special number which means "default level" (level 3 as of this writing). + return {1, ZSTD_maxCLevel()}; + default: + return {0, -1}; + } +} + WIAFileReader::WIAFileReader(File::IOFile file, const std::string& path) : m_file(std::move(file)), m_encryption_cache(this) { @@ -110,9 +129,9 @@ bool WIAFileReader::Initialize(const std::string& path) const u32 compression_type = Common::swap32(m_header_2.compression_type); m_compression_type = static_cast(compression_type); - if (m_compression_type > WIACompressionType::LZMA2) + if (m_compression_type > (m_rvz ? WIACompressionType::Zstd : WIACompressionType::LZMA2)) { - ERROR_LOG(DISCIO, "Unsupported WIA compression type %u in %s", compression_type, path.c_str()); + ERROR_LOG(DISCIO, "Unsupported compression type %u in %s", compression_type, path.c_str()); return false; } @@ -460,6 +479,9 @@ WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 decompressor = std::make_unique(true, m_header_2.compressor_data, m_header_2.compressor_data_size); break; + case WIACompressionType::Zstd: + decompressor = std::make_unique(); + break; } const bool compressed_exception_lists = m_compression_type > WIACompressionType::Purge; @@ -725,6 +747,34 @@ bool WIAFileReader::LZMADecompressor::Decompress(const DecompressionBuffer& in, return result == LZMA_OK || result == LZMA_STREAM_END; } +WIAFileReader::ZstdDecompressor::ZstdDecompressor() +{ + m_stream = ZSTD_createDStream(); +} + +WIAFileReader::ZstdDecompressor::~ZstdDecompressor() +{ + ZSTD_freeDStream(m_stream); +} + +bool WIAFileReader::ZstdDecompressor::Decompress(const DecompressionBuffer& in, + DecompressionBuffer* out, size_t* in_bytes_read) +{ + if (!m_stream) + return false; + + ZSTD_inBuffer in_buffer{in.data.data(), in.bytes_written, *in_bytes_read}; + ZSTD_outBuffer out_buffer{out->data.data(), out->data.size(), out->bytes_written}; + + const size_t result = ZSTD_decompressStream(m_stream, &out_buffer, &in_buffer); + + *in_bytes_read = in_buffer.pos; + out->bytes_written = out_buffer.pos; + + m_done = result == 0; + return !ZSTD_isError(result); +} + WIAFileReader::Compressor::~Compressor() = default; WIAFileReader::PurgeCompressor::PurgeCompressor() @@ -1032,6 +1082,71 @@ size_t WIAFileReader::LZMACompressor::GetSize() const return static_cast(m_stream.next_out - m_buffer.data()); } +WIAFileReader::ZstdCompressor::ZstdCompressor(int compression_level) +{ + m_stream = ZSTD_createCStream(); + + if (ZSTD_isError(ZSTD_CCtx_setParameter(m_stream, ZSTD_c_compressionLevel, compression_level))) + m_stream = nullptr; +} + +WIAFileReader::ZstdCompressor::~ZstdCompressor() +{ + ZSTD_freeCStream(m_stream); +} + +bool WIAFileReader::ZstdCompressor::Start() +{ + if (!m_stream) + return false; + + m_buffer.clear(); + m_out_buffer = {}; + + return !ZSTD_isError(ZSTD_CCtx_reset(m_stream, ZSTD_reset_session_only)); +} + +bool WIAFileReader::ZstdCompressor::Compress(const u8* data, size_t size) +{ + ZSTD_inBuffer in_buffer{data, size, 0}; + + ExpandBuffer(size); + + while (in_buffer.size != in_buffer.pos) + { + if (m_out_buffer.size == m_out_buffer.pos) + ExpandBuffer(0x100); + + if (ZSTD_isError(ZSTD_compressStream(m_stream, &m_out_buffer, &in_buffer))) + return false; + } + + return true; +} + +bool WIAFileReader::ZstdCompressor::End() +{ + while (true) + { + if (m_out_buffer.size == m_out_buffer.pos) + ExpandBuffer(0x100); + + const size_t result = ZSTD_endStream(m_stream, &m_out_buffer); + if (ZSTD_isError(result)) + return false; + if (result == 0) + return true; + } +} + +void WIAFileReader::ZstdCompressor::ExpandBuffer(size_t bytes_to_add) +{ + m_buffer.resize(m_buffer.size() + bytes_to_add); + + m_out_buffer.dst = m_buffer.data(); + m_out_buffer.size = m_buffer.size(); +} + WIAFileReader::Chunk::Chunk() = default; WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, @@ -1138,8 +1253,14 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) if (m_out.bytes_written > expected_out_bytes) return false; // Decompressed size is larger than expected - if (m_out.bytes_written == expected_out_bytes && !m_decompressor->Done()) + // The reason why we need the m_in.bytes_written == m_in.data.size() check as part of + // this conditional is because (for example) zstd can finish writing all data to m_out + // before becoming done if we've given it all input data except the checksum at the end. + if (m_out.bytes_written == expected_out_bytes && !m_decompressor->Done() && + m_in.bytes_written == m_in.data.size()) + { return false; // Decompressed size is larger than expected + } if (m_decompressor->Done() && m_in_bytes_read != m_in.data.size()) return false; // Compressed size is smaller than expected @@ -1432,6 +1553,9 @@ void WIAFileReader::SetUpCompressor(std::unique_ptr* compressor, compressor_data_size); break; } + case WIACompressionType::Zstd: + *compressor = std::make_unique(compression_level); + break; } } diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 356d8a3edba7..63580efe894e 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -15,6 +15,7 @@ #include #include #include +#include #include "Common/CommonTypes.h" #include "Common/File.h" @@ -34,8 +35,11 @@ enum class WIACompressionType : u32 Bzip2 = 2, LZMA = 3, LZMA2 = 4, + Zstd = 5, }; +std::pair GetAllowedCompressionLevels(WIACompressionType compression_type); + constexpr u32 WIA_MAGIC = 0x01414957; // "WIA\x1" (byteswapped to little endian) constexpr u32 RVZ_MAGIC = 0x015A5652; // "RVZ\x1" (byteswapped to little endian) @@ -250,6 +254,19 @@ class WIAFileReader : public BlobReader bool m_error_occurred = false; }; + class ZstdDecompressor final : public Decompressor + { + public: + ZstdDecompressor(); + ~ZstdDecompressor(); + + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; + + private: + ZSTD_DStream* m_stream; + }; + class Compressor { public: @@ -332,6 +349,27 @@ class WIAFileReader : public BlobReader bool m_initialization_failed = false; }; + class ZstdCompressor final : public Compressor + { + public: + ZstdCompressor(int compression_level); + ~ZstdCompressor(); + + bool Start() override; + bool Compress(const u8* data, size_t size) override; + bool End() override; + + const u8* GetData() const override { return m_buffer.data(); } + size_t GetSize() const override { return m_out_buffer.pos; } + + private: + void ExpandBuffer(size_t bytes_to_add); + + ZSTD_CStream* m_stream; + ZSTD_outBuffer m_out_buffer; + std::vector m_buffer; + }; + class Chunk { public: diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index 120b557c6b03..f739de801a2d 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -230,6 +230,11 @@ void ConvertDialog::OnFormatChanged() AddToCompressionComboBox(slow.arg(QStringLiteral("bzip2")), DiscIO::WIACompressionType::Bzip2); AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA")), DiscIO::WIACompressionType::LZMA); AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA2")), DiscIO::WIACompressionType::LZMA2); + if (format == DiscIO::BlobType::RVZ) + { + AddToCompressionComboBox(QStringLiteral("Zstandard"), DiscIO::WIACompressionType::Zstd); + m_compression->setCurrentIndex(m_compression->count() - 1); + } break; } @@ -246,19 +251,16 @@ void ConvertDialog::OnCompressionChanged() { m_compression_level->clear(); - switch (static_cast(m_compression->currentData().toInt())) - { - case DiscIO::WIACompressionType::Bzip2: - case DiscIO::WIACompressionType::LZMA: - case DiscIO::WIACompressionType::LZMA2: - for (int i = 1; i <= 9; ++i) - AddToCompressionLevelComboBox(i); + const auto compression_type = + static_cast(m_compression->currentData().toInt()); - m_compression_level->setCurrentIndex(4); + const std::pair range = DiscIO::GetAllowedCompressionLevels(compression_type); - break; - default: - break; + for (int i = range.first; i <= range.second; ++i) + { + AddToCompressionLevelComboBox(i); + if (i == 5) + m_compression_level->setCurrentIndex(m_compression_level->count() - 1); } m_compression_level->setEnabled(m_compression_level->count() > 1); From 0d433baeb5b0f0fb19d691d2d7b37ed5fc737535 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Mon, 4 May 2020 18:21:49 +0200 Subject: [PATCH 25/36] RVZ: Remove PURGE support PURGE isn't especially useful, while requiring some annoying special handling in the file format. If you want no compression, use NONE. If you want fast compression, use Zstandard. --- Source/Core/DiscIO/WIABlob.cpp | 3 ++- Source/Core/DolphinQt/ConvertDialog.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index b3c9553d4920..d5a8086789bd 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -129,7 +129,8 @@ bool WIAFileReader::Initialize(const std::string& path) const u32 compression_type = Common::swap32(m_header_2.compression_type); m_compression_type = static_cast(compression_type); - if (m_compression_type > (m_rvz ? WIACompressionType::Zstd : WIACompressionType::LZMA2)) + if (m_compression_type > (m_rvz ? WIACompressionType::Zstd : WIACompressionType::LZMA2) || + (m_rvz && m_compression_type == WIACompressionType::Purge)) { ERROR_LOG(DISCIO, "Unsupported compression type %u in %s", compression_type, path.c_str()); return false; diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index f739de801a2d..db801581a0d0 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -226,7 +226,8 @@ void ConvertDialog::OnFormatChanged() const QString slow = tr("%1 (slow)"); AddToCompressionComboBox(tr("No Compression"), DiscIO::WIACompressionType::None); - AddToCompressionComboBox(QStringLiteral("Purge"), DiscIO::WIACompressionType::Purge); + if (format == DiscIO::BlobType::WIA) + AddToCompressionComboBox(QStringLiteral("Purge"), DiscIO::WIACompressionType::Purge); AddToCompressionComboBox(slow.arg(QStringLiteral("bzip2")), DiscIO::WIACompressionType::Bzip2); AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA")), DiscIO::WIACompressionType::LZMA); AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA2")), DiscIO::WIACompressionType::LZMA2); From b06c50ed2e8740b600e546b4d5e7aa097c7a5cb9 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Tue, 5 May 2020 00:43:32 +0200 Subject: [PATCH 26/36] RVZ: Support chunk sizes between 32 KiB and 2 MiB WIA doesn't support smaller than 2 MiB. --- Source/Core/DiscIO/WIABlob.cpp | 72 +++++++++++++++++-------- Source/Core/DiscIO/WIABlob.h | 11 +++- Source/Core/DolphinQt/ConvertDialog.cpp | 8 ++- 3 files changed, 67 insertions(+), 24 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index d5a8086789bd..8d64cbb1e02f 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -27,6 +27,7 @@ #include "Common/FileUtil.h" #include "Common/Logging/Log.h" #include "Common/MsgHandler.h" +#include "Common/ScopeGuard.h" #include "Common/StringUtil.h" #include "Common/Swap.h" @@ -124,8 +125,12 @@ bool WIAFileReader::Initialize(const std::string& path) } const u32 chunk_size = Common::swap32(m_header_2.chunk_size); - if (chunk_size % VolumeWii::GROUP_TOTAL_SIZE != 0) + const auto is_power_of_two = [](u32 x) { return (x & (x - 1)) == 0; }; + if ((!m_rvz || chunk_size < VolumeWii::BLOCK_TOTAL_SIZE || !is_power_of_two(chunk_size)) && + chunk_size % VolumeWii::GROUP_TOTAL_SIZE != 0) + { return false; + } const u32 compression_type = Common::swap32(m_header_2.compression_type); m_compression_type = static_cast(compression_type); @@ -319,19 +324,20 @@ bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) const u64 bytes_to_read = std::min(data_size - (offset - data_offset), size); + m_exception_list.clear(); + m_write_to_exception_list = true; + m_exception_list_last_group_index = std::numeric_limits::max(); + Common::ScopeGuard guard([this] { m_write_to_exception_list = false; }); + bool hash_exception_error = false; if (!m_encryption_cache.EncryptGroups( offset - partition_data_offset, bytes_to_read, out_ptr, partition_data_offset, partition_total_sectors * VolumeWii::BLOCK_DATA_SIZE, partition.partition_key, - [this, chunk_size, first_sector, partition_first_sector, &hash_exception_error]( + [this, &hash_exception_error]( VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP], u64 offset) { - const u64 partition_part_offset = - (first_sector - partition_first_sector) * VolumeWii::BLOCK_TOTAL_SIZE; - const u64 index = - (offset - partition_part_offset) % chunk_size / VolumeWii::GROUP_TOTAL_SIZE; - - // EncryptGroups calls ReadWiiDecrypted, which populates m_cached_chunk - if (!m_cached_chunk.ApplyHashExceptions(hash_blocks, index)) + // EncryptGroups calls ReadWiiDecrypted, which calls ReadFromGroups, + // which populates m_exception_list when m_write_to_exception_list == true + if (!ApplyHashExceptions(m_exception_list, hash_blocks)) hash_exception_error = true; })) { @@ -392,7 +398,7 @@ bool WIAFileReader::ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 part if (!ReadFromGroups(&offset, &size, &out_ptr, chunk_size, VolumeWii::BLOCK_DATA_SIZE, data_offset, data_size, Common::swap32(data.group_index), Common::swap32(data.number_of_groups), - chunk_size / VolumeWii::GROUP_DATA_SIZE)) + std::max(1, chunk_size / VolumeWii::GROUP_DATA_SIZE))) { return false; } @@ -423,10 +429,10 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu return false; const GroupEntry group = m_group_entries[total_group_index]; - const u64 group_offset = data_offset + i * chunk_size; - const u64 offset_in_group = *offset - group_offset; + const u64 group_offset_in_data = i * chunk_size; + const u64 offset_in_group = *offset - group_offset_in_data - data_offset; - chunk_size = std::min(chunk_size, data_offset + data_size - group_offset); + chunk_size = std::min(chunk_size, data_size - group_offset_in_data); const u64 bytes_to_read = std::min(chunk_size - offset_in_group, *size); const u32 group_data_size = Common::swap32(group.data_size); @@ -444,6 +450,17 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu m_cached_chunk_offset = std::numeric_limits::max(); // Invalidate the cache return false; } + + if (m_write_to_exception_list && m_exception_list_last_group_index != total_group_index) + { + const u64 exception_list_index = offset_in_group / VolumeWii::GROUP_DATA_SIZE; + const u16 additional_offset = + static_cast(group_offset_in_data % VolumeWii::GROUP_DATA_SIZE / + VolumeWii::BLOCK_DATA_SIZE * VolumeWii::BLOCK_HEADER_SIZE); + + chunk.GetHashExceptions(&m_exception_list, exception_list_index, additional_offset); + m_exception_list_last_group_index = total_group_index; + } } *offset += bytes_to_read; @@ -1306,13 +1323,13 @@ bool WIAFileReader::Chunk::HandleExceptions(const u8* data, size_t bytes_allocat return true; } -bool WIAFileReader::Chunk::ApplyHashExceptions( - VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP], u64 exception_list_index) const +void WIAFileReader::Chunk::GetHashExceptions(std::vector* exception_list, + u64 exception_list_index, u16 additional_offset) const { - if (m_exception_lists > 0) - return false; // We still have exception lists left to read + ASSERT(m_exception_lists == 0); - const u8* data = m_compressed_exception_lists ? m_out.data.data() : m_in.data.data(); + const u8* data_start = m_compressed_exception_lists ? m_out.data.data() : m_in.data.data(); + const u8* data = data_start; for (u64 i = exception_list_index; i > 0; --i) data += Common::swap16(data) * sizeof(HashExceptionEntry) + sizeof(u16); @@ -1322,10 +1339,23 @@ bool WIAFileReader::Chunk::ApplyHashExceptions( for (size_t i = 0; i < exceptions; ++i) { - HashExceptionEntry exception; - std::memcpy(&exception, data, sizeof(HashExceptionEntry)); + std::memcpy(&exception_list->emplace_back(), data, sizeof(HashExceptionEntry)); data += sizeof(HashExceptionEntry); + u16& offset = exception_list->back().offset; + offset = Common::swap16(Common::swap16(offset) + additional_offset); + } + + ASSERT(data <= data_start + (m_compressed_exception_lists ? m_out_bytes_used_for_exceptions : + m_in_bytes_used_for_exceptions)); +} + +bool WIAFileReader::ApplyHashExceptions( + const std::vector& exception_list, + VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]) +{ + for (const HashExceptionEntry& exception : exception_list) + { const u16 offset = Common::swap16(exception.offset); const size_t block_index = offset / VolumeWii::BLOCK_HEADER_SIZE; @@ -1874,7 +1904,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, ASSERT(chunk_size > 0); const u64 iso_size = infile->GetDataSize(); - const u64 exception_lists_per_chunk = chunk_size / VolumeWii::GROUP_TOTAL_SIZE; + const u64 exception_lists_per_chunk = std::max(1, chunk_size / VolumeWii::GROUP_TOTAL_SIZE); const bool compressed_exception_lists = compression_type > WIACompressionType::Purge; u64 bytes_read = 0; diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 63580efe894e..0de24c8a042d 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -381,8 +381,8 @@ class WIAFileReader : public BlobReader bool Read(u64 offset, u64 size, u8* out_ptr); // This can only be called once at least one byte of data has been read - bool ApplyHashExceptions(VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP], - u64 exception_list_index) const; + void GetHashExceptions(std::vector* exception_list, + u64 exception_list_index, u16 additional_offset) const; template bool ReadAll(std::vector* vector) @@ -419,6 +419,9 @@ class WIAFileReader : public BlobReader Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size, u32 exception_lists); + static bool ApplyHashExceptions(const std::vector& exception_list, + VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]); + static std::string VersionToString(u32 version); static u32 LZMA2DictionarySize(u8 p); @@ -536,6 +539,10 @@ class WIAFileReader : public BlobReader u64 m_cached_chunk_offset = std::numeric_limits::max(); WiiEncryptionCache m_encryption_cache; + std::vector m_exception_list; + bool m_write_to_exception_list = false; + u64 m_exception_list_last_group_index; + WIAHeader1 m_header_1; WIAHeader2 m_header_2; std::vector m_partition_entries; diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index db801581a0d0..1883918d9560 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -199,12 +199,18 @@ void ConvertDialog::OnFormatChanged() break; } case DiscIO::BlobType::WIA: - case DiscIO::BlobType::RVZ: m_block_size->setEnabled(true); // This is the smallest block size supported by WIA. For performance, larger sizes are avoided. AddToBlockSizeComboBox(0x200000); + break; + case DiscIO::BlobType::RVZ: + m_block_size->setEnabled(true); + + for (int block_size = MIN_BLOCK_SIZE; block_size <= MAX_BLOCK_SIZE; block_size *= 2) + AddToBlockSizeComboBox(block_size); + break; default: break; From f5ef70fc76d8f9916b77ef70f035adda1417e0cb Mon Sep 17 00:00:00 2001 From: JosJuice Date: Tue, 5 May 2020 14:41:46 +0200 Subject: [PATCH 27/36] RVZ: Don't store redundant exceptions when chunk size is < 2 MiB --- Source/Core/DiscIO/WIABlob.cpp | 444 ++++++++++++++++++++------------- Source/Core/DiscIO/WIABlob.h | 13 +- 2 files changed, 278 insertions(+), 179 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 8d64cbb1e02f..0d279e4c40bd 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -457,7 +457,6 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu const u16 additional_offset = static_cast(group_offset_in_data % VolumeWii::GROUP_DATA_SIZE / VolumeWii::BLOCK_DATA_SIZE * VolumeWii::BLOCK_HEADER_SIZE); - chunk.GetHashExceptions(&m_exception_list, exception_list_index, additional_offset); m_exception_list_last_group_index = total_group_index; } @@ -1590,24 +1589,39 @@ void WIAFileReader::SetUpCompressor(std::unique_ptr* compressor, } } +static bool AllAre(const std::vector& data, u8 x) +{ + return std::all_of(data.begin(), data.end(), [x](u8 y) { return x == y; }); +}; + +static bool AllAre(const u8* begin, const u8* end, u8 x) +{ + return std::all_of(begin, end, [x](u8 y) { return x == y; }); +}; + +static bool AllZero(const std::vector& data) +{ + return AllAre(data, 0); +}; + +static bool AllSame(const std::vector& data) +{ + return AllAre(data, data.front()); +}; + +static bool AllSame(const u8* begin, const u8* end) +{ + return AllAre(begin, end, *begin); +}; + ConversionResult WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, const std::vector& partition_entries, const std::vector& data_entries, std::map* reusable_groups, - std::mutex* reusable_groups_mutex, u64 exception_lists_per_chunk, - bool compressed_exception_lists) + std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, + u64 exception_lists_per_chunk, bool compressed_exception_lists) { - const auto all_are = [](const std::vector& data, u8 x) { - return std::all_of(data.begin(), data.end(), [x](u8 y) { return x == y; }); - }; - - const auto all_zero = [&all_are](const std::vector& data) { return all_are(data, 0); }; - - const auto all_same = [&all_are](const std::vector& data) { - return all_are(data, data.front()); - }; - const auto reuse_id_exists = [reusable_groups, reusable_groups_mutex](const std::optional& reuse_id) { if (!reuse_id) @@ -1618,14 +1632,15 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters return it != reusable_groups->end(); }; - std::optional reuse_id; - - state->exceptions_buffer.clear(); + std::vector output_entries; if (!parameters.data_entry->is_partition) { - if (all_same(parameters.data)) - reuse_id = ReuseID{nullptr, parameters.data.size(), false, parameters.data.front()}; + OutputParametersEntry& entry = output_entries.emplace_back(); + entry.main_data = std::move(parameters.data); + + if (AllSame(entry.main_data)) + entry.reuse_id = ReuseID{nullptr, entry.main_data.size(), false, entry.main_data.front()}; } else { @@ -1639,177 +1654,248 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters ASSERT(parameters.data.size() % VolumeWii::BLOCK_TOTAL_SIZE == 0); const u64 blocks = parameters.data.size() / VolumeWii::BLOCK_TOTAL_SIZE; - const u64 bytes_to_write = blocks * VolumeWii::BLOCK_DATA_SIZE; - const auto create_reuse_id = [&partition_entry, bytes_to_write](u8 value, bool decrypted) { - return ReuseID{&partition_entry.partition_key, bytes_to_write, decrypted, value}; - }; + const u64 blocks_per_chunk = chunks_per_wii_group == 1 ? + exception_lists_per_chunk * VolumeWii::BLOCKS_PER_GROUP : + VolumeWii::BLOCKS_PER_GROUP / chunks_per_wii_group; - // Set this group as reusable if the encrypted data is all_same - if (all_same(parameters.data)) - reuse_id = create_reuse_id(parameters.data.front(), false); + const u64 chunks = Common::AlignUp(blocks, blocks_per_chunk) / blocks_per_chunk; - if (reuse_id_exists(reuse_id)) - return OutputParameters{{}, {}, reuse_id, parameters.bytes_read, parameters.group_index}; + const u64 in_data_per_chunk = blocks_per_chunk * VolumeWii::BLOCK_TOTAL_SIZE; + const u64 out_data_per_chunk = blocks_per_chunk * VolumeWii::BLOCK_DATA_SIZE; - std::vector> exception_lists(exception_lists_per_chunk); + const auto create_reuse_id = [&partition_entry, blocks, + blocks_per_chunk](u8 value, bool decrypted, u64 block) { + const u64 size = std::min(blocks - block, blocks_per_chunk) * VolumeWii::BLOCK_DATA_SIZE; + return ReuseID{&partition_entry.partition_key, size, decrypted, value}; + }; - for (u64 i = 0; i < groups; ++i) + const u8* parameters_data_end = parameters.data.data() + parameters.data.size(); + for (u64 i = 0; i < chunks; ++i) { - const u64 offset_of_group = i * VolumeWii::GROUP_TOTAL_SIZE; - const u64 write_offset_of_group = i * VolumeWii::GROUP_DATA_SIZE; + const u64 block_index = i * blocks_per_chunk; + + OutputParametersEntry& entry = output_entries.emplace_back(); + std::optional& reuse_id = entry.reuse_id; - const u64 blocks_in_this_group = - std::min(VolumeWii::BLOCKS_PER_GROUP, blocks - i * VolumeWii::BLOCKS_PER_GROUP); + // Set this chunk as reusable if the encrypted data is AllSame + const u8* data = parameters.data.data() + block_index * VolumeWii::BLOCK_TOTAL_SIZE; + if (AllSame(data, std::min(parameters_data_end, data + in_data_per_chunk))) + reuse_id = create_reuse_id(parameters.data.front(), false, i * blocks_per_chunk); - for (u32 j = 0; j < VolumeWii::BLOCKS_PER_GROUP; ++j) + if (!reuse_id_exists(reuse_id) && + !(reuse_id && std::any_of(output_entries.begin(), output_entries.begin() + i, + [reuse_id](const auto& e) { return e.reuse_id == reuse_id; }))) { - if (j < blocks_in_this_group) + const u64 bytes_left = (blocks - block_index) * VolumeWii::BLOCK_DATA_SIZE; + entry.main_data.resize(std::min(out_data_per_chunk, bytes_left)); + } + } + + if (!std::all_of(output_entries.begin(), output_entries.end(), + [](const OutputParametersEntry& entry) { return entry.main_data.empty(); })) + { + const u64 number_of_exception_lists = + chunks_per_wii_group == 1 ? exception_lists_per_chunk : chunks; + std::vector> exception_lists(number_of_exception_lists); + + for (u64 i = 0; i < groups; ++i) + { + const u64 offset_of_group = i * VolumeWii::GROUP_TOTAL_SIZE; + const u64 write_offset_of_group = i * VolumeWii::GROUP_DATA_SIZE; + + const u64 blocks_in_this_group = + std::min(VolumeWii::BLOCKS_PER_GROUP, blocks - i * VolumeWii::BLOCKS_PER_GROUP); + + for (u32 j = 0; j < VolumeWii::BLOCKS_PER_GROUP; ++j) { - const u64 offset_of_block = offset_of_group + j * VolumeWii::BLOCK_TOTAL_SIZE; - VolumeWii::DecryptBlockData(parameters.data.data() + offset_of_block, - state->decryption_buffer[j].data(), &aes_context); + if (j < blocks_in_this_group) + { + const u64 offset_of_block = offset_of_group + j * VolumeWii::BLOCK_TOTAL_SIZE; + VolumeWii::DecryptBlockData(parameters.data.data() + offset_of_block, + state->decryption_buffer[j].data(), &aes_context); + } + else + { + state->decryption_buffer[j].fill(0); + } } - else + + VolumeWii::HashGroup(state->decryption_buffer.data(), state->hash_buffer.data()); + + for (u64 j = 0; j < blocks_in_this_group; ++j) { - state->decryption_buffer[j].fill(0); - } - } + const u64 chunk_index = j / blocks_per_chunk; + const u64 block_index_in_chunk = j % blocks_per_chunk; - VolumeWii::HashGroup(state->decryption_buffer.data(), state->hash_buffer.data()); + if (output_entries[chunk_index].main_data.empty()) + continue; - for (u64 j = 0; j < blocks_in_this_group; ++j) - { - const u64 offset_of_block = offset_of_group + j * VolumeWii::BLOCK_TOTAL_SIZE; - const u64 hash_offset_of_block = j * VolumeWii::BLOCK_HEADER_SIZE; + const u64 exception_list_index = chunks_per_wii_group == 1 ? i : chunk_index; - VolumeWii::HashBlock hashes; - VolumeWii::DecryptBlockHashes(parameters.data.data() + offset_of_block, &hashes, - &aes_context); + const u64 offset_of_block = offset_of_group + j * VolumeWii::BLOCK_TOTAL_SIZE; + const u64 hash_offset_of_block = block_index_in_chunk * VolumeWii::BLOCK_HEADER_SIZE; + + VolumeWii::HashBlock hashes; + VolumeWii::DecryptBlockHashes(parameters.data.data() + offset_of_block, &hashes, + &aes_context); + + const auto compare_hash = [&](size_t offset_in_block) { + ASSERT(offset_in_block + sizeof(SHA1) <= VolumeWii::BLOCK_HEADER_SIZE); + + const u8* desired_hash = reinterpret_cast(&hashes) + offset_in_block; + const u8* computed_hash = + reinterpret_cast(&state->hash_buffer[j]) + offset_in_block; + + // We want to store a hash exception either if there is a hash mismatch, or if this + // chunk might get reused in a context where it is paired up (within a 2 MiB Wii group) + // with chunks that are different from the chunks it currently is paired up with, since + // that affects the recalculated hashes. Chunks which have been marked as reusable at + // this point normally have zero matching hashes anyway, so this shouldn't waste space. + if ((chunks_per_wii_group != 1 && output_entries[chunk_index].reuse_id) || + !std::equal(desired_hash, desired_hash + sizeof(SHA1), computed_hash)) + { + const u64 hash_offset = hash_offset_of_block + offset_in_block; + ASSERT(hash_offset <= std::numeric_limits::max()); + + HashExceptionEntry& exception = exception_lists[exception_list_index].emplace_back(); + exception.offset = static_cast(Common::swap16(hash_offset)); + std::memcpy(exception.hash.data(), desired_hash, sizeof(SHA1)); + } + }; + + const auto compare_hashes = [&compare_hash](size_t offset, size_t size) { + for (size_t l = 0; l < size; l += sizeof(SHA1)) + // The std::min is to ensure that we don't go beyond the end of HashBlock with + // padding_2, which is 32 bytes long (not divisible by sizeof(SHA1), which is 20). + compare_hash(offset + std::min(l, size - sizeof(SHA1))); + }; + + using HashBlock = VolumeWii::HashBlock; + compare_hashes(offsetof(HashBlock, h0), sizeof(HashBlock::h0)); + compare_hashes(offsetof(HashBlock, padding_0), sizeof(HashBlock::padding_0)); + compare_hashes(offsetof(HashBlock, h1), sizeof(HashBlock::h1)); + compare_hashes(offsetof(HashBlock, padding_1), sizeof(HashBlock::padding_1)); + compare_hashes(offsetof(HashBlock, h2), sizeof(HashBlock::h2)); + compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); + } - const auto compare_hash = [&](size_t offset_in_block) { - ASSERT(offset_in_block + sizeof(SHA1) <= VolumeWii::BLOCK_HEADER_SIZE); + for (u64 j = 0; j < blocks_in_this_group; ++j) + { + const u64 chunk_index = j / blocks_per_chunk; + const u64 block_index_in_chunk = j % blocks_per_chunk; - const u8* desired_hash = reinterpret_cast(&hashes) + offset_in_block; - const u8* computed_hash = reinterpret_cast(&state->hash_buffer[j]) + offset_in_block; + OutputParametersEntry& entry = output_entries[chunk_index]; + if (entry.main_data.empty()) + continue; - if (!std::equal(desired_hash, desired_hash + sizeof(SHA1), computed_hash)) - { - const u64 hash_offset = hash_offset_of_block + offset_in_block; - ASSERT(hash_offset <= std::numeric_limits::max()); + const u64 write_offset_of_block = + write_offset_of_group + block_index_in_chunk * VolumeWii::BLOCK_DATA_SIZE; - HashExceptionEntry& exception = exception_lists[i].emplace_back(); - exception.offset = static_cast(Common::swap16(hash_offset)); - std::memcpy(exception.hash.data(), desired_hash, sizeof(SHA1)); - } - }; - - const auto compare_hashes = [&compare_hash](size_t offset, size_t size) { - for (size_t l = 0; l < size; l += sizeof(SHA1)) - // The std::min is to ensure that we don't go beyond the end of HashBlock with - // padding_2, which is 32 bytes long (not divisible by sizeof(SHA1), which is 20). - compare_hash(offset + std::min(l, size - sizeof(SHA1))); - }; - - using HashBlock = VolumeWii::HashBlock; - compare_hashes(offsetof(HashBlock, h0), sizeof(HashBlock::h0)); - compare_hashes(offsetof(HashBlock, padding_0), sizeof(HashBlock::padding_0)); - compare_hashes(offsetof(HashBlock, h1), sizeof(HashBlock::h1)); - compare_hashes(offsetof(HashBlock, padding_1), sizeof(HashBlock::padding_1)); - compare_hashes(offsetof(HashBlock, h2), sizeof(HashBlock::h2)); - compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); + std::memcpy(entry.main_data.data() + write_offset_of_block, + state->decryption_buffer[j].data(), VolumeWii::BLOCK_DATA_SIZE); + } } - for (u64 j = 0; j < blocks_in_this_group; ++j) + for (size_t i = 0; i < exception_lists.size(); ++i) { - const u64 write_offset_of_block = write_offset_of_group + j * VolumeWii::BLOCK_DATA_SIZE; - std::memcpy(parameters.data.data() + write_offset_of_block, - state->decryption_buffer[j].data(), VolumeWii::BLOCK_DATA_SIZE); - } - } + OutputParametersEntry& entry = output_entries[chunks_per_wii_group == 1 ? 0 : i]; + if (entry.main_data.empty()) + continue; - bool have_exceptions = false; + const std::vector& in = exception_lists[i]; + std::vector& out = entry.exception_lists; - for (const std::vector& exception_list : exception_lists) - { - const u16 exceptions = Common::swap16(static_cast(exception_list.size())); - PushBack(&state->exceptions_buffer, exceptions); - for (const HashExceptionEntry& exception : exception_list) - PushBack(&state->exceptions_buffer, exception); - if (!exception_list.empty()) - have_exceptions = true; - } + const u16 exceptions = Common::swap16(static_cast(in.size())); + PushBack(&out, exceptions); + for (const HashExceptionEntry& exception : in) + PushBack(&out, exception); + } - parameters.data.resize(bytes_to_write); + for (u64 i = 0; i < output_entries.size(); ++i) + { + OutputParametersEntry& entry = output_entries[i]; + if (entry.main_data.empty() || entry.reuse_id) + continue; - // Set this group as reusable if it lacks exceptions and the decrypted data is all_same - if (!reuse_id && !have_exceptions && all_same(parameters.data)) - reuse_id = create_reuse_id(parameters.data.front(), true); + // Set this chunk as reusable if it lacks exceptions and the decrypted data is AllSame + if (AllZero(entry.exception_lists) && AllSame(parameters.data)) + entry.reuse_id = create_reuse_id(parameters.data.front(), true, i * blocks_per_chunk); + } + } } - // Special case - a compressed size of zero is treated by WIA as meaning the data is all zeroes - if (all_zero(state->exceptions_buffer) && all_zero(parameters.data)) - return OutputParameters{{}, {}, reuse_id, parameters.bytes_read, parameters.group_index}; + for (OutputParametersEntry& entry : output_entries) + { + if (entry.main_data.empty()) + continue; - if (reuse_id_exists(reuse_id)) - return OutputParameters{{}, {}, reuse_id, parameters.bytes_read, parameters.group_index}; + // Special case - a compressed size of zero is treated by WIA as meaning the data is all zeroes + const bool all_zero = AllZero(entry.exception_lists) && AllZero(entry.main_data); - if (state->compressor) - { - if (!state->compressor->Start()) - return ConversionResultCode::InternalError; - } + if (all_zero || reuse_id_exists(entry.reuse_id)) + { + entry.exception_lists.clear(); + entry.main_data.clear(); + continue; + } - if (!state->exceptions_buffer.empty()) - { - if (compressed_exception_lists && state->compressor) + if (state->compressor) { - if (!state->compressor->Compress(state->exceptions_buffer.data(), - state->exceptions_buffer.size())) - { + if (!state->compressor->Start()) return ConversionResultCode::InternalError; - } - - state->exceptions_buffer.clear(); } - else + + if (!entry.exception_lists.empty()) { - if (!compressed_exception_lists) + if (compressed_exception_lists && state->compressor) { - while (state->exceptions_buffer.size() % 4 != 0) - state->exceptions_buffer.push_back(0); - } + if (!state->compressor->Compress(entry.exception_lists.data(), + entry.exception_lists.size())) + { + return ConversionResultCode::InternalError; + } - if (state->compressor) + entry.exception_lists.clear(); + } + else { - if (!state->compressor->AddPrecedingDataOnlyForPurgeHashing( - state->exceptions_buffer.data(), state->exceptions_buffer.size())) + if (!compressed_exception_lists) { - return ConversionResultCode::InternalError; + while (entry.exception_lists.size() % 4 != 0) + entry.exception_lists.push_back(0); + } + + if (state->compressor) + { + if (!state->compressor->AddPrecedingDataOnlyForPurgeHashing(entry.exception_lists.data(), + entry.exception_lists.size())) + { + return ConversionResultCode::InternalError; + } } } } - } - if (state->compressor) - { - if (!state->compressor->Compress(parameters.data.data(), parameters.data.size())) - return ConversionResultCode::InternalError; - if (!state->compressor->End()) - return ConversionResultCode::InternalError; - } + if (state->compressor) + { + if (!state->compressor->Compress(entry.main_data.data(), entry.main_data.size())) + return ConversionResultCode::InternalError; + if (!state->compressor->End()) + return ConversionResultCode::InternalError; + } - if (state->compressor) - { - const u8* data = state->compressor->GetData(); - const size_t size = state->compressor->GetSize(); + if (state->compressor) + { + const u8* data = state->compressor->GetData(); + const size_t size = state->compressor->GetSize(); - parameters.data.resize(size); - std::copy(data, data + size, parameters.data.data()); + entry.main_data.resize(size); + std::copy(data, data + size, entry.main_data.data()); + } } - return OutputParameters{state->exceptions_buffer, std::move(parameters.data), reuse_id, - parameters.bytes_read, parameters.group_index}; + return OutputParameters{std::move(output_entries), parameters.bytes_read, parameters.group_index}; } ConversionResultCode WIAFileReader::Output(const OutputParameters& parameters, @@ -1818,41 +1904,47 @@ ConversionResultCode WIAFileReader::Output(const OutputParameters& parameters, std::mutex* reusable_groups_mutex, GroupEntry* group_entry, u64* bytes_written) { - if (parameters.reuse_id) + for (const OutputParametersEntry& entry : parameters.entries) { - std::lock_guard guard(*reusable_groups_mutex); - const auto it = reusable_groups->find(*parameters.reuse_id); - if (it != reusable_groups->end()) + if (entry.reuse_id) { - *group_entry = it->second; - return ConversionResultCode::Success; + std::lock_guard guard(*reusable_groups_mutex); + const auto it = reusable_groups->find(*entry.reuse_id); + if (it != reusable_groups->end()) + { + *group_entry = it->second; + ++group_entry; + continue; + } } - } - const size_t data_size = parameters.exception_lists.size() + parameters.main_data.size(); + const size_t data_size = entry.exception_lists.size() + entry.main_data.size(); - if (*bytes_written >> 2 > std::numeric_limits::max()) - return ConversionResultCode::InternalError; + if (*bytes_written >> 2 > std::numeric_limits::max()) + return ConversionResultCode::InternalError; - ASSERT((*bytes_written & 3) == 0); - group_entry->data_offset = Common::swap32(static_cast(*bytes_written >> 2)); - group_entry->data_size = Common::swap32(static_cast(data_size)); + ASSERT((*bytes_written & 3) == 0); + group_entry->data_offset = Common::swap32(static_cast(*bytes_written >> 2)); + group_entry->data_size = Common::swap32(static_cast(data_size)); - if (!outfile->WriteArray(parameters.exception_lists.data(), parameters.exception_lists.size())) - return ConversionResultCode::WriteFailed; - if (!outfile->WriteArray(parameters.main_data.data(), parameters.main_data.size())) - return ConversionResultCode::WriteFailed; + if (!outfile->WriteArray(entry.exception_lists.data(), entry.exception_lists.size())) + return ConversionResultCode::WriteFailed; + if (!outfile->WriteArray(entry.main_data.data(), entry.main_data.size())) + return ConversionResultCode::WriteFailed; - *bytes_written += data_size; + *bytes_written += data_size; - if (parameters.reuse_id) - { - std::lock_guard guard(*reusable_groups_mutex); - reusable_groups->emplace(*parameters.reuse_id, *group_entry); - } + if (entry.reuse_id) + { + std::lock_guard guard(*reusable_groups_mutex); + reusable_groups->emplace(*entry.reuse_id, *group_entry); + } - if (!PadTo4(outfile, bytes_written)) - return ConversionResultCode::WriteFailed; + if (!PadTo4(outfile, bytes_written)) + return ConversionResultCode::WriteFailed; + + ++group_entry; + } return ConversionResultCode::Success; } @@ -1904,6 +1996,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, ASSERT(chunk_size > 0); const u64 iso_size = infile->GetDataSize(); + const u64 chunks_per_wii_group = std::max(1, VolumeWii::GROUP_TOTAL_SIZE / chunk_size); const u64 exception_lists_per_chunk = std::max(1, chunk_size / VolumeWii::GROUP_TOTAL_SIZE); const bool compressed_exception_lists = compression_type > WIACompressionType::Purge; @@ -1961,8 +2054,8 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, const auto process_and_compress = [&](CompressThreadState* state, CompressParameters parameters) { return ProcessAndCompress(state, std::move(parameters), partition_entries, data_entries, - &reusable_groups, &reusable_groups_mutex, exception_lists_per_chunk, - compressed_exception_lists); + &reusable_groups, &reusable_groups_mutex, chunks_per_wii_group, + exception_lists_per_chunk, compressed_exception_lists); }; const auto output = [&](OutputParameters parameters) { @@ -1973,8 +2066,8 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, if (result != ConversionResultCode::Success) return result; - return RunCallback(parameters.group_index + 1, parameters.bytes_read, bytes_written, - total_groups, iso_size, callback, arg); + return RunCallback(parameters.group_index + parameters.entries.size(), parameters.bytes_read, + bytes_written, total_groups, iso_size, callback, arg); }; MultithreadedCompressor mt_compressor( @@ -2019,13 +2112,16 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, ASSERT(groups_processed == first_group); ASSERT(bytes_read == data_offset); - for (u32 i = first_group; i < last_group; ++i) + while (groups_processed < last_group) { const ConversionResultCode status = mt_compressor.GetStatus(); if (status != ConversionResultCode::Success) return status; - const u64 bytes_to_read = std::min(chunk_size, data_offset + data_size - bytes_read); + u64 bytes_to_read = chunk_size; + if (data_entry.is_partition) + bytes_to_read = std::max(bytes_to_read, VolumeWii::GROUP_TOTAL_SIZE); + bytes_to_read = std::min(bytes_to_read, data_offset + data_size - bytes_read); buffer.resize(bytes_to_read); if (!infile->Read(bytes_read, bytes_to_read, buffer.data())) @@ -2035,7 +2131,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, mt_compressor.CompressAndWrite( CompressParameters{buffer, &data_entry, bytes_read, groups_processed}); - ++groups_processed; + groups_processed += Common::AlignUp(bytes_to_read, chunk_size) / chunk_size; } } diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 0de24c8a042d..a9a9387fabf6 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -464,8 +464,6 @@ class WIAFileReader : public BlobReader std::vector hash_buffer = std::vector(VolumeWii::BLOCKS_PER_GROUP); - - std::vector exceptions_buffer; }; struct CompressParameters @@ -476,11 +474,16 @@ class WIAFileReader : public BlobReader size_t group_index; }; - struct OutputParameters + struct OutputParametersEntry { std::vector exception_lists; std::vector main_data; std::optional reuse_id; + }; + + struct OutputParameters + { + std::vector entries; u64 bytes_read; size_t group_index; }; @@ -511,8 +514,8 @@ class WIAFileReader : public BlobReader const std::vector& partition_entries, const std::vector& data_entries, std::map* reusable_groups, - std::mutex* reusable_groups_mutex, u64 exception_lists_per_chunk, - bool compressed_exception_lists); + std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, + u64 exception_lists_per_chunk, bool compressed_exception_lists); static ConversionResultCode Output(const OutputParameters& parameters, File::IOFile* outfile, std::map* reusable_groups, std::mutex* reusable_groups_mutex, GroupEntry* group_entry, From 1e92b54bf576d9fafad79ef1650b58222d468b51 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 10 May 2020 20:20:08 +0200 Subject: [PATCH 28/36] WIA/RVZ: Skip some memory allocations when reusing chunks --- Source/Core/DiscIO/WIABlob.cpp | 127 ++++++++++++++++++++------------- Source/Core/DiscIO/WIABlob.h | 6 +- 2 files changed, 83 insertions(+), 50 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 0d279e4c40bd..e56794681d19 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -1589,6 +1590,24 @@ void WIAFileReader::SetUpCompressor(std::unique_ptr* compressor, } } +bool WIAFileReader::TryReuse(std::map* reusable_groups, + std::mutex* reusable_groups_mutex, OutputParametersEntry* entry) +{ + if (entry->reused_group) + return true; + + if (!entry->reuse_id) + return false; + + std::lock_guard guard(*reusable_groups_mutex); + const auto it = reusable_groups->find(*entry->reuse_id); + if (it == reusable_groups->end()) + return false; + + entry->reused_group = it->second; + return true; +} + static bool AllAre(const std::vector& data, u8 x) { return std::all_of(data.begin(), data.end(), [x](u8 y) { return x == y; }); @@ -1622,16 +1641,6 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, u64 exception_lists_per_chunk, bool compressed_exception_lists) { - const auto reuse_id_exists = [reusable_groups, - reusable_groups_mutex](const std::optional& reuse_id) { - if (!reuse_id) - return false; - - std::lock_guard guard(*reusable_groups_mutex); - const auto it = reusable_groups->find(*reuse_id); - return it != reusable_groups->end(); - }; - std::vector output_entries; if (!parameters.data_entry->is_partition) @@ -1664,6 +1673,8 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters const u64 in_data_per_chunk = blocks_per_chunk * VolumeWii::BLOCK_TOTAL_SIZE; const u64 out_data_per_chunk = blocks_per_chunk * VolumeWii::BLOCK_DATA_SIZE; + const size_t first_chunk = output_entries.size(); + const auto create_reuse_id = [&partition_entry, blocks, blocks_per_chunk](u8 value, bool decrypted, u64 block) { const u64 size = std::min(blocks - block, blocks_per_chunk) * VolumeWii::BLOCK_DATA_SIZE; @@ -1683,17 +1694,18 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters if (AllSame(data, std::min(parameters_data_end, data + in_data_per_chunk))) reuse_id = create_reuse_id(parameters.data.front(), false, i * blocks_per_chunk); - if (!reuse_id_exists(reuse_id) && - !(reuse_id && std::any_of(output_entries.begin(), output_entries.begin() + i, - [reuse_id](const auto& e) { return e.reuse_id == reuse_id; }))) + TryReuse(reusable_groups, reusable_groups_mutex, &entry); + if (!entry.reused_group && reuse_id) { - const u64 bytes_left = (blocks - block_index) * VolumeWii::BLOCK_DATA_SIZE; - entry.main_data.resize(std::min(out_data_per_chunk, bytes_left)); + const auto it = std::find_if(output_entries.begin(), output_entries.begin() + i, + [reuse_id](const auto& e) { return e.reuse_id == reuse_id; }); + if (it != output_entries.begin() + i) + entry.reused_group = it->reused_group; } } if (!std::all_of(output_entries.begin(), output_entries.end(), - [](const OutputParametersEntry& entry) { return entry.main_data.empty(); })) + [](const OutputParametersEntry& entry) { return entry.reused_group; })) { const u64 number_of_exception_lists = chunks_per_wii_group == 1 ? exception_lists_per_chunk : chunks; @@ -1728,7 +1740,7 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters const u64 chunk_index = j / blocks_per_chunk; const u64 block_index_in_chunk = j % blocks_per_chunk; - if (output_entries[chunk_index].main_data.empty()) + if (output_entries[chunk_index].reused_group) continue; const u64 exception_list_index = chunks_per_wii_group == 1 ? i : chunk_index; @@ -1780,27 +1792,50 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); } - for (u64 j = 0; j < blocks_in_this_group; ++j) + static_assert(std::is_trivially_copyable_v); + const u8* in_ptr = state->decryption_buffer[0].data(); + for (u64 j = 0; j < chunks; ++j) { - const u64 chunk_index = j / blocks_per_chunk; - const u64 block_index_in_chunk = j % blocks_per_chunk; + OutputParametersEntry& entry = output_entries[first_chunk + j]; - OutputParametersEntry& entry = output_entries[chunk_index]; - if (entry.main_data.empty()) - continue; + if (!entry.reused_group) + { + const u64 bytes_left = (blocks - j * blocks_per_chunk) * VolumeWii::BLOCK_DATA_SIZE; + const u64 bytes_to_write_total = std::min(out_data_per_chunk, bytes_left); + + if (i == 0) + entry.main_data.resize(bytes_to_write_total); + + const u64 bytes_to_write = std::min(bytes_to_write_total, VolumeWii::GROUP_DATA_SIZE); + + std::memcpy(entry.main_data.data() + write_offset_of_group, in_ptr, bytes_to_write); - const u64 write_offset_of_block = - write_offset_of_group + block_index_in_chunk * VolumeWii::BLOCK_DATA_SIZE; + // Set this chunk as reusable if the decrypted data is AllSame. + // There is also a requirement that it lacks exceptions, but this is checked later + if (i == 0 && !entry.reuse_id) + { + if (AllSame(in_ptr, in_ptr + bytes_to_write)) + entry.reuse_id = create_reuse_id(*in_ptr, true, j * blocks_per_chunk); + } + else + { + if (entry.reuse_id && entry.reuse_id->decrypted && + (!AllSame(in_ptr, in_ptr + bytes_to_write) || entry.reuse_id->value != *in_ptr)) + { + entry.reuse_id.reset(); + } + } + } - std::memcpy(entry.main_data.data() + write_offset_of_block, - state->decryption_buffer[j].data(), VolumeWii::BLOCK_DATA_SIZE); + in_ptr += out_data_per_chunk; } } for (size_t i = 0; i < exception_lists.size(); ++i) { OutputParametersEntry& entry = output_entries[chunks_per_wii_group == 1 ? 0 : i]; - if (entry.main_data.empty()) + if (entry.reused_group) continue; const std::vector& in = exception_lists[i]; @@ -1815,25 +1850,23 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters for (u64 i = 0; i < output_entries.size(); ++i) { OutputParametersEntry& entry = output_entries[i]; - if (entry.main_data.empty() || entry.reuse_id) - continue; - // Set this chunk as reusable if it lacks exceptions and the decrypted data is AllSame - if (AllZero(entry.exception_lists) && AllSame(parameters.data)) - entry.reuse_id = create_reuse_id(parameters.data.front(), true, i * blocks_per_chunk); + // If this chunk was set as reusable because the decrypted data is AllSame, + // but it has exceptions, unmark it as reusable + if (entry.reuse_id && entry.reuse_id->decrypted && !AllZero(entry.exception_lists)) + entry.reuse_id.reset(); } } } for (OutputParametersEntry& entry : output_entries) { - if (entry.main_data.empty()) + TryReuse(reusable_groups, reusable_groups_mutex, &entry); + if (entry.reused_group) continue; // Special case - a compressed size of zero is treated by WIA as meaning the data is all zeroes - const bool all_zero = AllZero(entry.exception_lists) && AllZero(entry.main_data); - - if (all_zero || reuse_id_exists(entry.reuse_id)) + if (AllZero(entry.exception_lists) && AllZero(entry.main_data)) { entry.exception_lists.clear(); entry.main_data.clear(); @@ -1898,24 +1931,20 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters return OutputParameters{std::move(output_entries), parameters.bytes_read, parameters.group_index}; } -ConversionResultCode WIAFileReader::Output(const OutputParameters& parameters, +ConversionResultCode WIAFileReader::Output(std::vector* entries, File::IOFile* outfile, std::map* reusable_groups, std::mutex* reusable_groups_mutex, GroupEntry* group_entry, u64* bytes_written) { - for (const OutputParametersEntry& entry : parameters.entries) + for (OutputParametersEntry& entry : *entries) { - if (entry.reuse_id) + TryReuse(reusable_groups, reusable_groups_mutex, &entry); + if (entry.reused_group) { - std::lock_guard guard(*reusable_groups_mutex); - const auto it = reusable_groups->find(*entry.reuse_id); - if (it != reusable_groups->end()) - { - *group_entry = it->second; - ++group_entry; - continue; - } + *group_entry = *entry.reused_group; + ++group_entry; + continue; } const size_t data_size = entry.exception_lists.size() + entry.main_data.size(); @@ -2060,7 +2089,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, const auto output = [&](OutputParameters parameters) { const ConversionResultCode result = - Output(parameters, outfile, &reusable_groups, &reusable_groups_mutex, + Output(¶meters.entries, outfile, &reusable_groups, &reusable_groups_mutex, &group_entries[parameters.group_index], &bytes_written); if (result != ConversionResultCode::Success) diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index a9a9387fabf6..6b2af6f904ad 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -479,6 +479,7 @@ class WIAFileReader : public BlobReader std::vector exception_lists; std::vector main_data; std::optional reuse_id; + std::optional reused_group; }; struct OutputParameters @@ -509,6 +510,8 @@ class WIAFileReader : public BlobReader static void SetUpCompressor(std::unique_ptr* compressor, WIACompressionType compression_type, int compression_level, WIAHeader2* header_2); + static bool TryReuse(std::map* reusable_groups, + std::mutex* reusable_groups_mutex, OutputParametersEntry* entry); static ConversionResult ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, const std::vector& partition_entries, @@ -516,7 +519,8 @@ class WIAFileReader : public BlobReader std::map* reusable_groups, std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, u64 exception_lists_per_chunk, bool compressed_exception_lists); - static ConversionResultCode Output(const OutputParameters& parameters, File::IOFile* outfile, + static ConversionResultCode Output(std::vector* entries, + File::IOFile* outfile, std::map* reusable_groups, std::mutex* reusable_groups_mutex, GroupEntry* group_entry, u64* bytes_written); From 4b74993374348185c304a68b2b159f3cdbb89360 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 10 May 2020 19:21:05 +0200 Subject: [PATCH 29/36] RVZ: Store pseudorandom junk data efficiently --- Source/Core/DiscIO/CMakeLists.txt | 2 + Source/Core/DiscIO/DiscIO.vcxproj | 2 + Source/Core/DiscIO/DiscIO.vcxproj.filters | 6 + .../Core/DiscIO/LaggedFibonacciGenerator.cpp | 212 +++++++++ Source/Core/DiscIO/LaggedFibonacciGenerator.h | 51 +++ Source/Core/DiscIO/WIABlob.cpp | 409 +++++++++++++++--- Source/Core/DiscIO/WIABlob.h | 74 +++- Source/Core/DolphinQt/ConvertDialog.cpp | 4 + 8 files changed, 687 insertions(+), 73 deletions(-) create mode 100644 Source/Core/DiscIO/LaggedFibonacciGenerator.cpp create mode 100644 Source/Core/DiscIO/LaggedFibonacciGenerator.h diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index dcbab6cce478..f0deb14ca203 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -21,6 +21,8 @@ add_library(discio FileSystemGCWii.h Filesystem.cpp Filesystem.h + LaggedFibonacciGenerator.cpp + LaggedFibonacciGenerator.h MultithreadedCompressor.h NANDImporter.cpp NANDImporter.h diff --git a/Source/Core/DiscIO/DiscIO.vcxproj b/Source/Core/DiscIO/DiscIO.vcxproj index 2a60bfacee42..21d1223311ae 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj +++ b/Source/Core/DiscIO/DiscIO.vcxproj @@ -55,6 +55,7 @@ + @@ -81,6 +82,7 @@ + diff --git a/Source/Core/DiscIO/DiscIO.vcxproj.filters b/Source/Core/DiscIO/DiscIO.vcxproj.filters index b7990b19d676..fb8b477649a2 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj.filters +++ b/Source/Core/DiscIO/DiscIO.vcxproj.filters @@ -93,6 +93,9 @@ Volume\Blob + + Volume\Blob + @@ -170,6 +173,9 @@ Volume\Blob + + Volume\Blob + diff --git a/Source/Core/DiscIO/LaggedFibonacciGenerator.cpp b/Source/Core/DiscIO/LaggedFibonacciGenerator.cpp new file mode 100644 index 000000000000..e5539b334d1c --- /dev/null +++ b/Source/Core/DiscIO/LaggedFibonacciGenerator.cpp @@ -0,0 +1,212 @@ +// This file is under the public domain. + +#include "DiscIO/LaggedFibonacciGenerator.h" + +#include +#include +#include + +#include "Common/Align.h" +#include "Common/Assert.h" +#include "Common/CommonTypes.h" +#include "Common/Swap.h" + +namespace DiscIO +{ +void LaggedFibonacciGenerator::SetSeed(const u32 seed[SEED_SIZE]) +{ + SetSeed(reinterpret_cast(seed)); +} + +void LaggedFibonacciGenerator::SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)]) +{ + m_position_bytes = 0; + + for (size_t i = 0; i < SEED_SIZE; ++i) + m_buffer[i] = Common::swap32(seed + i * sizeof(u32)); + + Initialize(false); +} + +size_t LaggedFibonacciGenerator::GetSeed(const u8* data, size_t size, size_t data_offset, + u32 seed_out[SEED_SIZE]) +{ + if ((reinterpret_cast(data) - data_offset) % alignof(u32) != 0) + { + ASSERT(false); + return 0; + } + + // For code simplicity, only include whole u32 words when regenerating the seed. It would be + // possible to get rid of this restriction and use a few additional bytes, but it's probably more + // effort than it's worth considering that junk data often starts or ends on 4-byte offsets. + const size_t bytes_to_skip = Common::AlignUp(data_offset, sizeof(u32)) - data_offset; + const u32* u32_data = reinterpret_cast(data + bytes_to_skip); + const size_t u32_size = (size - bytes_to_skip) / sizeof(u32); + const size_t u32_data_offset = (data_offset + bytes_to_skip) / sizeof(u32); + + LaggedFibonacciGenerator lfg; + if (!GetSeed(u32_data, u32_size, u32_data_offset, &lfg, seed_out)) + return false; + + lfg.m_position_bytes = data_offset % (LFG_K * sizeof(u32)); + + const u8* end = data + size; + size_t reconstructed_bytes = 0; + while (data < end && lfg.GetByte() == *data) + { + ++reconstructed_bytes; + ++data; + } + return reconstructed_bytes; +} + +bool LaggedFibonacciGenerator::GetSeed(const u32* data, size_t size, size_t data_offset, + LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE]) +{ + if (size < LFG_K) + return false; + + // If the data doesn't look like something we can regenerate, return early to save time + if (!std::all_of(data, data + LFG_K, [](u32 x) { + return (Common::swap32(x) & 0x00C00000) == (Common::swap32(x) >> 2 & 0x00C00000); + })) + { + return false; + } + + const size_t data_offset_mod_k = data_offset % LFG_K; + const size_t data_offset_div_k = data_offset / LFG_K; + + std::copy(data, data + LFG_K - data_offset_mod_k, lfg->m_buffer.data() + data_offset_mod_k); + std::copy(data + LFG_K - data_offset_mod_k, data + LFG_K, lfg->m_buffer.data()); + + lfg->Backward(0, data_offset_mod_k); + + for (size_t i = 0; i < data_offset_div_k; ++i) + lfg->Backward(); + + if (!lfg->Reinitialize(seed_out)) + return false; + + for (size_t i = 0; i < data_offset_div_k; ++i) + lfg->Forward(); + + return true; +} + +void LaggedFibonacciGenerator::GetBytes(size_t count, u8* out) +{ + while (count > 0) + { + const size_t length = std::min(count, LFG_K * sizeof(u32) - m_position_bytes); + + std::memcpy(out, reinterpret_cast(m_buffer.data()) + m_position_bytes, length); + + m_position_bytes += length; + count -= length; + out += length; + + if (m_position_bytes == LFG_K * sizeof(u32)) + { + Forward(); + m_position_bytes = 0; + } + } +} + +u8 LaggedFibonacciGenerator::GetByte() +{ + const u8 result = reinterpret_cast(m_buffer.data())[m_position_bytes]; + + ++m_position_bytes; + + if (m_position_bytes == LFG_K * sizeof(u32)) + { + Forward(); + m_position_bytes = 0; + } + + return result; +} + +void LaggedFibonacciGenerator::Forward(size_t count) +{ + m_position_bytes += count; + while (m_position_bytes >= LFG_K * sizeof(u32)) + { + Forward(); + m_position_bytes -= LFG_K * sizeof(u32); + } +} + +void LaggedFibonacciGenerator::Forward() +{ + for (size_t i = 0; i < LFG_J; ++i) + m_buffer[i] ^= m_buffer[i + LFG_K - LFG_J]; + + for (size_t i = LFG_J; i < LFG_K; ++i) + m_buffer[i] ^= m_buffer[i - LFG_J]; +} + +void LaggedFibonacciGenerator::Backward(size_t start_word, size_t end_word) +{ + const size_t loop_end = std::max(LFG_J, start_word); + for (size_t i = std::min(end_word, LFG_K); i > loop_end; --i) + m_buffer[i - 1] ^= m_buffer[i - 1 - LFG_J]; + + for (size_t i = std::min(end_word, LFG_J); i > start_word; --i) + m_buffer[i - 1] ^= m_buffer[i - 1 + LFG_K - LFG_J]; +} + +bool LaggedFibonacciGenerator::Reinitialize(u32 seed_out[SEED_SIZE]) +{ + for (size_t i = 0; i < 4; ++i) + Backward(); + + for (u32& x : m_buffer) + x = Common::swap32(x); + + // Reconstruct the bits which are missing due to the output code shifting by 18 instead of 16. + // Unfortunately we can't reconstruct bits 16 and 17 (counting LSB as 0) for the first word, + // but the observable result (when shifting by 18 instead of 16) is not affected by this. + for (size_t i = 0; i < SEED_SIZE; ++i) + { + m_buffer[i] = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000) | + ((m_buffer[i + 16] ^ m_buffer[i + 15]) << 9 & 0x00030000); + } + + for (size_t i = 0; i < SEED_SIZE; ++i) + seed_out[i] = Common::swap32(m_buffer[i]); + + return Initialize(true); +} + +bool LaggedFibonacciGenerator::Initialize(bool check_existing_data) +{ + for (size_t i = SEED_SIZE; i < LFG_K; ++i) + { + const u32 calculated = (m_buffer[i - 17] << 23) ^ (m_buffer[i - 16] >> 9) ^ m_buffer[i - 1]; + + if (check_existing_data) + { + const u32 actual = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000); + if ((calculated & 0xFFFCFFFF) != actual) + return false; + } + + m_buffer[i] = calculated; + } + + // Instead of doing the "shift by 18 instead of 16" oddity when actually outputting the data, + // we can do the shifting (and byteswapping) at this point to make the output code simpler. + for (u32& x : m_buffer) + x = Common::swap32((x & 0xFF00FFFF) | ((x >> 2) & 0x00FF0000)); + + for (size_t i = 0; i < 4; ++i) + Forward(); + + return true; +} + +} // namespace DiscIO diff --git a/Source/Core/DiscIO/LaggedFibonacciGenerator.h b/Source/Core/DiscIO/LaggedFibonacciGenerator.h new file mode 100644 index 000000000000..9520700d0533 --- /dev/null +++ b/Source/Core/DiscIO/LaggedFibonacciGenerator.h @@ -0,0 +1,51 @@ +// This file is under the public domain. + +#pragma once + +#include +#include + +#include "Common/CommonTypes.h" + +namespace DiscIO +{ +class LaggedFibonacciGenerator +{ +public: + static constexpr size_t SEED_SIZE = 17; + + // Reconstructs a seed and writes it to seed_out, then returns the number of bytes which can + // be reconstructed using that seed. Can return any number between 0 and size, inclusive. + // data - data_offset must be 4-byte aligned. + static size_t GetSeed(const u8* data, size_t size, size_t data_offset, u32 seed_out[SEED_SIZE]); + + // SetSeed must be called before using the functions below + void SetSeed(const u32 seed[SEED_SIZE]); + void SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)]); + + // Outputs a number of bytes and advances the internal state by the same amount. + void GetBytes(size_t count, u8* out); + u8 GetByte(); + + // Advances the internal state like GetBytes, but without outputting data. O(N), like GetBytes. + void Forward(size_t count); + +private: + static bool GetSeed(const u32* data, size_t size, size_t data_offset, + LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE]); + + void Forward(); + void Backward(size_t start_word = 0, size_t end_word = LFG_K); + + bool Reinitialize(u32 seed_out[SEED_SIZE]); + bool Initialize(bool check_existing_data); + + static constexpr size_t LFG_K = 521; + static constexpr size_t LFG_J = 32; + + std::array m_buffer; + + size_t m_position_bytes = 0; +}; + +} // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index e56794681d19..da5f5a86a17a 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -34,6 +34,7 @@ #include "DiscIO/Blob.h" #include "DiscIO/DiscExtractor.h" +#include "DiscIO/LaggedFibonacciGenerator.h" #include "DiscIO/MultithreadedCompressor.h" #include "DiscIO/Volume.h" #include "DiscIO/VolumeWii.h" @@ -192,10 +193,9 @@ bool WIAFileReader::Initialize(const std::string& path) const u32 number_of_raw_data_entries = Common::swap32(m_header_2.number_of_raw_data_entries); m_raw_data_entries.resize(number_of_raw_data_entries); - Chunk& raw_data_entries = - ReadCompressedData(Common::swap64(m_header_2.raw_data_entries_offset), - Common::swap32(m_header_2.raw_data_entries_size), - number_of_raw_data_entries * sizeof(RawDataEntry), false); + Chunk& raw_data_entries = ReadCompressedData(Common::swap64(m_header_2.raw_data_entries_offset), + Common::swap32(m_header_2.raw_data_entries_size), + number_of_raw_data_entries * sizeof(RawDataEntry)); if (!raw_data_entries.ReadAll(&m_raw_data_entries)) return false; @@ -211,7 +211,7 @@ bool WIAFileReader::Initialize(const std::string& path) m_group_entries.resize(number_of_group_entries); Chunk& group_entries = ReadCompressedData(Common::swap64(m_header_2.group_entries_offset), Common::swap32(m_header_2.group_entries_size), - number_of_group_entries * sizeof(GroupEntry), false); + number_of_group_entries * sizeof(GroupEntry)); if (!group_entries.ReadAll(&m_group_entries)) return false; @@ -444,8 +444,8 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu else { const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; - Chunk& chunk = - ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, exception_lists); + Chunk& chunk = ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, + exception_lists, m_rvz, group_offset_in_data); if (!chunk.Read(offset_in_group, bytes_to_read, *out_ptr)) { m_cached_chunk_offset = std::numeric_limits::max(); // Invalidate the cache @@ -472,7 +472,8 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu } WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 compressed_size, - u64 decompressed_size, u32 exception_lists) + u64 decompressed_size, u32 exception_lists, + bool rvz_pack, u64 data_offset) { if (offset_in_file == m_cached_chunk_offset) return m_cached_chunk; @@ -504,8 +505,9 @@ WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 const bool compressed_exception_lists = m_compression_type > WIACompressionType::Purge; - m_cached_chunk = Chunk(&m_file, offset_in_file, compressed_size, decompressed_size, - exception_lists, compressed_exception_lists, std::move(decompressor)); + m_cached_chunk = + Chunk(&m_file, offset_in_file, compressed_size, decompressed_size, exception_lists, + compressed_exception_lists, rvz_pack, data_offset, std::move(decompressor)); m_cached_chunk_offset = offset_in_file; return m_cached_chunk; } @@ -793,6 +795,135 @@ bool WIAFileReader::ZstdDecompressor::Decompress(const DecompressionBuffer& in, return !ZSTD_isError(result); } +WIAFileReader::RVZPackDecompressor::RVZPackDecompressor(std::unique_ptr decompressor, + DecompressionBuffer decompressed, + u64 data_offset) + : m_decompressor(std::move(decompressor)), m_decompressed(std::move(decompressed)), + m_data_offset(data_offset) +{ +} + +std::optional WIAFileReader::RVZPackDecompressor::ReadToDecompressed( + const DecompressionBuffer& in, size_t* in_bytes_read, size_t decompressed_bytes_read, + size_t bytes_to_read) +{ + if (m_decompressed.data.size() < decompressed_bytes_read + bytes_to_read) + m_decompressed.data.resize(decompressed_bytes_read + bytes_to_read); + + if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read) + { + if (!m_decompressor->Decompress(in, &m_decompressed, in_bytes_read)) + return false; + + if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read) + return true; + } + + return std::nullopt; +} + +bool WIAFileReader::RVZPackDecompressor::Decompress(const DecompressionBuffer& in, + DecompressionBuffer* out, size_t* in_bytes_read) +{ + while (out->data.size() != out->bytes_written && !Done()) + { + if (m_size == 0) + { + if (m_decompressed.bytes_written == m_decompressed_bytes_read) + { + m_decompressed.data.resize(sizeof(u32)); + m_decompressed.bytes_written = 0; + m_decompressed_bytes_read = 0; + } + + std::optional result = + ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read, sizeof(u32)); + if (result) + return *result; + + m_size = Common::swap32(m_decompressed.data.data() + m_decompressed_bytes_read); + + m_junk = m_size & 0x80000000; + if (m_junk) + { + m_size &= 0x7FFFFFFF; + + constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32); + + result = ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read + sizeof(u32), + SEED_SIZE); + if (result) + return *result; + + m_lfg.SetSeed(m_decompressed.data.data() + m_decompressed_bytes_read + sizeof(u32)); + m_lfg.Forward(m_data_offset % VolumeWii::BLOCK_TOTAL_SIZE); + + m_decompressed_bytes_read += SEED_SIZE; + } + + m_decompressed_bytes_read += sizeof(u32); + } + + size_t bytes_to_write = std::min(m_size, out->data.size() - out->bytes_written); + if (m_junk) + { + m_lfg.GetBytes(bytes_to_write, out->data.data() + out->bytes_written); + out->bytes_written += bytes_to_write; + } + else + { + if (m_decompressed.bytes_written != m_decompressed_bytes_read) + { + bytes_to_write = + std::min(bytes_to_write, m_decompressed.bytes_written - m_decompressed_bytes_read); + + std::memcpy(out->data.data() + out->bytes_written, + m_decompressed.data.data() + m_decompressed_bytes_read, bytes_to_write); + + m_decompressed_bytes_read += bytes_to_write; + out->bytes_written += bytes_to_write; + } + else + { + const size_t prev_out_bytes_written = out->bytes_written; + const size_t old_out_size = out->data.size(); + const size_t new_out_size = out->bytes_written + bytes_to_write; + + if (new_out_size < old_out_size) + out->data.resize(new_out_size); + + if (!m_decompressor->Decompress(in, out, in_bytes_read)) + return false; + + out->data.resize(old_out_size); + + bytes_to_write = out->bytes_written - prev_out_bytes_written; + if (bytes_to_write == 0) + return true; + } + } + + m_data_offset += bytes_to_write; + m_size -= static_cast(bytes_to_write); + } + + // If out is full but not all data has been read from in, give the decompressor a chance to read + // from in anyway. This is needed for the case where zstd has read everything except the checksum. + if (out->data.size() == out->bytes_written && in.bytes_written != *in_bytes_read) + { + if (!m_decompressor->Decompress(in, out, in_bytes_read)) + return false; + } + + return true; +} + +bool WIAFileReader::RVZPackDecompressor::Done() const +{ + return m_size == 0 && m_decompressed.bytes_written == m_decompressed_bytes_read && + m_decompressor->Done(); +} + WIAFileReader::Compressor::~Compressor() = default; WIAFileReader::PurgeCompressor::PurgeCompressor() @@ -1169,11 +1300,11 @@ WIAFileReader::Chunk::Chunk() = default; WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size, u32 exception_lists, - bool compressed_exception_lists, + bool compressed_exception_lists, bool rvz_pack, u64 data_offset, std::unique_ptr decompressor) : m_file(file), m_offset_in_file(offset_in_file), m_exception_lists(exception_lists), - m_compressed_exception_lists(compressed_exception_lists), - m_decompressor(std::move(decompressor)) + m_compressed_exception_lists(compressed_exception_lists), m_rvz_pack(rvz_pack), + m_data_offset(data_offset), m_decompressor(std::move(decompressor)) { constexpr size_t MAX_SIZE_PER_EXCEPTION_LIST = Common::AlignUp(VolumeWii::BLOCK_HEADER_SIZE, sizeof(SHA1)) / sizeof(SHA1) * @@ -1250,7 +1381,7 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) if (m_exception_lists == 0 || m_compressed_exception_lists) { - if (!m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read)) + if (!Decompress()) return false; } @@ -1261,6 +1392,12 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) { return false; } + + if (m_rvz_pack && m_exception_lists == 0) + { + if (!Decompress()) + return false; + } } if (m_exception_lists == 0) @@ -1289,6 +1426,26 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) return true; } +bool WIAFileReader::Chunk::Decompress() +{ + if (m_rvz_pack && m_exception_lists == 0) + { + m_rvz_pack = false; + + const size_t bytes_to_move = m_out.bytes_written - m_out_bytes_used_for_exceptions; + + DecompressionBuffer in{std::vector(bytes_to_move), bytes_to_move}; + std::memcpy(in.data.data(), m_out.data.data() + m_out_bytes_used_for_exceptions, bytes_to_move); + + m_out.bytes_written = m_out_bytes_used_for_exceptions; + + m_decompressor = std::make_unique(std::move(m_decompressor), std::move(in), + m_data_offset); + } + + return m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read); +} + bool WIAFileReader::Chunk::HandleExceptions(const u8* data, size_t bytes_allocated, size_t bytes_written, size_t* bytes_used, bool align) { @@ -1633,23 +1790,120 @@ static bool AllSame(const u8* begin, const u8* end) return AllAre(begin, end, *begin); }; -ConversionResult -WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, - const std::vector& partition_entries, - const std::vector& data_entries, - std::map* reusable_groups, - std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, - u64 exception_lists_per_chunk, bool compressed_exception_lists) +void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, + size_t chunks, u64 total_size, u64 data_offset, u64 in_offset, + bool allow_junk_reuse) +{ + using Seed = std::array; + struct JunkInfo + { + size_t start_offset; + Seed seed; + }; + + // Maps end_offset -> (start_offset, seed) + std::map junk_info; + + size_t position = 0; + while (position < total_size) + { + const size_t bytes_to_read = + std::min(Common::AlignUp(data_offset + 1, VolumeWii::BLOCK_TOTAL_SIZE) - data_offset, + total_size - position); + + const size_t data_offset_mod = static_cast(data_offset % VolumeWii::BLOCK_TOTAL_SIZE); + + Seed seed; + const size_t bytes_reconstructed = LaggedFibonacciGenerator::GetSeed( + in + in_offset + position, bytes_to_read, data_offset_mod, seed.data()); + + if (bytes_reconstructed > 0) + junk_info.emplace(position + bytes_reconstructed, JunkInfo{position, seed}); + + position += bytes_to_read; + data_offset += bytes_to_read; + } + + for (size_t i = 0; i < chunks; ++i) + { + OutputParametersEntry& entry = out[i]; + if (entry.reused_group) + continue; + + u64 current_offset = i * bytes_per_chunk; + const u64 end_offset = std::min(current_offset + bytes_per_chunk, total_size); + + const bool store_junk_efficiently = allow_junk_reuse || !entry.reuse_id; + + while (current_offset < end_offset) + { + constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32); + + u64 next_junk_start = end_offset; + u64 next_junk_end = end_offset; + Seed* seed = nullptr; + if (store_junk_efficiently && end_offset - current_offset > SEED_SIZE) + { + const auto next_junk_it = junk_info.upper_bound(current_offset + SEED_SIZE); + if (next_junk_it != junk_info.end() && + next_junk_it->second.start_offset + SEED_SIZE < end_offset) + { + next_junk_start = std::max(current_offset, next_junk_it->second.start_offset); + next_junk_end = std::min(end_offset, next_junk_it->first); + seed = &next_junk_it->second.seed; + } + } + + const u64 non_junk_bytes = next_junk_start - current_offset; + if (non_junk_bytes > 0) + { + const u8* ptr = in + in_offset + current_offset; + + PushBack(&entry.main_data, Common::swap32(static_cast(non_junk_bytes))); + PushBack(&entry.main_data, ptr, ptr + non_junk_bytes); + + current_offset += non_junk_bytes; + } + + const u64 junk_bytes = next_junk_end - current_offset; + if (junk_bytes > 0) + { + PushBack(&entry.main_data, Common::swap32(static_cast(junk_bytes) | 0x80000000)); + PushBack(&entry.main_data, *seed); + + current_offset += junk_bytes; + } + } + } +} + +void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset, + bool allow_junk_reuse) +{ + RVZPack(in, out, size, 1, size, data_offset, 0, allow_junk_reuse); +} + +ConversionResult WIAFileReader::ProcessAndCompress( + CompressThreadState* state, CompressParameters parameters, + const std::vector& partition_entries, + const std::vector& data_entries, std::map* reusable_groups, + std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, u64 exception_lists_per_chunk, + bool compressed_exception_lists, bool rvz) { std::vector output_entries; if (!parameters.data_entry->is_partition) { OutputParametersEntry& entry = output_entries.emplace_back(); - entry.main_data = std::move(parameters.data); + std::vector& data = parameters.data; - if (AllSame(entry.main_data)) - entry.reuse_id = ReuseID{nullptr, entry.main_data.size(), false, entry.main_data.front()}; + if (AllSame(data)) + entry.reuse_id = ReuseID{nullptr, data.size(), false, data.front()}; + + if (rvz) + RVZPack(data.data(), output_entries.data(), data.size(), parameters.data_offset, true); + else + entry.main_data = std::move(data); } else { @@ -1676,9 +1930,9 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters const size_t first_chunk = output_entries.size(); const auto create_reuse_id = [&partition_entry, blocks, - blocks_per_chunk](u8 value, bool decrypted, u64 block) { + blocks_per_chunk](u8 value, bool encrypted, u64 block) { const u64 size = std::min(blocks - block, blocks_per_chunk) * VolumeWii::BLOCK_DATA_SIZE; - return ReuseID{&partition_entry.partition_key, size, decrypted, value}; + return ReuseID{&partition_entry.partition_key, size, encrypted, value}; }; const u8* parameters_data_end = parameters.data.data() + parameters.data.size(); @@ -1692,7 +1946,7 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters // Set this chunk as reusable if the encrypted data is AllSame const u8* data = parameters.data.data() + block_index * VolumeWii::BLOCK_TOTAL_SIZE; if (AllSame(data, std::min(parameters_data_end, data + in_data_per_chunk))) - reuse_id = create_reuse_id(parameters.data.front(), false, i * blocks_per_chunk); + reuse_id = create_reuse_id(parameters.data.front(), true, i * blocks_per_chunk); TryReuse(reusable_groups, reusable_groups_mutex, &entry); if (!entry.reused_group && reuse_id) @@ -1794,41 +2048,58 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters static_assert(std::is_trivially_copyable_v); - const u8* in_ptr = state->decryption_buffer[0].data(); - for (u64 j = 0; j < chunks; ++j) + if (rvz) { - OutputParametersEntry& entry = output_entries[first_chunk + j]; + // We must not store junk efficiently for chunks that may get reused at a position + // which has a different value of data_offset % VolumeWii::BLOCK_TOTAL_SIZE + const bool allow_junk_reuse = chunks_per_wii_group == 1; + + const u64 bytes_per_chunk = std::min(out_data_per_chunk, VolumeWii::GROUP_DATA_SIZE); + const u64 total_size = blocks_in_this_group * VolumeWii::BLOCK_DATA_SIZE; + const u64 data_offset = parameters.data_offset + write_offset_of_group; - if (!entry.reused_group) + RVZPack(state->decryption_buffer[0].data(), output_entries.data() + first_chunk, + bytes_per_chunk, chunks, total_size, data_offset, write_offset_of_group, + allow_junk_reuse); + } + else + { + const u8* in_ptr = state->decryption_buffer[0].data(); + for (u64 j = 0; j < chunks; ++j) { - const u64 bytes_left = (blocks - j * blocks_per_chunk) * VolumeWii::BLOCK_DATA_SIZE; - const u64 bytes_to_write_total = std::min(out_data_per_chunk, bytes_left); + OutputParametersEntry& entry = output_entries[first_chunk + j]; - if (i == 0) - entry.main_data.resize(bytes_to_write_total); + if (!entry.reused_group) + { + const u64 bytes_left = (blocks - j * blocks_per_chunk) * VolumeWii::BLOCK_DATA_SIZE; + const u64 bytes_to_write_total = std::min(out_data_per_chunk, bytes_left); - const u64 bytes_to_write = std::min(bytes_to_write_total, VolumeWii::GROUP_DATA_SIZE); + if (i == 0) + entry.main_data.resize(bytes_to_write_total); - std::memcpy(entry.main_data.data() + write_offset_of_group, in_ptr, bytes_to_write); + const u64 bytes_to_write = std::min(bytes_to_write_total, VolumeWii::GROUP_DATA_SIZE); - // Set this chunk as reusable if the decrypted data is AllSame. - // There is also a requirement that it lacks exceptions, but this is checked later - if (i == 0 && !entry.reuse_id) - { - if (AllSame(in_ptr, in_ptr + bytes_to_write)) - entry.reuse_id = create_reuse_id(*in_ptr, true, j * blocks_per_chunk); - } - else - { - if (entry.reuse_id && entry.reuse_id->decrypted && - (!AllSame(in_ptr, in_ptr + bytes_to_write) || entry.reuse_id->value != *in_ptr)) + std::memcpy(entry.main_data.data() + write_offset_of_group, in_ptr, bytes_to_write); + + // Set this chunk as reusable if the decrypted data is AllSame. + // There is also a requirement that it lacks exceptions, but this is checked later + if (i == 0 && !entry.reuse_id) + { + if (AllSame(in_ptr, in_ptr + bytes_to_write)) + entry.reuse_id = create_reuse_id(*in_ptr, false, j * blocks_per_chunk); + } + else { - entry.reuse_id.reset(); + if (entry.reuse_id && !entry.reuse_id->encrypted && + (!AllSame(in_ptr, in_ptr + bytes_to_write) || entry.reuse_id->value != *in_ptr)) + { + entry.reuse_id.reset(); + } } } - } - in_ptr += out_data_per_chunk; + in_ptr += out_data_per_chunk; + } } } @@ -1853,7 +2124,7 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters // If this chunk was set as reusable because the decrypted data is AllSame, // but it has exceptions, unmark it as reusable - if (entry.reuse_id && entry.reuse_id->decrypted && !AllZero(entry.exception_lists)) + if (entry.reuse_id && !entry.reuse_id->encrypted && !AllZero(entry.exception_lists)) entry.reuse_id.reset(); } } @@ -1866,7 +2137,7 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters continue; // Special case - a compressed size of zero is treated by WIA as meaning the data is all zeroes - if (AllZero(entry.exception_lists) && AllZero(entry.main_data)) + if (entry.reuse_id && !entry.reuse_id->encrypted && entry.reuse_id->value == 0) { entry.exception_lists.clear(); entry.main_data.clear(); @@ -2084,7 +2355,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, const auto process_and_compress = [&](CompressThreadState* state, CompressParameters parameters) { return ProcessAndCompress(state, std::move(parameters), partition_entries, data_entries, &reusable_groups, &reusable_groups_mutex, chunks_per_wii_group, - exception_lists_per_chunk, compressed_exception_lists); + exception_lists_per_chunk, compressed_exception_lists, rvz); }; const auto output = [&](OutputParameters parameters) { @@ -2110,6 +2381,8 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, u64 data_offset; u64 data_size; + u64 data_offset_in_partition; + if (data_entry.is_partition) { const PartitionEntry& partition_entry = partition_entries[data_entry.index]; @@ -2119,9 +2392,14 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, first_group = Common::swap32(partition_data_entry.group_index); last_group = first_group + Common::swap32(partition_data_entry.number_of_groups); - data_offset = Common::swap32(partition_data_entry.first_sector) * VolumeWii::BLOCK_TOTAL_SIZE; + const u32 first_sector = Common::swap32(partition_data_entry.first_sector); + data_offset = first_sector * VolumeWii::BLOCK_TOTAL_SIZE; data_size = Common::swap32(partition_data_entry.number_of_sectors) * VolumeWii::BLOCK_TOTAL_SIZE; + + const u32 block_in_partition = + first_sector - Common::swap32(partition_entry.data_entries[0].first_sector); + data_offset_in_partition = block_in_partition * VolumeWii::BLOCK_DATA_SIZE; } else { @@ -2136,6 +2414,8 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, const u64 skipped_data = data_offset % VolumeWii::BLOCK_TOTAL_SIZE; data_offset -= skipped_data; data_size += skipped_data; + + data_offset_in_partition = data_offset; } ASSERT(groups_processed == first_group); @@ -2157,11 +2437,26 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, return ConversionResultCode::ReadFailed; bytes_read += bytes_to_read; - mt_compressor.CompressAndWrite( - CompressParameters{buffer, &data_entry, bytes_read, groups_processed}); + mt_compressor.CompressAndWrite(CompressParameters{ + buffer, &data_entry, data_offset_in_partition, bytes_read, groups_processed}); + + data_offset += bytes_to_read; + data_size -= bytes_to_read; + + if (data_entry.is_partition) + { + data_offset_in_partition += + bytes_to_read / VolumeWii::BLOCK_TOTAL_SIZE * VolumeWii::BLOCK_DATA_SIZE; + } + else + { + data_offset_in_partition += bytes_to_read; + } groups_processed += Common::AlignUp(bytes_to_read, chunk_size) / chunk_size; } + + ASSERT(data_size == 0); } ASSERT(groups_processed == total_groups); diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 6b2af6f904ad..35a431346037 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -21,6 +22,7 @@ #include "Common/File.h" #include "Common/Swap.h" #include "DiscIO/Blob.h" +#include "DiscIO/LaggedFibonacciGenerator.h" #include "DiscIO/MultithreadedCompressor.h" #include "DiscIO/WiiEncryptionCache.h" @@ -267,6 +269,31 @@ class WIAFileReader : public BlobReader ZSTD_DStream* m_stream; }; + class RVZPackDecompressor final : public Decompressor + { + public: + RVZPackDecompressor(std::unique_ptr decompressor, + DecompressionBuffer decompressed, u64 data_offset); + + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; + + bool Done() const override; + + private: + std::optional ReadToDecompressed(const DecompressionBuffer& in, size_t* in_bytes_read, + size_t decompressed_bytes_read, size_t bytes_to_read); + + std::unique_ptr m_decompressor; + DecompressionBuffer m_decompressed; + size_t m_decompressed_bytes_read = 0; + u64 m_data_offset; + + u32 m_size = 0; + bool m_junk; + LaggedFibonacciGenerator m_lfg; + }; + class Compressor { public: @@ -375,7 +402,7 @@ class WIAFileReader : public BlobReader public: Chunk(); Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size, - u32 exception_lists, bool compressed_exception_lists, + u32 exception_lists, bool compressed_exception_lists, bool rvz_pack, u64 data_offset, std::unique_ptr decompressor); bool Read(u64 offset, u64 size, u8* out_ptr); @@ -391,6 +418,7 @@ class WIAFileReader : public BlobReader } private: + bool Decompress(); bool HandleExceptions(const u8* data, size_t bytes_allocated, size_t bytes_written, size_t* bytes_used, bool align); @@ -407,6 +435,8 @@ class WIAFileReader : public BlobReader size_t m_in_bytes_used_for_exceptions = 0; u32 m_exception_lists = 0; bool m_compressed_exception_lists = false; + bool m_rvz_pack = false; + u64 m_data_offset = 0; }; explicit WIAFileReader(File::IOFile file, const std::string& path); @@ -417,7 +447,7 @@ class WIAFileReader : public BlobReader u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups, u32 exception_lists); Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size, - u32 exception_lists); + u32 exception_lists = 0, bool rvz_pack = false, u64 data_offset = 0); static bool ApplyHashExceptions(const std::vector& exception_list, VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]); @@ -430,18 +460,18 @@ class WIAFileReader : public BlobReader { bool operator==(const ReuseID& other) const { - return std::tie(partition_key, data_size, decrypted, value) == - std::tie(other.partition_key, other.data_size, other.decrypted, other.value); + return std::tie(partition_key, data_size, encrypted, value) == + std::tie(other.partition_key, other.data_size, other.encrypted, other.value); } bool operator<(const ReuseID& other) const { - return std::tie(partition_key, data_size, decrypted, value) < - std::tie(other.partition_key, other.data_size, other.decrypted, other.value); + return std::tie(partition_key, data_size, encrypted, value) < + std::tie(other.partition_key, other.data_size, other.encrypted, other.value); } bool operator>(const ReuseID& other) const { - return std::tie(partition_key, data_size, decrypted, value) > - std::tie(other.partition_key, other.data_size, other.decrypted, other.value); + return std::tie(partition_key, data_size, encrypted, value) > + std::tie(other.partition_key, other.data_size, other.encrypted, other.value); } bool operator!=(const ReuseID& other) const { return !operator==(other); } bool operator>=(const ReuseID& other) const { return !operator<(other); } @@ -449,7 +479,7 @@ class WIAFileReader : public BlobReader const WiiKey* partition_key; u64 data_size; - bool decrypted; + bool encrypted; u8 value; }; @@ -470,6 +500,7 @@ class WIAFileReader : public BlobReader { std::vector data; const DataEntry* data_entry; + u64 data_offset; u64 bytes_read; size_t group_index; }; @@ -512,13 +543,17 @@ class WIAFileReader : public BlobReader WIAHeader2* header_2); static bool TryReuse(std::map* reusable_groups, std::mutex* reusable_groups_mutex, OutputParametersEntry* entry); + static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, size_t chunks, + u64 total_size, u64 data_offset, u64 in_offset, bool allow_junk_reuse); + static void RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset, + bool allow_junk_reuse); static ConversionResult ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, const std::vector& partition_entries, const std::vector& data_entries, std::map* reusable_groups, std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, - u64 exception_lists_per_chunk, bool compressed_exception_lists); + u64 exception_lists_per_chunk, bool compressed_exception_lists, bool rvz); static ConversionResultCode Output(std::vector* entries, File::IOFile* outfile, std::map* reusable_groups, @@ -528,13 +563,20 @@ class WIAFileReader : public BlobReader u32 total_groups, u64 iso_size, CompressCB callback, void* arg); + static void PushBack(std::vector* vector, const u8* begin, const u8* end) + { + const size_t offset_in_vector = vector->size(); + vector->resize(offset_in_vector + (end - begin)); + std::copy(begin, end, vector->data() + offset_in_vector); + } + template static void PushBack(std::vector* vector, const T& x) { - const size_t offset_in_vector = vector->size(); - vector->resize(offset_in_vector + sizeof(T)); + static_assert(std::is_trivially_copyable_v); + const u8* x_ptr = reinterpret_cast(&x); - std::copy(x_ptr, x_ptr + sizeof(T), vector->data() + offset_in_vector); + PushBack(vector, x_ptr, x_ptr + sizeof(T)); } bool m_valid; @@ -566,9 +608,9 @@ class WIAFileReader : public BlobReader static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000; static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000; - static constexpr u32 RVZ_VERSION = 0x00010000; - static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00010000; - static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00010000; + static constexpr u32 RVZ_VERSION = 0x00020000; + static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00020000; + static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00020000; }; } // namespace DiscIO diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index 1883918d9560..7639c78b8a51 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -252,6 +252,10 @@ void ConvertDialog::OnFormatChanged() m_block_size->setEnabled(m_block_size->count() > 1); m_compression->setEnabled(m_compression->count() > 1); + + m_scrub->setEnabled(format != DiscIO::BlobType::RVZ); + if (format == DiscIO::BlobType::RVZ) + m_scrub->setChecked(false); } void ConvertDialog::OnCompressionChanged() From 3f753fc87d456c482e847c7de99c7ccd3b758f08 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 13 May 2020 15:35:36 +0200 Subject: [PATCH 30/36] RVZ: Detect junk data in the same block as a file --- Source/Core/DiscIO/WIABlob.cpp | 85 +++++++++++++++++++++++++++------- Source/Core/DiscIO/WIABlob.h | 20 ++++---- 2 files changed, 80 insertions(+), 25 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index da5f5a86a17a..8bf207f1cd11 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -34,6 +34,7 @@ #include "DiscIO/Blob.h" #include "DiscIO/DiscExtractor.h" +#include "DiscIO/Filesystem.h" #include "DiscIO/LaggedFibonacciGenerator.h" #include "DiscIO/MultithreadedCompressor.h" #include "DiscIO/Volume.h" @@ -1581,7 +1582,7 @@ WIAFileReader::PartitionDataEntry WIAFileReader::CreatePartitionDataEntry( ConversionResultCode WIAFileReader::SetUpDataEntriesForWriting( const VolumeDisc* volume, int chunk_size, u64 iso_size, u32* total_groups, std::vector* partition_entries, std::vector* raw_data_entries, - std::vector* data_entries) + std::vector* data_entries, std::vector* partition_file_systems) { std::vector partitions; if (volume && volume->IsEncryptedAndHashed()) @@ -1687,6 +1688,7 @@ ConversionResultCode WIAFileReader::SetUpDataEntriesForWriting( VolumeWii::BLOCK_TOTAL_SIZE; partition_entries->emplace_back(std::move(partition_entry)); + partition_file_systems->emplace_back(volume->GetFileSystem(partition)); } add_raw_data_entry(last_partition_end_offset, iso_size - last_partition_end_offset); @@ -1792,7 +1794,7 @@ static bool AllSame(const u8* begin, const u8* end) void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, size_t chunks, u64 total_size, u64 data_offset, u64 in_offset, - bool allow_junk_reuse) + bool allow_junk_reuse, bool compression, const FileSystem* file_system) { using Seed = std::array; struct JunkInfo @@ -1801,12 +1803,28 @@ void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_ Seed seed; }; + constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32); + // Maps end_offset -> (start_offset, seed) std::map junk_info; size_t position = 0; while (position < total_size) { + // Skip the 0 to 32 zero bytes that typically come after a file + size_t zeroes = 0; + while (position + zeroes < total_size && in[in_offset + position + zeroes] == 0) + ++zeroes; + + // If there are very many zero bytes (perhaps the PRNG junk data has been scrubbed?) + // and we aren't using compression, it makes sense to encode the zero bytes as junk. + // If we are using compression, the compressor will likely encode zeroes better than we can + if (!compression && zeroes > SEED_SIZE) + junk_info.emplace(position + zeroes, JunkInfo{position, {}}); + + position += zeroes; + data_offset += zeroes; + const size_t bytes_to_read = std::min(Common::AlignUp(data_offset + 1, VolumeWii::BLOCK_TOTAL_SIZE) - data_offset, total_size - position); @@ -1820,6 +1838,25 @@ void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_ if (bytes_reconstructed > 0) junk_info.emplace(position + bytes_reconstructed, JunkInfo{position, seed}); + if (file_system) + { + const std::unique_ptr file_info = + file_system->FindFileInfo(data_offset + bytes_reconstructed); + + // If we're at a file and there's more space in this block after the file, + // continue after the file instead of skipping to the next block + if (file_info) + { + const u64 file_end_offset = file_info->GetOffset() + file_info->GetSize(); + if (file_end_offset < data_offset + bytes_to_read) + { + position += file_end_offset - data_offset; + data_offset = file_end_offset; + continue; + } + } + } + position += bytes_to_read; data_offset += bytes_to_read; } @@ -1837,8 +1874,6 @@ void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_ while (current_offset < end_offset) { - constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32); - u64 next_junk_start = end_offset; u64 next_junk_end = end_offset; Seed* seed = nullptr; @@ -1878,17 +1913,18 @@ void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_ } void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset, - bool allow_junk_reuse) + bool allow_junk_reuse, bool compression, const FileSystem* file_system) { - RVZPack(in, out, size, 1, size, data_offset, 0, allow_junk_reuse); + RVZPack(in, out, size, 1, size, data_offset, 0, allow_junk_reuse, compression, file_system); } ConversionResult WIAFileReader::ProcessAndCompress( CompressThreadState* state, CompressParameters parameters, const std::vector& partition_entries, - const std::vector& data_entries, std::map* reusable_groups, - std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, u64 exception_lists_per_chunk, - bool compressed_exception_lists, bool rvz) + const std::vector& data_entries, const FileSystem* file_system, + std::map* reusable_groups, std::mutex* reusable_groups_mutex, + u64 chunks_per_wii_group, u64 exception_lists_per_chunk, bool compressed_exception_lists, + bool compression, bool rvz) { std::vector output_entries; @@ -1901,9 +1937,14 @@ ConversionResult WIAFileReader::ProcessAndCompr entry.reuse_id = ReuseID{nullptr, data.size(), false, data.front()}; if (rvz) - RVZPack(data.data(), output_entries.data(), data.size(), parameters.data_offset, true); + { + RVZPack(data.data(), output_entries.data(), data.size(), parameters.data_offset, true, + compression, file_system); + } else + { entry.main_data = std::move(data); + } } else { @@ -2060,7 +2101,7 @@ ConversionResult WIAFileReader::ProcessAndCompr RVZPack(state->decryption_buffer[0].data(), output_entries.data() + first_chunk, bytes_per_chunk, chunks, total_size, data_offset, write_offset_of_group, - allow_junk_reuse); + allow_junk_reuse, compression, file_system); } else { @@ -2314,9 +2355,13 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, u32 total_groups; std::vector data_entries; - const ConversionResultCode set_up_data_entries_result = - SetUpDataEntriesForWriting(infile_volume, chunk_size, iso_size, &total_groups, - &partition_entries, &raw_data_entries, &data_entries); + const FileSystem* non_partition_file_system = + infile_volume ? infile_volume->GetFileSystem(PARTITION_NONE) : nullptr; + std::vector partition_file_systems; + + const ConversionResultCode set_up_data_entries_result = SetUpDataEntriesForWriting( + infile_volume, chunk_size, iso_size, &total_groups, &partition_entries, &raw_data_entries, + &data_entries, &partition_file_systems); if (set_up_data_entries_result != ConversionResultCode::Success) return set_up_data_entries_result; @@ -2353,9 +2398,17 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, }; const auto process_and_compress = [&](CompressThreadState* state, CompressParameters parameters) { + const DataEntry& data_entry = *parameters.data_entry; + const FileSystem* file_system = data_entry.is_partition ? + partition_file_systems[data_entry.index] : + non_partition_file_system; + + const bool compression = compression_type != WIACompressionType::None; + return ProcessAndCompress(state, std::move(parameters), partition_entries, data_entries, - &reusable_groups, &reusable_groups_mutex, chunks_per_wii_group, - exception_lists_per_chunk, compressed_exception_lists, rvz); + file_system, &reusable_groups, &reusable_groups_mutex, + chunks_per_wii_group, exception_lists_per_chunk, + compressed_exception_lists, compression, rvz); }; const auto output = [&](OutputParameters parameters) { diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 35a431346037..78562a3fc43d 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -28,6 +28,7 @@ namespace DiscIO { +class FileSystem; class VolumeDisc; enum class WIACompressionType : u32 @@ -528,11 +529,10 @@ class WIAFileReader : public BlobReader CreatePartitionDataEntry(u64 offset, u64 size, u32 index, int chunk_size, u32* total_groups, const std::vector& partition_entries, std::vector* data_entries); - static ConversionResultCode - SetUpDataEntriesForWriting(const VolumeDisc* volume, int chunk_size, u64 iso_size, - u32* total_groups, std::vector* partition_entries, - std::vector* raw_data_entries, - std::vector* data_entries); + static ConversionResultCode SetUpDataEntriesForWriting( + const VolumeDisc* volume, int chunk_size, u64 iso_size, u32* total_groups, + std::vector* partition_entries, std::vector* raw_data_entries, + std::vector* data_entries, std::vector* partition_file_systems); static std::optional> Compress(Compressor* compressor, const u8* data, size_t size); static bool WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound, @@ -544,16 +544,18 @@ class WIAFileReader : public BlobReader static bool TryReuse(std::map* reusable_groups, std::mutex* reusable_groups_mutex, OutputParametersEntry* entry); static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, size_t chunks, - u64 total_size, u64 data_offset, u64 in_offset, bool allow_junk_reuse); + u64 total_size, u64 data_offset, u64 in_offset, bool allow_junk_reuse, + bool compression, const FileSystem* file_system); static void RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset, - bool allow_junk_reuse); + bool allow_junk_reuse, bool compression, const FileSystem* file_system); static ConversionResult ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, const std::vector& partition_entries, - const std::vector& data_entries, + const std::vector& data_entries, const FileSystem* file_system, std::map* reusable_groups, std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, - u64 exception_lists_per_chunk, bool compressed_exception_lists, bool rvz); + u64 exception_lists_per_chunk, bool compressed_exception_lists, + bool compression, bool rvz); static ConversionResultCode Output(std::vector* entries, File::IOFile* outfile, std::map* reusable_groups, From 2ec608f0598925c874441ac2bcdfe4af2296bf7c Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 13 May 2020 16:43:59 +0200 Subject: [PATCH 31/36] DolphinQt: Set block size to 128 KiB by default --- Source/Core/DolphinQt/ConvertDialog.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index 7639c78b8a51..a587af0debd2 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -131,6 +131,12 @@ ConvertDialog::ConvertDialog(QList> fi void ConvertDialog::AddToBlockSizeComboBox(int size) { m_block_size->addItem(QString::fromStdString(UICommon::FormatSize(size, 0)), size); + + // Select 128 KiB by default, or if it is not available, the size closest to it. + // This code assumes that sizes get added to the combo box in increasing order. + constexpr int DEFAULT_SIZE = 0x20000; + if (size <= DEFAULT_SIZE) + m_block_size->setCurrentIndex(m_block_size->count() - 1); } void ConvertDialog::AddToCompressionComboBox(const QString& name, DiscIO::WIACompressionType type) From 39caac925a5196861b81fa6e68171e97ad80ec95 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 13 May 2020 19:38:31 +0200 Subject: [PATCH 32/36] RVZ: Add documentation --- docs/WIA.md | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/docs/WIA.md b/docs/WIA.md index 894078cd52ca..16688d5bddea 100644 --- a/docs/WIA.md +++ b/docs/WIA.md @@ -168,3 +168,67 @@ This struct is used by the simple compression method PURGE, which stores runs of |`u8 data[size]`|Data.| Each PURGE chunk contains zero or more `wia_segment_t` structs stored in order of ascending `offset`, followed by a SHA-1 hash (0x14 bytes) of the `wia_except_list_t` structs (if any) and the `wia_segment_t` structs. Bytes in the decompressed data that are not covered by any `wia_segment_t` struct are set to `0x00`. + +# RVZ file format description + +RVZ is a file format which is closely based on WIA. The differences are as follows: + +* Zstandard has been added as a compression method. `compression` in `wia_disc_t` is set to 5 when Zstandard is used, and there is no compressor specific data. `compr_level` in `wia_disc_t` should be treated as signed instead of unsigned because Zstandard supports negative compression levels. +* PURGE has been removed as a compression method. +* Chunk sizes smaller than 2 MiB are supported. The following applies when using a chunk size smaller than 2 MiB: + * The chunk size must be at least 32 KiB and must be a power of two. (Just like with WIA, sizes larger than 2 MiB do not have to be a power of two, they just have to be an integer multiple of 2 MiB.) + * For Wii partition data, each chunk contains one `wia_except_list_t` which contains exceptions for that chunk (and no other chunks). Offset 0 refers to the first hash of the current chunk, not the first hash of the full 2 MiB of data. +* An encoding scheme which is described below is used to store pseudorandom padding data losslessly. + +## RVZ packing + +The RVZ packing encoding scheme is applied to all `wia_group_t` data, with any bzip2/LZMA/Zstandard compression being applied on top of it. (In other words, when reading an RVZ file, bzip2/LZMA/Zstandard decompression is done before decoding the RVZ packing.) RVZ packed data can be decoded as follows: + +1. Read 4 bytes of data and interpret it as a 32-bit unsigned big endian integer. Call this `size`. +2. If the most significant bit of `size` is not set, read `size` bytes and output them unchanged. If the most significant bit of `size` is set, unset the most significant bit of `size`, then read 68 bytes of PRNG seed data and output `size` bytes using the PRNG algorithm described below. +3. Repeat until all input has been read. + +### PRNG algorithm + +The PRNG algorithm used for generating padding data on GameCube and Wii discs is a Lagged Fibonacci generator with the parameters f = xor, j = 32, k = 521. + +Start by allocating a buffer of 521 32-bit words. + +``` +u32 buffer[521]; +``` + +Copy the 68 bytes (17 words) of seed data into the start of the buffer. This seed data is stored in big endian in RVZ files, so remember to byteswap each word if the system is not big endian. Then, use the following code to fill the remaining part of the buffer: + +``` +for (size_t i = 17; i < 521; i++) + buffer[i] = (buffer[i - 17] << 23) ^ (buffer[i - 16] >> 9) ^ buffer[i - 1]; +``` + +The following code is used for advancing the state of the PRNG by a full buffer length. You must run it 4 times before you can start outputting data, and must then run it once after every 521 words of data you output. + +``` +for (size_t i = 0; i < 32; i++) + buffer[i] ^= buffer[i + 521 - 32]; + +for (size_t i = 32; i < 521; i++) + buffer[i] ^= buffer[i - 32]; +``` + +After running the above code 4 times, you are ready to output data from the buffer -- but only if the offset (relative to the start of the disc for `wia_raw_data_t` and relative to the start of the partition data for `wia_part_t`) at which you are outputting data is evenly divisible by 32 KiB. Otherwise, you first have to advance the state of the PRNG by `offset % 0x8000` bytes. Please note that the hashes are not counted in the offset for `wia_part_t`, yet the number is still 32 KiB and not 31 KiB. + +To finally output a word of data from the buffer, use the following code: + +``` +u8* out; +u32* buffer_ptr; + +/* ... */ + +*(out++) = *buffer_ptr >> 24; +*(out++) = *buffer_ptr >> 18; // NB: 18, not 16 +*(out++) = *buffer_ptr >> 8; +*(out++) = *buffer_ptr; + +buffer_ptr++; +``` From ca4e4a62078a4aa7514b5b0fba252292ef0a4082 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 17 May 2020 15:21:34 +0200 Subject: [PATCH 33/36] WIA/RVZ: Move (de)compression to a different file --- Source/Core/DiscIO/CMakeLists.txt | 2 + Source/Core/DiscIO/DiscIO.vcxproj | 2 + Source/Core/DiscIO/DiscIO.vcxproj.filters | 6 + Source/Core/DiscIO/WIABlob.cpp | 774 +-------------------- Source/Core/DiscIO/WIABlob.h | 235 +------ Source/Core/DiscIO/WIACompression.cpp | 793 ++++++++++++++++++++++ Source/Core/DiscIO/WIACompression.h | 249 +++++++ 7 files changed, 1054 insertions(+), 1007 deletions(-) create mode 100644 Source/Core/DiscIO/WIACompression.cpp create mode 100644 Source/Core/DiscIO/WIACompression.h diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index f0deb14ca203..4c0f04f78ae4 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -46,6 +46,8 @@ add_library(discio WbfsBlob.h WIABlob.cpp WIABlob.h + WIACompression.cpp + WIACompression.h WiiEncryptionCache.cpp WiiEncryptionCache.h WiiSaveBanner.cpp diff --git a/Source/Core/DiscIO/DiscIO.vcxproj b/Source/Core/DiscIO/DiscIO.vcxproj index 21d1223311ae..10f870fe6902 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj +++ b/Source/Core/DiscIO/DiscIO.vcxproj @@ -67,6 +67,7 @@ + @@ -95,6 +96,7 @@ + diff --git a/Source/Core/DiscIO/DiscIO.vcxproj.filters b/Source/Core/DiscIO/DiscIO.vcxproj.filters index fb8b477649a2..fd67a4487f30 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj.filters +++ b/Source/Core/DiscIO/DiscIO.vcxproj.filters @@ -96,6 +96,9 @@ Volume\Blob + + Volume\Blob + @@ -176,6 +179,9 @@ Volume\Blob + + Volume\Blob + diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 8bf207f1cd11..48731d69c120 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -16,8 +16,6 @@ #include #include -#include -#include #include #include @@ -39,6 +37,7 @@ #include "DiscIO/MultithreadedCompressor.h" #include "DiscIO/Volume.h" #include "DiscIO/VolumeWii.h" +#include "DiscIO/WIACompression.h" #include "DiscIO/WiiEncryptionCache.h" namespace DiscIO @@ -526,777 +525,6 @@ std::string WIAFileReader::VersionToString(u32 version) return StringFromFormat("%u.%02x.%02x.beta%u", a, b, c, d); } -u32 WIAFileReader::LZMA2DictionarySize(u8 p) -{ - return (static_cast(2) | (p & 1)) << (p / 2 + 11); -} - -WIAFileReader::Decompressor::~Decompressor() = default; - -bool WIAFileReader::NoneDecompressor::Decompress(const DecompressionBuffer& in, - DecompressionBuffer* out, size_t* in_bytes_read) -{ - const size_t length = - std::min(in.bytes_written - *in_bytes_read, out->data.size() - out->bytes_written); - - std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read, length); - - *in_bytes_read += length; - out->bytes_written += length; - - m_done = in.data.size() == *in_bytes_read; - return true; -} - -WIAFileReader::PurgeDecompressor::PurgeDecompressor(u64 decompressed_size) - : m_decompressed_size(decompressed_size) -{ - mbedtls_sha1_init(&m_sha1_context); -} - -bool WIAFileReader::PurgeDecompressor::Decompress(const DecompressionBuffer& in, - DecompressionBuffer* out, size_t* in_bytes_read) -{ - if (!m_started) - { - mbedtls_sha1_starts_ret(&m_sha1_context); - - // Include the exception lists in the SHA-1 calculation (but not in the compression...) - mbedtls_sha1_update_ret(&m_sha1_context, in.data.data(), *in_bytes_read); - - m_started = true; - } - - while (!m_done && in.bytes_written != *in_bytes_read && - (m_segment_bytes_written < sizeof(m_segment) || out->data.size() != out->bytes_written)) - { - if (m_segment_bytes_written == 0 && *in_bytes_read == in.data.size() - sizeof(SHA1)) - { - const size_t zeroes_to_write = std::min(m_decompressed_size - m_out_bytes_written, - out->data.size() - out->bytes_written); - - std::memset(out->data.data() + out->bytes_written, 0, zeroes_to_write); - - out->bytes_written += zeroes_to_write; - m_out_bytes_written += zeroes_to_write; - - if (m_out_bytes_written == m_decompressed_size && in.bytes_written == in.data.size()) - { - SHA1 actual_hash; - mbedtls_sha1_finish_ret(&m_sha1_context, actual_hash.data()); - - SHA1 expected_hash; - std::memcpy(expected_hash.data(), in.data.data() + *in_bytes_read, expected_hash.size()); - - *in_bytes_read += expected_hash.size(); - m_done = true; - - if (actual_hash != expected_hash) - return false; - } - - return true; - } - - if (m_segment_bytes_written < sizeof(m_segment)) - { - const size_t bytes_to_copy = - std::min(in.bytes_written - *in_bytes_read, sizeof(m_segment) - m_segment_bytes_written); - - std::memcpy(reinterpret_cast(&m_segment) + m_segment_bytes_written, - in.data.data() + *in_bytes_read, bytes_to_copy); - mbedtls_sha1_update_ret(&m_sha1_context, in.data.data() + *in_bytes_read, bytes_to_copy); - - *in_bytes_read += bytes_to_copy; - m_bytes_read += bytes_to_copy; - m_segment_bytes_written += bytes_to_copy; - } - - if (m_segment_bytes_written < sizeof(m_segment)) - return true; - - const size_t offset = Common::swap32(m_segment.offset); - const size_t size = Common::swap32(m_segment.size); - - if (m_out_bytes_written < offset) - { - const size_t zeroes_to_write = - std::min(offset - m_out_bytes_written, out->data.size() - out->bytes_written); - - std::memset(out->data.data() + out->bytes_written, 0, zeroes_to_write); - - out->bytes_written += zeroes_to_write; - m_out_bytes_written += zeroes_to_write; - } - - if (m_out_bytes_written >= offset && m_out_bytes_written < offset + size) - { - const size_t bytes_to_copy = std::min( - std::min(offset + size - m_out_bytes_written, out->data.size() - out->bytes_written), - in.bytes_written - *in_bytes_read); - - std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read, - bytes_to_copy); - mbedtls_sha1_update_ret(&m_sha1_context, in.data.data() + *in_bytes_read, bytes_to_copy); - - *in_bytes_read += bytes_to_copy; - m_bytes_read += bytes_to_copy; - out->bytes_written += bytes_to_copy; - m_out_bytes_written += bytes_to_copy; - } - - if (m_out_bytes_written >= offset + size) - m_segment_bytes_written = 0; - } - - return true; -} - -WIAFileReader::Bzip2Decompressor::~Bzip2Decompressor() -{ - if (m_started) - BZ2_bzDecompressEnd(&m_stream); -} - -bool WIAFileReader::Bzip2Decompressor::Decompress(const DecompressionBuffer& in, - DecompressionBuffer* out, size_t* in_bytes_read) -{ - if (!m_started) - { - if (BZ2_bzDecompressInit(&m_stream, 0, 0) != BZ_OK) - return false; - - m_started = true; - } - - constexpr auto clamped_cast = [](size_t x) { - return static_cast( - std::min(std::numeric_limits().max(), x)); - }; - - char* const in_ptr = reinterpret_cast(const_cast(in.data.data() + *in_bytes_read)); - m_stream.next_in = in_ptr; - m_stream.avail_in = clamped_cast(in.bytes_written - *in_bytes_read); - - char* const out_ptr = reinterpret_cast(out->data.data() + out->bytes_written); - m_stream.next_out = out_ptr; - m_stream.avail_out = clamped_cast(out->data.size() - out->bytes_written); - - const int result = BZ2_bzDecompress(&m_stream); - - *in_bytes_read += m_stream.next_in - in_ptr; - out->bytes_written += m_stream.next_out - out_ptr; - - m_done = result == BZ_STREAM_END; - return result == BZ_OK || result == BZ_STREAM_END; -} - -WIAFileReader::LZMADecompressor::LZMADecompressor(bool lzma2, const u8* filter_options, - size_t filter_options_size) -{ - m_options.preset_dict = nullptr; - - if (!lzma2 && filter_options_size == 5) - { - // The dictionary size is stored as a 32-bit little endian unsigned integer - static_assert(sizeof(m_options.dict_size) == sizeof(u32)); - std::memcpy(&m_options.dict_size, filter_options + 1, sizeof(u32)); - - const u8 d = filter_options[0]; - if (d >= (9 * 5 * 5)) - { - m_error_occurred = true; - } - else - { - m_options.lc = d % 9; - const u8 e = d / 9; - m_options.pb = e / 5; - m_options.lp = e % 5; - } - } - else if (lzma2 && filter_options_size == 1) - { - const u8 d = filter_options[0]; - if (d > 40) - m_error_occurred = true; - else - m_options.dict_size = d == 40 ? 0xFFFFFFFF : LZMA2DictionarySize(d); - } - else - { - m_error_occurred = true; - } - - m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1; - m_filters[0].options = &m_options; - m_filters[1].id = LZMA_VLI_UNKNOWN; - m_filters[1].options = nullptr; -} - -WIAFileReader::LZMADecompressor::~LZMADecompressor() -{ - if (m_started) - lzma_end(&m_stream); -} - -bool WIAFileReader::LZMADecompressor::Decompress(const DecompressionBuffer& in, - DecompressionBuffer* out, size_t* in_bytes_read) -{ - if (!m_started) - { - if (m_error_occurred || lzma_raw_decoder(&m_stream, m_filters) != LZMA_OK) - return false; - - m_started = true; - } - - const u8* const in_ptr = in.data.data() + *in_bytes_read; - m_stream.next_in = in_ptr; - m_stream.avail_in = in.bytes_written - *in_bytes_read; - - u8* const out_ptr = out->data.data() + out->bytes_written; - m_stream.next_out = out_ptr; - m_stream.avail_out = out->data.size() - out->bytes_written; - - const lzma_ret result = lzma_code(&m_stream, LZMA_RUN); - - *in_bytes_read += m_stream.next_in - in_ptr; - out->bytes_written += m_stream.next_out - out_ptr; - - m_done = result == LZMA_STREAM_END; - return result == LZMA_OK || result == LZMA_STREAM_END; -} - -WIAFileReader::ZstdDecompressor::ZstdDecompressor() -{ - m_stream = ZSTD_createDStream(); -} - -WIAFileReader::ZstdDecompressor::~ZstdDecompressor() -{ - ZSTD_freeDStream(m_stream); -} - -bool WIAFileReader::ZstdDecompressor::Decompress(const DecompressionBuffer& in, - DecompressionBuffer* out, size_t* in_bytes_read) -{ - if (!m_stream) - return false; - - ZSTD_inBuffer in_buffer{in.data.data(), in.bytes_written, *in_bytes_read}; - ZSTD_outBuffer out_buffer{out->data.data(), out->data.size(), out->bytes_written}; - - const size_t result = ZSTD_decompressStream(m_stream, &out_buffer, &in_buffer); - - *in_bytes_read = in_buffer.pos; - out->bytes_written = out_buffer.pos; - - m_done = result == 0; - return !ZSTD_isError(result); -} - -WIAFileReader::RVZPackDecompressor::RVZPackDecompressor(std::unique_ptr decompressor, - DecompressionBuffer decompressed, - u64 data_offset) - : m_decompressor(std::move(decompressor)), m_decompressed(std::move(decompressed)), - m_data_offset(data_offset) -{ -} - -std::optional WIAFileReader::RVZPackDecompressor::ReadToDecompressed( - const DecompressionBuffer& in, size_t* in_bytes_read, size_t decompressed_bytes_read, - size_t bytes_to_read) -{ - if (m_decompressed.data.size() < decompressed_bytes_read + bytes_to_read) - m_decompressed.data.resize(decompressed_bytes_read + bytes_to_read); - - if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read) - { - if (!m_decompressor->Decompress(in, &m_decompressed, in_bytes_read)) - return false; - - if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read) - return true; - } - - return std::nullopt; -} - -bool WIAFileReader::RVZPackDecompressor::Decompress(const DecompressionBuffer& in, - DecompressionBuffer* out, size_t* in_bytes_read) -{ - while (out->data.size() != out->bytes_written && !Done()) - { - if (m_size == 0) - { - if (m_decompressed.bytes_written == m_decompressed_bytes_read) - { - m_decompressed.data.resize(sizeof(u32)); - m_decompressed.bytes_written = 0; - m_decompressed_bytes_read = 0; - } - - std::optional result = - ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read, sizeof(u32)); - if (result) - return *result; - - m_size = Common::swap32(m_decompressed.data.data() + m_decompressed_bytes_read); - - m_junk = m_size & 0x80000000; - if (m_junk) - { - m_size &= 0x7FFFFFFF; - - constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32); - - result = ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read + sizeof(u32), - SEED_SIZE); - if (result) - return *result; - - m_lfg.SetSeed(m_decompressed.data.data() + m_decompressed_bytes_read + sizeof(u32)); - m_lfg.Forward(m_data_offset % VolumeWii::BLOCK_TOTAL_SIZE); - - m_decompressed_bytes_read += SEED_SIZE; - } - - m_decompressed_bytes_read += sizeof(u32); - } - - size_t bytes_to_write = std::min(m_size, out->data.size() - out->bytes_written); - if (m_junk) - { - m_lfg.GetBytes(bytes_to_write, out->data.data() + out->bytes_written); - out->bytes_written += bytes_to_write; - } - else - { - if (m_decompressed.bytes_written != m_decompressed_bytes_read) - { - bytes_to_write = - std::min(bytes_to_write, m_decompressed.bytes_written - m_decompressed_bytes_read); - - std::memcpy(out->data.data() + out->bytes_written, - m_decompressed.data.data() + m_decompressed_bytes_read, bytes_to_write); - - m_decompressed_bytes_read += bytes_to_write; - out->bytes_written += bytes_to_write; - } - else - { - const size_t prev_out_bytes_written = out->bytes_written; - const size_t old_out_size = out->data.size(); - const size_t new_out_size = out->bytes_written + bytes_to_write; - - if (new_out_size < old_out_size) - out->data.resize(new_out_size); - - if (!m_decompressor->Decompress(in, out, in_bytes_read)) - return false; - - out->data.resize(old_out_size); - - bytes_to_write = out->bytes_written - prev_out_bytes_written; - if (bytes_to_write == 0) - return true; - } - } - - m_data_offset += bytes_to_write; - m_size -= static_cast(bytes_to_write); - } - - // If out is full but not all data has been read from in, give the decompressor a chance to read - // from in anyway. This is needed for the case where zstd has read everything except the checksum. - if (out->data.size() == out->bytes_written && in.bytes_written != *in_bytes_read) - { - if (!m_decompressor->Decompress(in, out, in_bytes_read)) - return false; - } - - return true; -} - -bool WIAFileReader::RVZPackDecompressor::Done() const -{ - return m_size == 0 && m_decompressed.bytes_written == m_decompressed_bytes_read && - m_decompressor->Done(); -} - -WIAFileReader::Compressor::~Compressor() = default; - -WIAFileReader::PurgeCompressor::PurgeCompressor() -{ - mbedtls_sha1_init(&m_sha1_context); -} - -WIAFileReader::PurgeCompressor::~PurgeCompressor() = default; - -bool WIAFileReader::PurgeCompressor::Start() -{ - m_buffer.clear(); - m_bytes_written = 0; - - mbedtls_sha1_starts_ret(&m_sha1_context); - - return true; -} - -bool WIAFileReader::PurgeCompressor::AddPrecedingDataOnlyForPurgeHashing(const u8* data, - size_t size) -{ - mbedtls_sha1_update_ret(&m_sha1_context, data, size); - return true; -} - -bool WIAFileReader::PurgeCompressor::Compress(const u8* data, size_t size) -{ - // We could add support for calling this twice if we're fine with - // making the code more complicated, but there's no need to support it - ASSERT_MSG(DISCIO, m_bytes_written == 0, - "Calling PurgeCompressor::Compress() twice is not supported"); - - m_buffer.resize(size + sizeof(PurgeSegment) + sizeof(SHA1)); - - size_t bytes_read = 0; - - while (true) - { - const auto first_non_zero = - std::find_if(data + bytes_read, data + size, [](u8 x) { return x != 0; }); - - const u32 non_zero_data_start = static_cast(first_non_zero - data); - if (non_zero_data_start == size) - break; - - size_t non_zero_data_end = non_zero_data_start; - size_t sequence_length = 0; - for (size_t i = non_zero_data_start; i < size; ++i) - { - if (data[i] == 0) - { - ++sequence_length; - } - else - { - sequence_length = 0; - non_zero_data_end = i + 1; - } - - // To avoid wasting space, only count runs of zeroes that are of a certain length - // (unless there is nothing after the run of zeroes, then we might as well always count it) - if (sequence_length > sizeof(PurgeSegment)) - break; - } - - const u32 non_zero_data_length = static_cast(non_zero_data_end - non_zero_data_start); - - const PurgeSegment segment{Common::swap32(non_zero_data_start), - Common::swap32(non_zero_data_length)}; - std::memcpy(m_buffer.data() + m_bytes_written, &segment, sizeof(segment)); - m_bytes_written += sizeof(segment); - - std::memcpy(m_buffer.data() + m_bytes_written, data + non_zero_data_start, - non_zero_data_length); - m_bytes_written += non_zero_data_length; - - bytes_read = non_zero_data_end; - } - - return true; -} - -bool WIAFileReader::PurgeCompressor::End() -{ - mbedtls_sha1_update_ret(&m_sha1_context, m_buffer.data(), m_bytes_written); - - mbedtls_sha1_finish_ret(&m_sha1_context, m_buffer.data() + m_bytes_written); - m_bytes_written += sizeof(SHA1); - - ASSERT(m_bytes_written <= m_buffer.size()); - - return true; -} - -const u8* WIAFileReader::PurgeCompressor::GetData() const -{ - return m_buffer.data(); -} - -size_t WIAFileReader::PurgeCompressor::GetSize() const -{ - return m_bytes_written; -} - -WIAFileReader::Bzip2Compressor::Bzip2Compressor(int compression_level) - : m_compression_level(compression_level) -{ -} - -WIAFileReader::Bzip2Compressor::~Bzip2Compressor() -{ - BZ2_bzCompressEnd(&m_stream); -} - -bool WIAFileReader::Bzip2Compressor::Start() -{ - ASSERT_MSG(DISCIO, m_stream.state == nullptr, - "Called Bzip2Compressor::Start() twice without calling Bzip2Compressor::End()"); - - m_buffer.clear(); - m_stream.next_out = reinterpret_cast(m_buffer.data()); - - return BZ2_bzCompressInit(&m_stream, m_compression_level, 0, 0) == BZ_OK; -} - -bool WIAFileReader::Bzip2Compressor::Compress(const u8* data, size_t size) -{ - m_stream.next_in = reinterpret_cast(const_cast(data)); - m_stream.avail_in = static_cast(size); - - ExpandBuffer(size); - - while (m_stream.avail_in != 0) - { - if (m_stream.avail_out == 0) - ExpandBuffer(0x100); - - if (BZ2_bzCompress(&m_stream, BZ_RUN) != BZ_RUN_OK) - return false; - } - - return true; -} - -bool WIAFileReader::Bzip2Compressor::End() -{ - bool success = true; - - while (true) - { - if (m_stream.avail_out == 0) - ExpandBuffer(0x100); - - const int result = BZ2_bzCompress(&m_stream, BZ_FINISH); - if (result != BZ_FINISH_OK && result != BZ_STREAM_END) - success = false; - if (result != BZ_FINISH_OK) - break; - } - - if (BZ2_bzCompressEnd(&m_stream) != BZ_OK) - success = false; - - return success; -} - -void WIAFileReader::Bzip2Compressor::ExpandBuffer(size_t bytes_to_add) -{ - const size_t bytes_written = GetSize(); - m_buffer.resize(m_buffer.size() + bytes_to_add); - m_stream.next_out = reinterpret_cast(m_buffer.data()) + bytes_written; - m_stream.avail_out = static_cast(m_buffer.size() - bytes_written); -} - -const u8* WIAFileReader::Bzip2Compressor::GetData() const -{ - return m_buffer.data(); -} - -size_t WIAFileReader::Bzip2Compressor::GetSize() const -{ - return static_cast(reinterpret_cast(m_stream.next_out) - m_buffer.data()); -} - -WIAFileReader::LZMACompressor::LZMACompressor(bool lzma2, int compression_level, - u8 compressor_data_out[7], - u8* compressor_data_size_out) -{ - // lzma_lzma_preset returns false on success for some reason - if (lzma_lzma_preset(&m_options, static_cast(compression_level))) - { - m_initialization_failed = true; - return; - } - - if (!lzma2) - { - if (compressor_data_size_out) - *compressor_data_size_out = 5; - - if (compressor_data_out) - { - ASSERT(m_options.lc < 9); - ASSERT(m_options.lp < 5); - ASSERT(m_options.pb < 5); - compressor_data_out[0] = - static_cast((m_options.pb * 5 + m_options.lp) * 9 + m_options.lc); - - // The dictionary size is stored as a 32-bit little endian unsigned integer - static_assert(sizeof(m_options.dict_size) == sizeof(u32)); - std::memcpy(compressor_data_out + 1, &m_options.dict_size, sizeof(u32)); - } - } - else - { - if (compressor_data_size_out) - *compressor_data_size_out = 1; - - if (compressor_data_out) - { - u8 encoded_dict_size = 0; - while (encoded_dict_size < 40 && m_options.dict_size > LZMA2DictionarySize(encoded_dict_size)) - ++encoded_dict_size; - - compressor_data_out[0] = encoded_dict_size; - } - } - - m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1; - m_filters[0].options = &m_options; - m_filters[1].id = LZMA_VLI_UNKNOWN; - m_filters[1].options = nullptr; -} - -WIAFileReader::LZMACompressor::~LZMACompressor() -{ - lzma_end(&m_stream); -} - -bool WIAFileReader::LZMACompressor::Start() -{ - if (m_initialization_failed) - return false; - - m_buffer.clear(); - m_stream.next_out = m_buffer.data(); - - return lzma_raw_encoder(&m_stream, m_filters) == LZMA_OK; -} - -bool WIAFileReader::LZMACompressor::Compress(const u8* data, size_t size) -{ - m_stream.next_in = data; - m_stream.avail_in = size; - - ExpandBuffer(size); - - while (m_stream.avail_in != 0) - { - if (m_stream.avail_out == 0) - ExpandBuffer(0x100); - - if (lzma_code(&m_stream, LZMA_RUN) != LZMA_OK) - return false; - } - - return true; -} - -bool WIAFileReader::LZMACompressor::End() -{ - while (true) - { - if (m_stream.avail_out == 0) - ExpandBuffer(0x100); - - switch (lzma_code(&m_stream, LZMA_FINISH)) - { - case LZMA_OK: - break; - case LZMA_STREAM_END: - return true; - default: - return false; - } - } -} - -void WIAFileReader::LZMACompressor::ExpandBuffer(size_t bytes_to_add) -{ - const size_t bytes_written = GetSize(); - m_buffer.resize(m_buffer.size() + bytes_to_add); - m_stream.next_out = m_buffer.data() + bytes_written; - m_stream.avail_out = m_buffer.size() - bytes_written; -} - -const u8* WIAFileReader::LZMACompressor::GetData() const -{ - return m_buffer.data(); -} - -size_t WIAFileReader::LZMACompressor::GetSize() const -{ - return static_cast(m_stream.next_out - m_buffer.data()); -} - -WIAFileReader::ZstdCompressor::ZstdCompressor(int compression_level) -{ - m_stream = ZSTD_createCStream(); - - if (ZSTD_isError(ZSTD_CCtx_setParameter(m_stream, ZSTD_c_compressionLevel, compression_level))) - m_stream = nullptr; -} - -WIAFileReader::ZstdCompressor::~ZstdCompressor() -{ - ZSTD_freeCStream(m_stream); -} - -bool WIAFileReader::ZstdCompressor::Start() -{ - if (!m_stream) - return false; - - m_buffer.clear(); - m_out_buffer = {}; - - return !ZSTD_isError(ZSTD_CCtx_reset(m_stream, ZSTD_reset_session_only)); -} - -bool WIAFileReader::ZstdCompressor::Compress(const u8* data, size_t size) -{ - ZSTD_inBuffer in_buffer{data, size, 0}; - - ExpandBuffer(size); - - while (in_buffer.size != in_buffer.pos) - { - if (m_out_buffer.size == m_out_buffer.pos) - ExpandBuffer(0x100); - - if (ZSTD_isError(ZSTD_compressStream(m_stream, &m_out_buffer, &in_buffer))) - return false; - } - - return true; -} - -bool WIAFileReader::ZstdCompressor::End() -{ - while (true) - { - if (m_out_buffer.size == m_out_buffer.pos) - ExpandBuffer(0x100); - - const size_t result = ZSTD_endStream(m_stream, &m_out_buffer); - if (ZSTD_isError(result)) - return false; - if (result == 0) - return true; - } -} - -void WIAFileReader::ZstdCompressor::ExpandBuffer(size_t bytes_to_add) -{ - m_buffer.resize(m_buffer.size() + bytes_to_add); - - m_out_buffer.dst = m_buffer.data(); - m_out_buffer.size = m_buffer.size(); -} - WIAFileReader::Chunk::Chunk() = default; WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 78562a3fc43d..3103a1979556 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -9,21 +9,15 @@ #include #include #include -#include #include #include -#include -#include -#include -#include - #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/Swap.h" #include "DiscIO/Blob.h" -#include "DiscIO/LaggedFibonacciGenerator.h" #include "DiscIO/MultithreadedCompressor.h" +#include "DiscIO/WIACompression.h" #include "DiscIO/WiiEncryptionCache.h" namespace DiscIO @@ -157,13 +151,6 @@ class WIAFileReader : public BlobReader SHA1 hash; }; static_assert(sizeof(HashExceptionEntry) == 0x16, "Wrong size for WIA hash exception entry"); - - struct PurgeSegment - { - u32 offset; - u32 size; - }; - static_assert(sizeof(PurgeSegment) == 0x08, "Wrong size for WIA purge segment"); #pragma pack(pop) struct DataEntry @@ -180,224 +167,6 @@ class WIAFileReader : public BlobReader } }; - struct DecompressionBuffer - { - std::vector data; - size_t bytes_written = 0; - }; - - class Decompressor - { - public: - virtual ~Decompressor(); - - virtual bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, - size_t* in_bytes_read) = 0; - virtual bool Done() const { return m_done; }; - - protected: - bool m_done = false; - }; - - class NoneDecompressor final : public Decompressor - { - public: - bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, - size_t* in_bytes_read) override; - }; - - // This class assumes that more bytes won't be added to in once in.bytes_written == in.data.size() - // and that *in_bytes_read initially will be equal to the size of the exception lists - class PurgeDecompressor final : public Decompressor - { - public: - PurgeDecompressor(u64 decompressed_size); - bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, - size_t* in_bytes_read) override; - - private: - const u64 m_decompressed_size; - - PurgeSegment m_segment = {}; - size_t m_bytes_read = 0; - size_t m_segment_bytes_written = 0; - size_t m_out_bytes_written = 0; - bool m_started = false; - - mbedtls_sha1_context m_sha1_context; - }; - - class Bzip2Decompressor final : public Decompressor - { - public: - ~Bzip2Decompressor(); - - bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, - size_t* in_bytes_read) override; - - private: - bz_stream m_stream = {}; - bool m_started = false; - }; - - class LZMADecompressor final : public Decompressor - { - public: - LZMADecompressor(bool lzma2, const u8* filter_options, size_t filter_options_size); - ~LZMADecompressor(); - - bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, - size_t* in_bytes_read) override; - - private: - lzma_stream m_stream = LZMA_STREAM_INIT; - lzma_options_lzma m_options = {}; - lzma_filter m_filters[2]; - bool m_started = false; - bool m_error_occurred = false; - }; - - class ZstdDecompressor final : public Decompressor - { - public: - ZstdDecompressor(); - ~ZstdDecompressor(); - - bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, - size_t* in_bytes_read) override; - - private: - ZSTD_DStream* m_stream; - }; - - class RVZPackDecompressor final : public Decompressor - { - public: - RVZPackDecompressor(std::unique_ptr decompressor, - DecompressionBuffer decompressed, u64 data_offset); - - bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, - size_t* in_bytes_read) override; - - bool Done() const override; - - private: - std::optional ReadToDecompressed(const DecompressionBuffer& in, size_t* in_bytes_read, - size_t decompressed_bytes_read, size_t bytes_to_read); - - std::unique_ptr m_decompressor; - DecompressionBuffer m_decompressed; - size_t m_decompressed_bytes_read = 0; - u64 m_data_offset; - - u32 m_size = 0; - bool m_junk; - LaggedFibonacciGenerator m_lfg; - }; - - class Compressor - { - public: - virtual ~Compressor(); - - // First call Start, then AddDataOnlyForPurgeHashing/Compress any number of times, - // then End, then GetData/GetSize any number of times. - - virtual bool Start() = 0; - virtual bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) { return true; } - virtual bool Compress(const u8* data, size_t size) = 0; - virtual bool End() = 0; - - virtual const u8* GetData() const = 0; - virtual size_t GetSize() const = 0; - }; - - class PurgeCompressor final : public Compressor - { - public: - PurgeCompressor(); - ~PurgeCompressor(); - - bool Start() override; - bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) override; - bool Compress(const u8* data, size_t size) override; - bool End() override; - - const u8* GetData() const override; - size_t GetSize() const override; - - private: - std::vector m_buffer; - size_t m_bytes_written; - mbedtls_sha1_context m_sha1_context; - }; - - class Bzip2Compressor final : public Compressor - { - public: - Bzip2Compressor(int compression_level); - ~Bzip2Compressor(); - - bool Start() override; - bool Compress(const u8* data, size_t size) override; - bool End() override; - - const u8* GetData() const override; - size_t GetSize() const override; - - private: - void ExpandBuffer(size_t bytes_to_add); - - bz_stream m_stream = {}; - std::vector m_buffer; - int m_compression_level; - }; - - class LZMACompressor final : public Compressor - { - public: - LZMACompressor(bool lzma2, int compression_level, u8 compressor_data_out[7], - u8* compressor_data_size_out); - ~LZMACompressor(); - - bool Start() override; - bool Compress(const u8* data, size_t size) override; - bool End() override; - - const u8* GetData() const override; - size_t GetSize() const override; - - private: - void ExpandBuffer(size_t bytes_to_add); - - lzma_stream m_stream = LZMA_STREAM_INIT; - lzma_options_lzma m_options = {}; - lzma_filter m_filters[2]; - std::vector m_buffer; - bool m_initialization_failed = false; - }; - - class ZstdCompressor final : public Compressor - { - public: - ZstdCompressor(int compression_level); - ~ZstdCompressor(); - - bool Start() override; - bool Compress(const u8* data, size_t size) override; - bool End() override; - - const u8* GetData() const override { return m_buffer.data(); } - size_t GetSize() const override { return m_out_buffer.pos; } - - private: - void ExpandBuffer(size_t bytes_to_add); - - ZSTD_CStream* m_stream; - ZSTD_outBuffer m_out_buffer; - std::vector m_buffer; - }; - class Chunk { public: @@ -455,8 +224,6 @@ class WIAFileReader : public BlobReader static std::string VersionToString(u32 version); - static u32 LZMA2DictionarySize(u8 p); - struct ReuseID { bool operator==(const ReuseID& other) const diff --git a/Source/Core/DiscIO/WIACompression.cpp b/Source/Core/DiscIO/WIACompression.cpp new file mode 100644 index 000000000000..c15380a3437a --- /dev/null +++ b/Source/Core/DiscIO/WIACompression.cpp @@ -0,0 +1,793 @@ +// Copyright 2020 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#include "DiscIO/WIACompression.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "Common/Assert.h" +#include "Common/CommonTypes.h" +#include "Common/Swap.h" +#include "DiscIO/LaggedFibonacciGenerator.h" + +namespace DiscIO +{ +static u32 LZMA2DictionarySize(u8 p) +{ + return (static_cast(2) | (p & 1)) << (p / 2 + 11); +} + +Decompressor::~Decompressor() = default; + +bool NoneDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) +{ + const size_t length = + std::min(in.bytes_written - *in_bytes_read, out->data.size() - out->bytes_written); + + std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read, length); + + *in_bytes_read += length; + out->bytes_written += length; + + m_done = in.data.size() == *in_bytes_read; + return true; +} + +PurgeDecompressor::PurgeDecompressor(u64 decompressed_size) : m_decompressed_size(decompressed_size) +{ + mbedtls_sha1_init(&m_sha1_context); +} + +bool PurgeDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) +{ + if (!m_started) + { + mbedtls_sha1_starts_ret(&m_sha1_context); + + // Include the exception lists in the SHA-1 calculation (but not in the compression...) + mbedtls_sha1_update_ret(&m_sha1_context, in.data.data(), *in_bytes_read); + + m_started = true; + } + + while (!m_done && in.bytes_written != *in_bytes_read && + (m_segment_bytes_written < sizeof(m_segment) || out->data.size() != out->bytes_written)) + { + if (m_segment_bytes_written == 0 && *in_bytes_read == in.data.size() - sizeof(SHA1)) + { + const size_t zeroes_to_write = std::min(m_decompressed_size - m_out_bytes_written, + out->data.size() - out->bytes_written); + + std::memset(out->data.data() + out->bytes_written, 0, zeroes_to_write); + + out->bytes_written += zeroes_to_write; + m_out_bytes_written += zeroes_to_write; + + if (m_out_bytes_written == m_decompressed_size && in.bytes_written == in.data.size()) + { + SHA1 actual_hash; + mbedtls_sha1_finish_ret(&m_sha1_context, actual_hash.data()); + + SHA1 expected_hash; + std::memcpy(expected_hash.data(), in.data.data() + *in_bytes_read, expected_hash.size()); + + *in_bytes_read += expected_hash.size(); + m_done = true; + + if (actual_hash != expected_hash) + return false; + } + + return true; + } + + if (m_segment_bytes_written < sizeof(m_segment)) + { + const size_t bytes_to_copy = + std::min(in.bytes_written - *in_bytes_read, sizeof(m_segment) - m_segment_bytes_written); + + std::memcpy(reinterpret_cast(&m_segment) + m_segment_bytes_written, + in.data.data() + *in_bytes_read, bytes_to_copy); + mbedtls_sha1_update_ret(&m_sha1_context, in.data.data() + *in_bytes_read, bytes_to_copy); + + *in_bytes_read += bytes_to_copy; + m_bytes_read += bytes_to_copy; + m_segment_bytes_written += bytes_to_copy; + } + + if (m_segment_bytes_written < sizeof(m_segment)) + return true; + + const size_t offset = Common::swap32(m_segment.offset); + const size_t size = Common::swap32(m_segment.size); + + if (m_out_bytes_written < offset) + { + const size_t zeroes_to_write = + std::min(offset - m_out_bytes_written, out->data.size() - out->bytes_written); + + std::memset(out->data.data() + out->bytes_written, 0, zeroes_to_write); + + out->bytes_written += zeroes_to_write; + m_out_bytes_written += zeroes_to_write; + } + + if (m_out_bytes_written >= offset && m_out_bytes_written < offset + size) + { + const size_t bytes_to_copy = std::min( + std::min(offset + size - m_out_bytes_written, out->data.size() - out->bytes_written), + in.bytes_written - *in_bytes_read); + + std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read, + bytes_to_copy); + mbedtls_sha1_update_ret(&m_sha1_context, in.data.data() + *in_bytes_read, bytes_to_copy); + + *in_bytes_read += bytes_to_copy; + m_bytes_read += bytes_to_copy; + out->bytes_written += bytes_to_copy; + m_out_bytes_written += bytes_to_copy; + } + + if (m_out_bytes_written >= offset + size) + m_segment_bytes_written = 0; + } + + return true; +} + +Bzip2Decompressor::~Bzip2Decompressor() +{ + if (m_started) + BZ2_bzDecompressEnd(&m_stream); +} + +bool Bzip2Decompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) +{ + if (!m_started) + { + if (BZ2_bzDecompressInit(&m_stream, 0, 0) != BZ_OK) + return false; + + m_started = true; + } + + constexpr auto clamped_cast = [](size_t x) { + return static_cast( + std::min(std::numeric_limits().max(), x)); + }; + + char* const in_ptr = reinterpret_cast(const_cast(in.data.data() + *in_bytes_read)); + m_stream.next_in = in_ptr; + m_stream.avail_in = clamped_cast(in.bytes_written - *in_bytes_read); + + char* const out_ptr = reinterpret_cast(out->data.data() + out->bytes_written); + m_stream.next_out = out_ptr; + m_stream.avail_out = clamped_cast(out->data.size() - out->bytes_written); + + const int result = BZ2_bzDecompress(&m_stream); + + *in_bytes_read += m_stream.next_in - in_ptr; + out->bytes_written += m_stream.next_out - out_ptr; + + m_done = result == BZ_STREAM_END; + return result == BZ_OK || result == BZ_STREAM_END; +} + +LZMADecompressor::LZMADecompressor(bool lzma2, const u8* filter_options, size_t filter_options_size) +{ + m_options.preset_dict = nullptr; + + if (!lzma2 && filter_options_size == 5) + { + // The dictionary size is stored as a 32-bit little endian unsigned integer + static_assert(sizeof(m_options.dict_size) == sizeof(u32)); + std::memcpy(&m_options.dict_size, filter_options + 1, sizeof(u32)); + + const u8 d = filter_options[0]; + if (d >= (9 * 5 * 5)) + { + m_error_occurred = true; + } + else + { + m_options.lc = d % 9; + const u8 e = d / 9; + m_options.pb = e / 5; + m_options.lp = e % 5; + } + } + else if (lzma2 && filter_options_size == 1) + { + const u8 d = filter_options[0]; + if (d > 40) + m_error_occurred = true; + else + m_options.dict_size = d == 40 ? 0xFFFFFFFF : LZMA2DictionarySize(d); + } + else + { + m_error_occurred = true; + } + + m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1; + m_filters[0].options = &m_options; + m_filters[1].id = LZMA_VLI_UNKNOWN; + m_filters[1].options = nullptr; +} + +LZMADecompressor::~LZMADecompressor() +{ + if (m_started) + lzma_end(&m_stream); +} + +bool LZMADecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) +{ + if (!m_started) + { + if (m_error_occurred || lzma_raw_decoder(&m_stream, m_filters) != LZMA_OK) + return false; + + m_started = true; + } + + const u8* const in_ptr = in.data.data() + *in_bytes_read; + m_stream.next_in = in_ptr; + m_stream.avail_in = in.bytes_written - *in_bytes_read; + + u8* const out_ptr = out->data.data() + out->bytes_written; + m_stream.next_out = out_ptr; + m_stream.avail_out = out->data.size() - out->bytes_written; + + const lzma_ret result = lzma_code(&m_stream, LZMA_RUN); + + *in_bytes_read += m_stream.next_in - in_ptr; + out->bytes_written += m_stream.next_out - out_ptr; + + m_done = result == LZMA_STREAM_END; + return result == LZMA_OK || result == LZMA_STREAM_END; +} + +ZstdDecompressor::ZstdDecompressor() +{ + m_stream = ZSTD_createDStream(); +} + +ZstdDecompressor::~ZstdDecompressor() +{ + ZSTD_freeDStream(m_stream); +} + +bool ZstdDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) +{ + if (!m_stream) + return false; + + ZSTD_inBuffer in_buffer{in.data.data(), in.bytes_written, *in_bytes_read}; + ZSTD_outBuffer out_buffer{out->data.data(), out->data.size(), out->bytes_written}; + + const size_t result = ZSTD_decompressStream(m_stream, &out_buffer, &in_buffer); + + *in_bytes_read = in_buffer.pos; + out->bytes_written = out_buffer.pos; + + m_done = result == 0; + return !ZSTD_isError(result); +} + +RVZPackDecompressor::RVZPackDecompressor(std::unique_ptr decompressor, + DecompressionBuffer decompressed, u64 data_offset) + : m_decompressor(std::move(decompressor)), m_decompressed(std::move(decompressed)), + m_data_offset(data_offset) +{ +} + +std::optional RVZPackDecompressor::ReadToDecompressed(const DecompressionBuffer& in, + size_t* in_bytes_read, + size_t decompressed_bytes_read, + size_t bytes_to_read) +{ + if (m_decompressed.data.size() < decompressed_bytes_read + bytes_to_read) + m_decompressed.data.resize(decompressed_bytes_read + bytes_to_read); + + if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read) + { + if (!m_decompressor->Decompress(in, &m_decompressed, in_bytes_read)) + return false; + + if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read) + return true; + } + + return std::nullopt; +} + +bool RVZPackDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) +{ + while (out->data.size() != out->bytes_written && !Done()) + { + if (m_size == 0) + { + if (m_decompressed.bytes_written == m_decompressed_bytes_read) + { + m_decompressed.data.resize(sizeof(u32)); + m_decompressed.bytes_written = 0; + m_decompressed_bytes_read = 0; + } + + std::optional result = + ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read, sizeof(u32)); + if (result) + return *result; + + m_size = Common::swap32(m_decompressed.data.data() + m_decompressed_bytes_read); + + m_junk = m_size & 0x80000000; + if (m_junk) + { + m_size &= 0x7FFFFFFF; + + constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32); + constexpr size_t BLOCK_SIZE = 0x8000; + + result = ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read + sizeof(u32), + SEED_SIZE); + if (result) + return *result; + + m_lfg.SetSeed(m_decompressed.data.data() + m_decompressed_bytes_read + sizeof(u32)); + m_lfg.Forward(m_data_offset % BLOCK_SIZE); + + m_decompressed_bytes_read += SEED_SIZE; + } + + m_decompressed_bytes_read += sizeof(u32); + } + + size_t bytes_to_write = std::min(m_size, out->data.size() - out->bytes_written); + if (m_junk) + { + m_lfg.GetBytes(bytes_to_write, out->data.data() + out->bytes_written); + out->bytes_written += bytes_to_write; + } + else + { + if (m_decompressed.bytes_written != m_decompressed_bytes_read) + { + bytes_to_write = + std::min(bytes_to_write, m_decompressed.bytes_written - m_decompressed_bytes_read); + + std::memcpy(out->data.data() + out->bytes_written, + m_decompressed.data.data() + m_decompressed_bytes_read, bytes_to_write); + + m_decompressed_bytes_read += bytes_to_write; + out->bytes_written += bytes_to_write; + } + else + { + const size_t prev_out_bytes_written = out->bytes_written; + const size_t old_out_size = out->data.size(); + const size_t new_out_size = out->bytes_written + bytes_to_write; + + if (new_out_size < old_out_size) + out->data.resize(new_out_size); + + if (!m_decompressor->Decompress(in, out, in_bytes_read)) + return false; + + out->data.resize(old_out_size); + + bytes_to_write = out->bytes_written - prev_out_bytes_written; + if (bytes_to_write == 0) + return true; + } + } + + m_data_offset += bytes_to_write; + m_size -= static_cast(bytes_to_write); + } + + // If out is full but not all data has been read from in, give the decompressor a chance to read + // from in anyway. This is needed for the case where zstd has read everything except the checksum. + if (out->data.size() == out->bytes_written && in.bytes_written != *in_bytes_read) + { + if (!m_decompressor->Decompress(in, out, in_bytes_read)) + return false; + } + + return true; +} + +bool RVZPackDecompressor::Done() const +{ + return m_size == 0 && m_decompressed.bytes_written == m_decompressed_bytes_read && + m_decompressor->Done(); +} + +Compressor::~Compressor() = default; + +PurgeCompressor::PurgeCompressor() +{ + mbedtls_sha1_init(&m_sha1_context); +} + +PurgeCompressor::~PurgeCompressor() = default; + +bool PurgeCompressor::Start() +{ + m_buffer.clear(); + m_bytes_written = 0; + + mbedtls_sha1_starts_ret(&m_sha1_context); + + return true; +} + +bool PurgeCompressor::AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) +{ + mbedtls_sha1_update_ret(&m_sha1_context, data, size); + return true; +} + +bool PurgeCompressor::Compress(const u8* data, size_t size) +{ + // We could add support for calling this twice if we're fine with + // making the code more complicated, but there's no need to support it + ASSERT_MSG(DISCIO, m_bytes_written == 0, + "Calling PurgeCompressor::Compress() twice is not supported"); + + m_buffer.resize(size + sizeof(PurgeSegment) + sizeof(SHA1)); + + size_t bytes_read = 0; + + while (true) + { + const auto first_non_zero = + std::find_if(data + bytes_read, data + size, [](u8 x) { return x != 0; }); + + const u32 non_zero_data_start = static_cast(first_non_zero - data); + if (non_zero_data_start == size) + break; + + size_t non_zero_data_end = non_zero_data_start; + size_t sequence_length = 0; + for (size_t i = non_zero_data_start; i < size; ++i) + { + if (data[i] == 0) + { + ++sequence_length; + } + else + { + sequence_length = 0; + non_zero_data_end = i + 1; + } + + // To avoid wasting space, only count runs of zeroes that are of a certain length + // (unless there is nothing after the run of zeroes, then we might as well always count it) + if (sequence_length > sizeof(PurgeSegment)) + break; + } + + const u32 non_zero_data_length = static_cast(non_zero_data_end - non_zero_data_start); + + const PurgeSegment segment{Common::swap32(non_zero_data_start), + Common::swap32(non_zero_data_length)}; + std::memcpy(m_buffer.data() + m_bytes_written, &segment, sizeof(segment)); + m_bytes_written += sizeof(segment); + + std::memcpy(m_buffer.data() + m_bytes_written, data + non_zero_data_start, + non_zero_data_length); + m_bytes_written += non_zero_data_length; + + bytes_read = non_zero_data_end; + } + + return true; +} + +bool PurgeCompressor::End() +{ + mbedtls_sha1_update_ret(&m_sha1_context, m_buffer.data(), m_bytes_written); + + mbedtls_sha1_finish_ret(&m_sha1_context, m_buffer.data() + m_bytes_written); + m_bytes_written += sizeof(SHA1); + + ASSERT(m_bytes_written <= m_buffer.size()); + + return true; +} + +const u8* PurgeCompressor::GetData() const +{ + return m_buffer.data(); +} + +size_t PurgeCompressor::GetSize() const +{ + return m_bytes_written; +} + +Bzip2Compressor::Bzip2Compressor(int compression_level) : m_compression_level(compression_level) +{ +} + +Bzip2Compressor::~Bzip2Compressor() +{ + BZ2_bzCompressEnd(&m_stream); +} + +bool Bzip2Compressor::Start() +{ + ASSERT_MSG(DISCIO, m_stream.state == nullptr, + "Called Bzip2Compressor::Start() twice without calling Bzip2Compressor::End()"); + + m_buffer.clear(); + m_stream.next_out = reinterpret_cast(m_buffer.data()); + + return BZ2_bzCompressInit(&m_stream, m_compression_level, 0, 0) == BZ_OK; +} + +bool Bzip2Compressor::Compress(const u8* data, size_t size) +{ + m_stream.next_in = reinterpret_cast(const_cast(data)); + m_stream.avail_in = static_cast(size); + + ExpandBuffer(size); + + while (m_stream.avail_in != 0) + { + if (m_stream.avail_out == 0) + ExpandBuffer(0x100); + + if (BZ2_bzCompress(&m_stream, BZ_RUN) != BZ_RUN_OK) + return false; + } + + return true; +} + +bool Bzip2Compressor::End() +{ + bool success = true; + + while (true) + { + if (m_stream.avail_out == 0) + ExpandBuffer(0x100); + + const int result = BZ2_bzCompress(&m_stream, BZ_FINISH); + if (result != BZ_FINISH_OK && result != BZ_STREAM_END) + success = false; + if (result != BZ_FINISH_OK) + break; + } + + if (BZ2_bzCompressEnd(&m_stream) != BZ_OK) + success = false; + + return success; +} + +void Bzip2Compressor::ExpandBuffer(size_t bytes_to_add) +{ + const size_t bytes_written = GetSize(); + m_buffer.resize(m_buffer.size() + bytes_to_add); + m_stream.next_out = reinterpret_cast(m_buffer.data()) + bytes_written; + m_stream.avail_out = static_cast(m_buffer.size() - bytes_written); +} + +const u8* Bzip2Compressor::GetData() const +{ + return m_buffer.data(); +} + +size_t Bzip2Compressor::GetSize() const +{ + return static_cast(reinterpret_cast(m_stream.next_out) - m_buffer.data()); +} + +LZMACompressor::LZMACompressor(bool lzma2, int compression_level, u8 compressor_data_out[7], + u8* compressor_data_size_out) +{ + // lzma_lzma_preset returns false on success for some reason + if (lzma_lzma_preset(&m_options, static_cast(compression_level))) + { + m_initialization_failed = true; + return; + } + + if (!lzma2) + { + if (compressor_data_size_out) + *compressor_data_size_out = 5; + + if (compressor_data_out) + { + ASSERT(m_options.lc < 9); + ASSERT(m_options.lp < 5); + ASSERT(m_options.pb < 5); + compressor_data_out[0] = + static_cast((m_options.pb * 5 + m_options.lp) * 9 + m_options.lc); + + // The dictionary size is stored as a 32-bit little endian unsigned integer + static_assert(sizeof(m_options.dict_size) == sizeof(u32)); + std::memcpy(compressor_data_out + 1, &m_options.dict_size, sizeof(u32)); + } + } + else + { + if (compressor_data_size_out) + *compressor_data_size_out = 1; + + if (compressor_data_out) + { + u8 encoded_dict_size = 0; + while (encoded_dict_size < 40 && m_options.dict_size > LZMA2DictionarySize(encoded_dict_size)) + ++encoded_dict_size; + + compressor_data_out[0] = encoded_dict_size; + } + } + + m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1; + m_filters[0].options = &m_options; + m_filters[1].id = LZMA_VLI_UNKNOWN; + m_filters[1].options = nullptr; +} + +LZMACompressor::~LZMACompressor() +{ + lzma_end(&m_stream); +} + +bool LZMACompressor::Start() +{ + if (m_initialization_failed) + return false; + + m_buffer.clear(); + m_stream.next_out = m_buffer.data(); + + return lzma_raw_encoder(&m_stream, m_filters) == LZMA_OK; +} + +bool LZMACompressor::Compress(const u8* data, size_t size) +{ + m_stream.next_in = data; + m_stream.avail_in = size; + + ExpandBuffer(size); + + while (m_stream.avail_in != 0) + { + if (m_stream.avail_out == 0) + ExpandBuffer(0x100); + + if (lzma_code(&m_stream, LZMA_RUN) != LZMA_OK) + return false; + } + + return true; +} + +bool LZMACompressor::End() +{ + while (true) + { + if (m_stream.avail_out == 0) + ExpandBuffer(0x100); + + switch (lzma_code(&m_stream, LZMA_FINISH)) + { + case LZMA_OK: + break; + case LZMA_STREAM_END: + return true; + default: + return false; + } + } +} + +void LZMACompressor::ExpandBuffer(size_t bytes_to_add) +{ + const size_t bytes_written = GetSize(); + m_buffer.resize(m_buffer.size() + bytes_to_add); + m_stream.next_out = m_buffer.data() + bytes_written; + m_stream.avail_out = m_buffer.size() - bytes_written; +} + +const u8* LZMACompressor::GetData() const +{ + return m_buffer.data(); +} + +size_t LZMACompressor::GetSize() const +{ + return static_cast(m_stream.next_out - m_buffer.data()); +} + +ZstdCompressor::ZstdCompressor(int compression_level) +{ + m_stream = ZSTD_createCStream(); + + if (ZSTD_isError(ZSTD_CCtx_setParameter(m_stream, ZSTD_c_compressionLevel, compression_level))) + m_stream = nullptr; +} + +ZstdCompressor::~ZstdCompressor() +{ + ZSTD_freeCStream(m_stream); +} + +bool ZstdCompressor::Start() +{ + if (!m_stream) + return false; + + m_buffer.clear(); + m_out_buffer = {}; + + return !ZSTD_isError(ZSTD_CCtx_reset(m_stream, ZSTD_reset_session_only)); +} + +bool ZstdCompressor::Compress(const u8* data, size_t size) +{ + ZSTD_inBuffer in_buffer{data, size, 0}; + + ExpandBuffer(size); + + while (in_buffer.size != in_buffer.pos) + { + if (m_out_buffer.size == m_out_buffer.pos) + ExpandBuffer(0x100); + + if (ZSTD_isError(ZSTD_compressStream(m_stream, &m_out_buffer, &in_buffer))) + return false; + } + + return true; +} + +bool ZstdCompressor::End() +{ + while (true) + { + if (m_out_buffer.size == m_out_buffer.pos) + ExpandBuffer(0x100); + + const size_t result = ZSTD_endStream(m_stream, &m_out_buffer); + if (ZSTD_isError(result)) + return false; + if (result == 0) + return true; + } +} + +void ZstdCompressor::ExpandBuffer(size_t bytes_to_add) +{ + m_buffer.resize(m_buffer.size() + bytes_to_add); + + m_out_buffer.dst = m_buffer.data(); + m_out_buffer.size = m_buffer.size(); +} + +} // namespace DiscIO diff --git a/Source/Core/DiscIO/WIACompression.h b/Source/Core/DiscIO/WIACompression.h new file mode 100644 index 000000000000..5015802802c3 --- /dev/null +++ b/Source/Core/DiscIO/WIACompression.h @@ -0,0 +1,249 @@ +// Copyright 2020 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "Common/CommonTypes.h" +#include "DiscIO/LaggedFibonacciGenerator.h" + +namespace DiscIO +{ +struct DecompressionBuffer +{ + std::vector data; + size_t bytes_written = 0; +}; + +using SHA1 = std::array; + +struct PurgeSegment +{ + u32 offset; + u32 size; +}; +static_assert(sizeof(PurgeSegment) == 0x08, "Wrong size for WIA purge segment"); + +class Decompressor +{ +public: + virtual ~Decompressor(); + + virtual bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) = 0; + virtual bool Done() const { return m_done; }; + +protected: + bool m_done = false; +}; + +class NoneDecompressor final : public Decompressor +{ +public: + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; +}; + +// This class assumes that more bytes won't be added to in once in.bytes_written == in.data.size() +// and that *in_bytes_read initially will be equal to the size of the exception lists +class PurgeDecompressor final : public Decompressor +{ +public: + PurgeDecompressor(u64 decompressed_size); + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; + +private: + const u64 m_decompressed_size; + + PurgeSegment m_segment = {}; + size_t m_bytes_read = 0; + size_t m_segment_bytes_written = 0; + size_t m_out_bytes_written = 0; + bool m_started = false; + + mbedtls_sha1_context m_sha1_context; +}; + +class Bzip2Decompressor final : public Decompressor +{ +public: + ~Bzip2Decompressor(); + + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; + +private: + bz_stream m_stream = {}; + bool m_started = false; +}; + +class LZMADecompressor final : public Decompressor +{ +public: + LZMADecompressor(bool lzma2, const u8* filter_options, size_t filter_options_size); + ~LZMADecompressor(); + + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; + +private: + lzma_stream m_stream = LZMA_STREAM_INIT; + lzma_options_lzma m_options = {}; + lzma_filter m_filters[2]; + bool m_started = false; + bool m_error_occurred = false; +}; + +class ZstdDecompressor final : public Decompressor +{ +public: + ZstdDecompressor(); + ~ZstdDecompressor(); + + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; + +private: + ZSTD_DStream* m_stream; +}; + +class RVZPackDecompressor final : public Decompressor +{ +public: + RVZPackDecompressor(std::unique_ptr decompressor, DecompressionBuffer decompressed, + u64 data_offset); + + bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, + size_t* in_bytes_read) override; + + bool Done() const override; + +private: + std::optional ReadToDecompressed(const DecompressionBuffer& in, size_t* in_bytes_read, + size_t decompressed_bytes_read, size_t bytes_to_read); + + std::unique_ptr m_decompressor; + DecompressionBuffer m_decompressed; + size_t m_decompressed_bytes_read = 0; + u64 m_data_offset; + + u32 m_size = 0; + bool m_junk; + LaggedFibonacciGenerator m_lfg; +}; + +class Compressor +{ +public: + virtual ~Compressor(); + + // First call Start, then AddDataOnlyForPurgeHashing/Compress any number of times, + // then End, then GetData/GetSize any number of times. + + virtual bool Start() = 0; + virtual bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) { return true; } + virtual bool Compress(const u8* data, size_t size) = 0; + virtual bool End() = 0; + + virtual const u8* GetData() const = 0; + virtual size_t GetSize() const = 0; +}; + +class PurgeCompressor final : public Compressor +{ +public: + PurgeCompressor(); + ~PurgeCompressor(); + + bool Start() override; + bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) override; + bool Compress(const u8* data, size_t size) override; + bool End() override; + + const u8* GetData() const override; + size_t GetSize() const override; + +private: + std::vector m_buffer; + size_t m_bytes_written; + mbedtls_sha1_context m_sha1_context; +}; + +class Bzip2Compressor final : public Compressor +{ +public: + Bzip2Compressor(int compression_level); + ~Bzip2Compressor(); + + bool Start() override; + bool Compress(const u8* data, size_t size) override; + bool End() override; + + const u8* GetData() const override; + size_t GetSize() const override; + +private: + void ExpandBuffer(size_t bytes_to_add); + + bz_stream m_stream = {}; + std::vector m_buffer; + int m_compression_level; +}; + +class LZMACompressor final : public Compressor +{ +public: + LZMACompressor(bool lzma2, int compression_level, u8 compressor_data_out[7], + u8* compressor_data_size_out); + ~LZMACompressor(); + + bool Start() override; + bool Compress(const u8* data, size_t size) override; + bool End() override; + + const u8* GetData() const override; + size_t GetSize() const override; + +private: + void ExpandBuffer(size_t bytes_to_add); + + lzma_stream m_stream = LZMA_STREAM_INIT; + lzma_options_lzma m_options = {}; + lzma_filter m_filters[2]; + std::vector m_buffer; + bool m_initialization_failed = false; +}; + +class ZstdCompressor final : public Compressor +{ +public: + ZstdCompressor(int compression_level); + ~ZstdCompressor(); + + bool Start() override; + bool Compress(const u8* data, size_t size) override; + bool End() override; + + const u8* GetData() const override { return m_buffer.data(); } + size_t GetSize() const override { return m_out_buffer.pos; } + +private: + void ExpandBuffer(size_t bytes_to_add); + + ZSTD_CStream* m_stream; + ZSTD_outBuffer m_out_buffer; + std::vector m_buffer; +}; + +} // namespace DiscIO From f2c38c0e67720ece41c317e0c04e80aea7fd3f36 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 17 May 2020 15:34:50 +0200 Subject: [PATCH 34/36] RVZ: Make m_rvz a template parameter Needed for the next commit. --- Source/Core/DiscIO/Blob.cpp | 3 +- Source/Core/DiscIO/Blob.h | 10 +- Source/Core/DiscIO/WIABlob.cpp | 284 +++++++++++++++--------- Source/Core/DiscIO/WIABlob.h | 53 ++--- Source/Core/DolphinQt/ConvertDialog.cpp | 38 ++-- Source/Core/DolphinQt/ConvertDialog.h | 4 +- 6 files changed, 222 insertions(+), 170 deletions(-) diff --git a/Source/Core/DiscIO/Blob.cpp b/Source/Core/DiscIO/Blob.cpp index c2c44caa60c9..50146c9c919b 100644 --- a/Source/Core/DiscIO/Blob.cpp +++ b/Source/Core/DiscIO/Blob.cpp @@ -207,8 +207,9 @@ std::unique_ptr CreateBlobReader(const std::string& filename) case WBFS_MAGIC: return WbfsFileReader::Create(std::move(file), filename); case WIA_MAGIC: - case RVZ_MAGIC: return WIAFileReader::Create(std::move(file), filename); + case RVZ_MAGIC: + return RVZFileReader::Create(std::move(file), filename); default: if (auto directory_blob = DirectoryBlobReader::Create(filename)) return std::move(directory_blob); diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 149df2afc339..e2a1c3c9e9b4 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -25,7 +25,7 @@ namespace DiscIO { -enum class WIACompressionType : u32; +enum class WIARVZCompressionType : u32; // Increment CACHE_REVISION (GameFileCache.cpp) if the enum below is modified enum class BlobType @@ -176,9 +176,9 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path, bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, const std::string& outfile_path, CompressCB callback = nullptr, void* arg = nullptr); -bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, - const std::string& outfile_path, bool rvz, WIACompressionType compression_type, - int compression_level, int chunk_size, CompressCB callback = nullptr, - void* arg = nullptr); +bool ConvertToWIAOrRVZ(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, bool rvz, + WIARVZCompressionType compression_type, int compression_level, + int chunk_size, CompressCB callback = nullptr, void* arg = nullptr); } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 48731d69c120..6af49b202f23 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -42,15 +42,31 @@ namespace DiscIO { -std::pair GetAllowedCompressionLevels(WIACompressionType compression_type) +static void PushBack(std::vector* vector, const u8* begin, const u8* end) +{ + const size_t offset_in_vector = vector->size(); + vector->resize(offset_in_vector + (end - begin)); + std::copy(begin, end, vector->data() + offset_in_vector); +} + +template +static void PushBack(std::vector* vector, const T& x) +{ + static_assert(std::is_trivially_copyable_v); + + const u8* x_ptr = reinterpret_cast(&x); + PushBack(vector, x_ptr, x_ptr + sizeof(T)); +} + +std::pair GetAllowedCompressionLevels(WIARVZCompressionType compression_type) { switch (compression_type) { - case WIACompressionType::Bzip2: - case WIACompressionType::LZMA: - case WIACompressionType::LZMA2: + case WIARVZCompressionType::Bzip2: + case WIARVZCompressionType::LZMA: + case WIARVZCompressionType::LZMA2: return {1, 9}; - case WIACompressionType::Zstd: + case WIARVZCompressionType::Zstd: // The actual minimum level can be gotten by calling ZSTD_minCLevel(). However, returning that // would make the UI rather weird, because it is a negative number with very large magnitude. // Note: Level 0 is a special number which means "default level" (level 3 as of this writing). @@ -60,27 +76,28 @@ std::pair GetAllowedCompressionLevels(WIACompressionType compression_t } } -WIAFileReader::WIAFileReader(File::IOFile file, const std::string& path) +template +WIARVZFileReader::WIARVZFileReader(File::IOFile file, const std::string& path) : m_file(std::move(file)), m_encryption_cache(this) { m_valid = Initialize(path); } -WIAFileReader::~WIAFileReader() = default; +template +WIARVZFileReader::~WIARVZFileReader() = default; -bool WIAFileReader::Initialize(const std::string& path) +template +bool WIARVZFileReader::Initialize(const std::string& path) { if (!m_file.Seek(0, SEEK_SET) || !m_file.ReadArray(&m_header_1, 1)) return false; - if (m_header_1.magic != WIA_MAGIC && m_header_1.magic != RVZ_MAGIC) + if ((!RVZ && m_header_1.magic != WIA_MAGIC) || (RVZ && m_header_1.magic != RVZ_MAGIC)) return false; - m_rvz = m_header_1.magic == RVZ_MAGIC; - - const u32 version = m_rvz ? RVZ_VERSION : WIA_VERSION; + const u32 version = RVZ ? RVZ_VERSION : WIA_VERSION; const u32 version_read_compatible = - m_rvz ? RVZ_VERSION_READ_COMPATIBLE : WIA_VERSION_READ_COMPATIBLE; + RVZ ? RVZ_VERSION_READ_COMPATIBLE : WIA_VERSION_READ_COMPATIBLE; const u32 file_version = Common::swap32(m_header_1.version); const u32 file_version_compatible = Common::swap32(m_header_1.version_compatible); @@ -128,16 +145,16 @@ bool WIAFileReader::Initialize(const std::string& path) const u32 chunk_size = Common::swap32(m_header_2.chunk_size); const auto is_power_of_two = [](u32 x) { return (x & (x - 1)) == 0; }; - if ((!m_rvz || chunk_size < VolumeWii::BLOCK_TOTAL_SIZE || !is_power_of_two(chunk_size)) && + if ((!RVZ || chunk_size < VolumeWii::BLOCK_TOTAL_SIZE || !is_power_of_two(chunk_size)) && chunk_size % VolumeWii::GROUP_TOTAL_SIZE != 0) { return false; } const u32 compression_type = Common::swap32(m_header_2.compression_type); - m_compression_type = static_cast(compression_type); - if (m_compression_type > (m_rvz ? WIACompressionType::Zstd : WIACompressionType::LZMA2) || - (m_rvz && m_compression_type == WIACompressionType::Purge)) + m_compression_type = static_cast(compression_type); + if (m_compression_type > (RVZ ? WIARVZCompressionType::Zstd : WIARVZCompressionType::LZMA2) || + (RVZ && m_compression_type == WIARVZCompressionType::Purge)) { ERROR_LOG(DISCIO, "Unsupported compression type %u in %s", compression_type, path.c_str()); return false; @@ -221,7 +238,8 @@ bool WIAFileReader::Initialize(const std::string& path) return true; } -bool WIAFileReader::HasDataOverlap() const +template +bool WIARVZFileReader::HasDataOverlap() const { for (size_t i = 0; i < m_partition_entries.size(); ++i) { @@ -256,18 +274,22 @@ bool WIAFileReader::HasDataOverlap() const return false; } -std::unique_ptr WIAFileReader::Create(File::IOFile file, const std::string& path) +template +std::unique_ptr> WIARVZFileReader::Create(File::IOFile file, + const std::string& path) { - std::unique_ptr blob(new WIAFileReader(std::move(file), path)); + std::unique_ptr blob(new WIARVZFileReader(std::move(file), path)); return blob->m_valid ? std::move(blob) : nullptr; } -BlobType WIAFileReader::GetBlobType() const +template +BlobType WIARVZFileReader::GetBlobType() const { - return m_rvz ? BlobType::RVZ : BlobType::WIA; + return RVZ ? BlobType::RVZ : BlobType::WIA; } -bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) +template +bool WIARVZFileReader::Read(u64 offset, u64 size, u8* out_ptr) { if (offset + size > Common::swap64(m_header_1.iso_file_size)) return false; @@ -368,12 +390,15 @@ bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) return true; } -bool WIAFileReader::SupportsReadWiiDecrypted() const +template +bool WIARVZFileReader::SupportsReadWiiDecrypted() const { return !m_partition_entries.empty(); } -bool WIAFileReader::ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) +template +bool WIARVZFileReader::ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, + u64 partition_data_offset) { const u64 chunk_size = Common::swap32(m_header_2.chunk_size) * VolumeWii::BLOCK_DATA_SIZE / VolumeWii::BLOCK_TOTAL_SIZE; @@ -408,9 +433,11 @@ bool WIAFileReader::ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 part return size == 0; } -bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, - u32 sector_size, u64 data_offset, u64 data_size, u32 group_index, - u32 number_of_groups, u32 exception_lists) +template +bool WIARVZFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, + u32 sector_size, u64 data_offset, u64 data_size, + u32 group_index, u32 number_of_groups, + u32 exception_lists) { if (data_offset + data_size <= *offset) return true; @@ -445,7 +472,7 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu { const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; Chunk& chunk = ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, - exception_lists, m_rvz, group_offset_in_data); + exception_lists, RVZ, group_offset_in_data); if (!chunk.Read(offset_in_group, bytes_to_read, *out_ptr)) { m_cached_chunk_offset = std::numeric_limits::max(); // Invalidate the cache @@ -471,9 +498,11 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu return true; } -WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 compressed_size, - u64 decompressed_size, u32 exception_lists, - bool rvz_pack, u64 data_offset) +template +typename WIARVZFileReader::Chunk& +WIARVZFileReader::ReadCompressedData(u64 offset_in_file, u64 compressed_size, + u64 decompressed_size, u32 exception_lists, bool rvz_pack, + u64 data_offset) { if (offset_in_file == m_cached_chunk_offset) return m_cached_chunk; @@ -481,29 +510,29 @@ WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 std::unique_ptr decompressor; switch (m_compression_type) { - case WIACompressionType::None: + case WIARVZCompressionType::None: decompressor = std::make_unique(); break; - case WIACompressionType::Purge: + case WIARVZCompressionType::Purge: decompressor = std::make_unique(decompressed_size); break; - case WIACompressionType::Bzip2: + case WIARVZCompressionType::Bzip2: decompressor = std::make_unique(); break; - case WIACompressionType::LZMA: + case WIARVZCompressionType::LZMA: decompressor = std::make_unique(false, m_header_2.compressor_data, m_header_2.compressor_data_size); break; - case WIACompressionType::LZMA2: + case WIARVZCompressionType::LZMA2: decompressor = std::make_unique(true, m_header_2.compressor_data, m_header_2.compressor_data_size); break; - case WIACompressionType::Zstd: + case WIARVZCompressionType::Zstd: decompressor = std::make_unique(); break; } - const bool compressed_exception_lists = m_compression_type > WIACompressionType::Purge; + const bool compressed_exception_lists = m_compression_type > WIARVZCompressionType::Purge; m_cached_chunk = Chunk(&m_file, offset_in_file, compressed_size, decompressed_size, exception_lists, @@ -512,7 +541,8 @@ WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 return m_cached_chunk; } -std::string WIAFileReader::VersionToString(u32 version) +template +std::string WIARVZFileReader::VersionToString(u32 version) { const u8 a = version >> 24; const u8 b = (version >> 16) & 0xff; @@ -525,12 +555,14 @@ std::string WIAFileReader::VersionToString(u32 version) return StringFromFormat("%u.%02x.%02x.beta%u", a, b, c, d); } -WIAFileReader::Chunk::Chunk() = default; +template +WIARVZFileReader::Chunk::Chunk() = default; -WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, - u64 decompressed_size, u32 exception_lists, - bool compressed_exception_lists, bool rvz_pack, u64 data_offset, - std::unique_ptr decompressor) +template +WIARVZFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, + u64 decompressed_size, u32 exception_lists, + bool compressed_exception_lists, bool rvz_pack, u64 data_offset, + std::unique_ptr decompressor) : m_file(file), m_offset_in_file(offset_in_file), m_exception_lists(exception_lists), m_compressed_exception_lists(compressed_exception_lists), m_rvz_pack(rvz_pack), m_data_offset(data_offset), m_decompressor(std::move(decompressor)) @@ -547,7 +579,8 @@ WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compress m_out.data.resize(decompressed_size + m_out_bytes_allocated_for_exceptions); } -bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) +template +bool WIARVZFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) { if (!m_decompressor || !m_file || offset + size > m_out.data.size() - m_out_bytes_allocated_for_exceptions) @@ -655,7 +688,8 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) return true; } -bool WIAFileReader::Chunk::Decompress() +template +bool WIARVZFileReader::Chunk::Decompress() { if (m_rvz_pack && m_exception_lists == 0) { @@ -675,8 +709,10 @@ bool WIAFileReader::Chunk::Decompress() return m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read); } -bool WIAFileReader::Chunk::HandleExceptions(const u8* data, size_t bytes_allocated, - size_t bytes_written, size_t* bytes_used, bool align) +template +bool WIARVZFileReader::Chunk::HandleExceptions(const u8* data, size_t bytes_allocated, + size_t bytes_written, size_t* bytes_used, + bool align) { while (m_exception_lists > 0) { @@ -709,8 +745,10 @@ bool WIAFileReader::Chunk::HandleExceptions(const u8* data, size_t bytes_allocat return true; } -void WIAFileReader::Chunk::GetHashExceptions(std::vector* exception_list, - u64 exception_list_index, u16 additional_offset) const +template +void WIARVZFileReader::Chunk::GetHashExceptions( + std::vector* exception_list, u64 exception_list_index, + u16 additional_offset) const { ASSERT(m_exception_lists == 0); @@ -736,7 +774,8 @@ void WIAFileReader::Chunk::GetHashExceptions(std::vector* ex m_in_bytes_used_for_exceptions)); } -bool WIAFileReader::ApplyHashExceptions( +template +bool WIARVZFileReader::ApplyHashExceptions( const std::vector& exception_list, VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]) { @@ -759,7 +798,8 @@ bool WIAFileReader::ApplyHashExceptions( return true; } -bool WIAFileReader::PadTo4(File::IOFile* file, u64* bytes_written) +template +bool WIARVZFileReader::PadTo4(File::IOFile* file, u64* bytes_written) { constexpr u32 ZEROES = 0; const u64 bytes_to_write = Common::AlignUp(*bytes_written, 4) - *bytes_written; @@ -770,9 +810,10 @@ bool WIAFileReader::PadTo4(File::IOFile* file, u64* bytes_written) return file->WriteBytes(&ZEROES, bytes_to_write); } -void WIAFileReader::AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups, - std::vector* raw_data_entries, - std::vector* data_entries) +template +void WIARVZFileReader::AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups, + std::vector* raw_data_entries, + std::vector* data_entries) { constexpr size_t SKIP_SIZE = sizeof(WIAHeader2::disc_header); const u64 skip = offset < SKIP_SIZE ? std::min(SKIP_SIZE - offset, size) : 0; @@ -792,7 +833,8 @@ void WIAFileReader::AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* t Common::swap32(group_index), Common::swap32(groups)}); } -WIAFileReader::PartitionDataEntry WIAFileReader::CreatePartitionDataEntry( +template +typename WIARVZFileReader::PartitionDataEntry WIARVZFileReader::CreatePartitionDataEntry( u64 offset, u64 size, u32 index, int chunk_size, u32* total_groups, const std::vector& partition_entries, std::vector* data_entries) { @@ -807,7 +849,8 @@ WIAFileReader::PartitionDataEntry WIAFileReader::CreatePartitionDataEntry( Common::swap32(group_index), Common::swap32(groups)}; } -ConversionResultCode WIAFileReader::SetUpDataEntriesForWriting( +template +ConversionResultCode WIARVZFileReader::SetUpDataEntriesForWriting( const VolumeDisc* volume, int chunk_size, u64 iso_size, u32* total_groups, std::vector* partition_entries, std::vector* raw_data_entries, std::vector* data_entries, std::vector* partition_file_systems) @@ -924,8 +967,9 @@ ConversionResultCode WIAFileReader::SetUpDataEntriesForWriting( return ConversionResultCode::Success; } -std::optional> WIAFileReader::Compress(Compressor* compressor, const u8* data, - size_t size) +template +std::optional> WIARVZFileReader::Compress(Compressor* compressor, + const u8* data, size_t size) { if (compressor) { @@ -939,23 +983,24 @@ std::optional> WIAFileReader::Compress(Compressor* compressor, c return std::vector(data, data + size); } -void WIAFileReader::SetUpCompressor(std::unique_ptr* compressor, - WIACompressionType compression_type, int compression_level, - WIAHeader2* header_2) +template +void WIARVZFileReader::SetUpCompressor(std::unique_ptr* compressor, + WIARVZCompressionType compression_type, + int compression_level, WIAHeader2* header_2) { switch (compression_type) { - case WIACompressionType::None: + case WIARVZCompressionType::None: *compressor = nullptr; break; - case WIACompressionType::Purge: + case WIARVZCompressionType::Purge: *compressor = std::make_unique(); break; - case WIACompressionType::Bzip2: + case WIARVZCompressionType::Bzip2: *compressor = std::make_unique(compression_level); break; - case WIACompressionType::LZMA: - case WIACompressionType::LZMA2: + case WIARVZCompressionType::LZMA: + case WIARVZCompressionType::LZMA2: { u8* compressor_data = nullptr; u8* compressor_data_size = nullptr; @@ -966,19 +1011,21 @@ void WIAFileReader::SetUpCompressor(std::unique_ptr* compressor, compressor_data_size = &header_2->compressor_data_size; } - const bool lzma2 = compression_type == WIACompressionType::LZMA2; + const bool lzma2 = compression_type == WIARVZCompressionType::LZMA2; *compressor = std::make_unique(lzma2, compression_level, compressor_data, compressor_data_size); break; } - case WIACompressionType::Zstd: + case WIARVZCompressionType::Zstd: *compressor = std::make_unique(compression_level); break; } } -bool WIAFileReader::TryReuse(std::map* reusable_groups, - std::mutex* reusable_groups_mutex, OutputParametersEntry* entry) +template +bool WIARVZFileReader::TryReuse(std::map* reusable_groups, + std::mutex* reusable_groups_mutex, + OutputParametersEntry* entry) { if (entry->reused_group) return true; @@ -1020,9 +1067,10 @@ static bool AllSame(const u8* begin, const u8* end) return AllAre(begin, end, *begin); }; -void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, - size_t chunks, u64 total_size, u64 data_offset, u64 in_offset, - bool allow_junk_reuse, bool compression, const FileSystem* file_system) +template +static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, size_t chunks, + u64 total_size, u64 data_offset, u64 in_offset, bool allow_junk_reuse, + bool compression, const FileSystem* file_system) { using Seed = std::array; struct JunkInfo @@ -1140,19 +1188,23 @@ void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_ } } -void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset, - bool allow_junk_reuse, bool compression, const FileSystem* file_system) +template +static void RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset, + bool allow_junk_reuse, bool compression, const FileSystem* file_system) { RVZPack(in, out, size, 1, size, data_offset, 0, allow_junk_reuse, compression, file_system); } -ConversionResult WIAFileReader::ProcessAndCompress( - CompressThreadState* state, CompressParameters parameters, - const std::vector& partition_entries, - const std::vector& data_entries, const FileSystem* file_system, - std::map* reusable_groups, std::mutex* reusable_groups_mutex, - u64 chunks_per_wii_group, u64 exception_lists_per_chunk, bool compressed_exception_lists, - bool compression, bool rvz) +template +ConversionResult::OutputParameters> +WIARVZFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, + const std::vector& partition_entries, + const std::vector& data_entries, + const FileSystem* file_system, + std::map* reusable_groups, + std::mutex* reusable_groups_mutex, + u64 chunks_per_wii_group, u64 exception_lists_per_chunk, + bool compressed_exception_lists, bool compression) { std::vector output_entries; @@ -1164,7 +1216,7 @@ ConversionResult WIAFileReader::ProcessAndCompr if (AllSame(data)) entry.reuse_id = ReuseID{nullptr, data.size(), false, data.front()}; - if (rvz) + if constexpr (RVZ) { RVZPack(data.data(), output_entries.data(), data.size(), parameters.data_offset, true, compression, file_system); @@ -1315,9 +1367,9 @@ ConversionResult WIAFileReader::ProcessAndCompr compare_hashes(offsetof(HashBlock, padding_2), sizeof(HashBlock::padding_2)); } - static_assert(std::is_trivially_copyable_v); - if (rvz) + if constexpr (RVZ) { // We must not store junk efficiently for chunks that may get reused at a position // which has a different value of data_offset % VolumeWii::BLOCK_TOTAL_SIZE @@ -1471,11 +1523,12 @@ ConversionResult WIAFileReader::ProcessAndCompr return OutputParameters{std::move(output_entries), parameters.bytes_read, parameters.group_index}; } -ConversionResultCode WIAFileReader::Output(std::vector* entries, - File::IOFile* outfile, - std::map* reusable_groups, - std::mutex* reusable_groups_mutex, - GroupEntry* group_entry, u64* bytes_written) +template +ConversionResultCode WIARVZFileReader::Output(std::vector* entries, + File::IOFile* outfile, + std::map* reusable_groups, + std::mutex* reusable_groups_mutex, + GroupEntry* group_entry, u64* bytes_written) { for (OutputParametersEntry& entry : *entries) { @@ -1518,9 +1571,10 @@ ConversionResultCode WIAFileReader::Output(std::vector* e return ConversionResultCode::Success; } -ConversionResultCode WIAFileReader::RunCallback(size_t groups_written, u64 bytes_read, - u64 bytes_written, u32 total_groups, u64 iso_size, - CompressCB callback, void* arg) +template +ConversionResultCode +WIARVZFileReader::RunCallback(size_t groups_written, u64 bytes_read, u64 bytes_written, + u32 total_groups, u64 iso_size, CompressCB callback, void* arg) { int ratio = 0; if (bytes_read != 0) @@ -1536,8 +1590,9 @@ ConversionResultCode WIAFileReader::RunCallback(size_t groups_written, u64 bytes ConversionResultCode::Canceled; } -bool WIAFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound, - u64* bytes_written, u64* offset_out) +template +bool WIARVZFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, + u64 upper_bound, u64* bytes_written, u64* offset_out) { // The first part of the check is to prevent this from running more than once. If *bytes_written // is past the upper bound, we are already at the end of the file, so we don't need to do anything @@ -1556,10 +1611,12 @@ bool WIAFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, return PadTo4(file, bytes_written); } +template ConversionResultCode -WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, bool rvz, WIACompressionType compression_type, - int compression_level, int chunk_size, CompressCB callback, void* arg) +WIARVZFileReader::Convert(BlobReader* infile, const VolumeDisc* infile_volume, + File::IOFile* outfile, WIARVZCompressionType compression_type, + int compression_level, int chunk_size, CompressCB callback, + void* arg) { ASSERT(infile->IsDataSizeAccurate()); ASSERT(chunk_size > 0); @@ -1567,7 +1624,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, const u64 iso_size = infile->GetDataSize(); const u64 chunks_per_wii_group = std::max(1, VolumeWii::GROUP_TOTAL_SIZE / chunk_size); const u64 exception_lists_per_chunk = std::max(1, chunk_size / VolumeWii::GROUP_TOTAL_SIZE); - const bool compressed_exception_lists = compression_type > WIACompressionType::Purge; + const bool compressed_exception_lists = compression_type > WIARVZCompressionType::Purge; u64 bytes_read = 0; u64 bytes_written = 0; @@ -1631,12 +1688,12 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, partition_file_systems[data_entry.index] : non_partition_file_system; - const bool compression = compression_type != WIACompressionType::None; + const bool compression = compression_type != WIARVZCompressionType::None; return ProcessAndCompress(state, std::move(parameters), partition_entries, data_entries, file_system, &reusable_groups, &reusable_groups_mutex, chunks_per_wii_group, exception_lists_per_chunk, - compressed_exception_lists, compression, rvz); + compressed_exception_lists, compression); }; const auto output = [&](OutputParameters parameters) { @@ -1820,10 +1877,10 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, header_2.group_entries_offset = Common::swap64(group_entries_offset); header_2.group_entries_size = Common::swap32(static_cast(compressed_group_entries->size())); - header_1.magic = rvz ? RVZ_MAGIC : WIA_MAGIC; - header_1.version = Common::swap32(rvz ? RVZ_VERSION : WIA_VERSION); + header_1.magic = RVZ ? RVZ_MAGIC : WIA_MAGIC; + header_1.version = Common::swap32(RVZ ? RVZ_VERSION : WIA_VERSION); header_1.version_compatible = - Common::swap32(rvz ? RVZ_VERSION_WRITE_COMPATIBLE : WIA_VERSION_WRITE_COMPATIBLE); + Common::swap32(RVZ ? RVZ_VERSION_WRITE_COMPATIBLE : WIA_VERSION_WRITE_COMPATIBLE); header_1.header_2_size = Common::swap32(sizeof(WIAHeader2)); mbedtls_sha1_ret(reinterpret_cast(&header_2), sizeof(header_2), header_1.header_2_hash.data()); @@ -1843,9 +1900,10 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, return ConversionResultCode::Success; } -bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, - const std::string& outfile_path, bool rvz, WIACompressionType compression_type, - int compression_level, int chunk_size, CompressCB callback, void* arg) +bool ConvertToWIAOrRVZ(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, bool rvz, + WIARVZCompressionType compression_type, int compression_level, + int chunk_size, CompressCB callback, void* arg) { File::IOFile outfile(outfile_path, "wb"); if (!outfile) @@ -1859,9 +1917,10 @@ bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, std::unique_ptr infile_volume = CreateDisc(infile_path); + const auto convert = rvz ? RVZFileReader::Convert : WIAFileReader::Convert; const ConversionResultCode result = - WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, rvz, compression_type, - compression_level, chunk_size, callback, arg); + convert(infile, infile_volume.get(), &outfile, compression_type, compression_level, + chunk_size, callback, arg); if (result == ConversionResultCode::ReadFailed) PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); @@ -1883,4 +1942,7 @@ bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, return result == ConversionResultCode::Success; } +template class WIARVZFileReader; +template class WIARVZFileReader; + } // namespace DiscIO diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 3103a1979556..d7e3573e7cb9 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -25,7 +25,7 @@ namespace DiscIO class FileSystem; class VolumeDisc; -enum class WIACompressionType : u32 +enum class WIARVZCompressionType : u32 { None = 0, Purge = 1, @@ -35,17 +35,18 @@ enum class WIACompressionType : u32 Zstd = 5, }; -std::pair GetAllowedCompressionLevels(WIACompressionType compression_type); +std::pair GetAllowedCompressionLevels(WIARVZCompressionType compression_type); constexpr u32 WIA_MAGIC = 0x01414957; // "WIA\x1" (byteswapped to little endian) constexpr u32 RVZ_MAGIC = 0x015A5652; // "RVZ\x1" (byteswapped to little endian) -class WIAFileReader : public BlobReader +template +class WIARVZFileReader : public BlobReader { public: - ~WIAFileReader(); + ~WIARVZFileReader(); - static std::unique_ptr Create(File::IOFile file, const std::string& path); + static std::unique_ptr Create(File::IOFile file, const std::string& path); BlobType GetBlobType() const override; @@ -60,11 +61,10 @@ class WIAFileReader : public BlobReader bool SupportsReadWiiDecrypted() const override; bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override; - static ConversionResultCode ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, bool rvz, - WIACompressionType compression_type, - int compression_level, int chunk_size, - CompressCB callback, void* arg); + static ConversionResultCode Convert(BlobReader* infile, const VolumeDisc* infile_volume, + File::IOFile* outfile, WIARVZCompressionType compression_type, + int compression_level, int chunk_size, CompressCB callback, + void* arg); private: using SHA1 = std::array; @@ -209,7 +209,7 @@ class WIAFileReader : public BlobReader u64 m_data_offset = 0; }; - explicit WIAFileReader(File::IOFile file, const std::string& path); + explicit WIARVZFileReader(File::IOFile file, const std::string& path); bool Initialize(const std::string& path); bool HasDataOverlap() const; @@ -306,15 +306,10 @@ class WIAFileReader : public BlobReader u64* bytes_written, u64* offset_out); static void SetUpCompressor(std::unique_ptr* compressor, - WIACompressionType compression_type, int compression_level, + WIARVZCompressionType compression_type, int compression_level, WIAHeader2* header_2); static bool TryReuse(std::map* reusable_groups, std::mutex* reusable_groups_mutex, OutputParametersEntry* entry); - static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, size_t chunks, - u64 total_size, u64 data_offset, u64 in_offset, bool allow_junk_reuse, - bool compression, const FileSystem* file_system); - static void RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset, - bool allow_junk_reuse, bool compression, const FileSystem* file_system); static ConversionResult ProcessAndCompress(CompressThreadState* state, CompressParameters parameters, const std::vector& partition_entries, @@ -322,7 +317,7 @@ class WIAFileReader : public BlobReader std::map* reusable_groups, std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, u64 exception_lists_per_chunk, bool compressed_exception_lists, - bool compression, bool rvz); + bool compression); static ConversionResultCode Output(std::vector* entries, File::IOFile* outfile, std::map* reusable_groups, @@ -332,25 +327,8 @@ class WIAFileReader : public BlobReader u32 total_groups, u64 iso_size, CompressCB callback, void* arg); - static void PushBack(std::vector* vector, const u8* begin, const u8* end) - { - const size_t offset_in_vector = vector->size(); - vector->resize(offset_in_vector + (end - begin)); - std::copy(begin, end, vector->data() + offset_in_vector); - } - - template - static void PushBack(std::vector* vector, const T& x) - { - static_assert(std::is_trivially_copyable_v); - - const u8* x_ptr = reinterpret_cast(&x); - PushBack(vector, x_ptr, x_ptr + sizeof(T)); - } - bool m_valid; - bool m_rvz; - WIACompressionType m_compression_type; + WIARVZCompressionType m_compression_type; File::IOFile m_file; Chunk m_cached_chunk; @@ -382,4 +360,7 @@ class WIAFileReader : public BlobReader static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00020000; }; +using WIAFileReader = WIARVZFileReader; +using RVZFileReader = WIARVZFileReader; + } // namespace DiscIO diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index a587af0debd2..d7676cc81a77 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -139,7 +139,8 @@ void ConvertDialog::AddToBlockSizeComboBox(int size) m_block_size->setCurrentIndex(m_block_size->count() - 1); } -void ConvertDialog::AddToCompressionComboBox(const QString& name, DiscIO::WIACompressionType type) +void ConvertDialog::AddToCompressionComboBox(const QString& name, + DiscIO::WIARVZCompressionType type) { m_compression->addItem(name, static_cast(type)); } @@ -227,7 +228,7 @@ void ConvertDialog::OnFormatChanged() { case DiscIO::BlobType::GCZ: m_compression->setEnabled(true); - AddToCompressionComboBox(QStringLiteral("Deflate"), DiscIO::WIACompressionType::None); + AddToCompressionComboBox(QStringLiteral("Deflate"), DiscIO::WIARVZCompressionType::None); break; case DiscIO::BlobType::WIA: case DiscIO::BlobType::RVZ: @@ -237,15 +238,22 @@ void ConvertDialog::OnFormatChanged() // i18n: %1 is the name of a compression method (e.g. LZMA) const QString slow = tr("%1 (slow)"); - AddToCompressionComboBox(tr("No Compression"), DiscIO::WIACompressionType::None); + AddToCompressionComboBox(tr("No Compression"), DiscIO::WIARVZCompressionType::None); + if (format == DiscIO::BlobType::WIA) - AddToCompressionComboBox(QStringLiteral("Purge"), DiscIO::WIACompressionType::Purge); - AddToCompressionComboBox(slow.arg(QStringLiteral("bzip2")), DiscIO::WIACompressionType::Bzip2); - AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA")), DiscIO::WIACompressionType::LZMA); - AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA2")), DiscIO::WIACompressionType::LZMA2); + AddToCompressionComboBox(QStringLiteral("Purge"), DiscIO::WIARVZCompressionType::Purge); + + AddToCompressionComboBox(slow.arg(QStringLiteral("bzip2")), + DiscIO::WIARVZCompressionType::Bzip2); + + AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA")), DiscIO::WIARVZCompressionType::LZMA); + + AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA2")), + DiscIO::WIARVZCompressionType::LZMA2); + if (format == DiscIO::BlobType::RVZ) { - AddToCompressionComboBox(QStringLiteral("Zstandard"), DiscIO::WIACompressionType::Zstd); + AddToCompressionComboBox(QStringLiteral("Zstandard"), DiscIO::WIARVZCompressionType::Zstd); m_compression->setCurrentIndex(m_compression->count() - 1); } @@ -269,7 +277,7 @@ void ConvertDialog::OnCompressionChanged() m_compression_level->clear(); const auto compression_type = - static_cast(m_compression->currentData().toInt()); + static_cast(m_compression->currentData().toInt()); const std::pair range = DiscIO::GetAllowedCompressionLevels(compression_type); @@ -299,8 +307,8 @@ void ConvertDialog::Convert() { const DiscIO::BlobType format = static_cast(m_format->currentData().toInt()); const int block_size = m_block_size->currentData().toInt(); - const DiscIO::WIACompressionType compression = - static_cast(m_compression->currentData().toInt()); + const DiscIO::WIARVZCompressionType compression = + static_cast(m_compression->currentData().toInt()); const int compression_level = m_compression_level->currentData().toInt(); const bool scrub = m_scrub->isChecked(); @@ -477,10 +485,10 @@ void ConvertDialog::Convert() case DiscIO::BlobType::WIA: case DiscIO::BlobType::RVZ: good = std::async(std::launch::async, [&] { - const bool good = - DiscIO::ConvertToWIA(blob_reader.get(), original_path, dst_path.toStdString(), - format == DiscIO::BlobType::RVZ, compression, compression_level, - block_size, &CompressCB, &progress_dialog); + const bool good = DiscIO::ConvertToWIAOrRVZ( + blob_reader.get(), original_path, dst_path.toStdString(), + format == DiscIO::BlobType::RVZ, compression, compression_level, block_size, + &CompressCB, &progress_dialog); progress_dialog.Reset(); return good; }); diff --git a/Source/Core/DolphinQt/ConvertDialog.h b/Source/Core/DolphinQt/ConvertDialog.h index b14ecb6c527e..9a53265f861d 100644 --- a/Source/Core/DolphinQt/ConvertDialog.h +++ b/Source/Core/DolphinQt/ConvertDialog.h @@ -16,7 +16,7 @@ class QComboBox; namespace DiscIO { -enum class WIACompressionType : u32; +enum class WIARVZCompressionType : u32; } namespace UICommon @@ -39,7 +39,7 @@ private slots: private: void AddToBlockSizeComboBox(int size); - void AddToCompressionComboBox(const QString& name, DiscIO::WIACompressionType type); + void AddToCompressionComboBox(const QString& name, DiscIO::WIARVZCompressionType type); void AddToCompressionLevelComboBox(int level); bool ShowAreYouSureDialog(const QString& text); From 224c6e799d4840dd4993c3f3246e78b5c89a4096 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 17 May 2020 15:50:41 +0200 Subject: [PATCH 35/36] RVZ: Extend GroupEntry --- Source/Core/DiscIO/WIABlob.cpp | 159 +++++++++++++++++++------- Source/Core/DiscIO/WIABlob.h | 44 +++++-- Source/Core/DiscIO/WIACompression.cpp | 25 +++- Source/Core/DiscIO/WIACompression.h | 5 +- docs/WIA.md | 17 ++- 5 files changed, 191 insertions(+), 59 deletions(-) diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 6af49b202f23..6f64b3ba01a6 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -210,9 +210,10 @@ bool WIARVZFileReader::Initialize(const std::string& path) const u32 number_of_raw_data_entries = Common::swap32(m_header_2.number_of_raw_data_entries); m_raw_data_entries.resize(number_of_raw_data_entries); - Chunk& raw_data_entries = ReadCompressedData(Common::swap64(m_header_2.raw_data_entries_offset), - Common::swap32(m_header_2.raw_data_entries_size), - number_of_raw_data_entries * sizeof(RawDataEntry)); + Chunk& raw_data_entries = + ReadCompressedData(Common::swap64(m_header_2.raw_data_entries_offset), + Common::swap32(m_header_2.raw_data_entries_size), + number_of_raw_data_entries * sizeof(RawDataEntry), m_compression_type); if (!raw_data_entries.ReadAll(&m_raw_data_entries)) return false; @@ -226,9 +227,10 @@ bool WIARVZFileReader::Initialize(const std::string& path) const u32 number_of_group_entries = Common::swap32(m_header_2.number_of_group_entries); m_group_entries.resize(number_of_group_entries); - Chunk& group_entries = ReadCompressedData(Common::swap64(m_header_2.group_entries_offset), - Common::swap32(m_header_2.group_entries_size), - number_of_group_entries * sizeof(GroupEntry)); + Chunk& group_entries = + ReadCompressedData(Common::swap64(m_header_2.group_entries_offset), + Common::swap32(m_header_2.group_entries_size), + number_of_group_entries * sizeof(GroupEntry), m_compression_type); if (!group_entries.ReadAll(&m_group_entries)) return false; @@ -463,7 +465,20 @@ bool WIARVZFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, chunk_size = std::min(chunk_size, data_size - group_offset_in_data); const u64 bytes_to_read = std::min(chunk_size - offset_in_group, *size); - const u32 group_data_size = Common::swap32(group.data_size); + u32 group_data_size = Common::swap32(group.data_size); + + WIARVZCompressionType compression_type = m_compression_type; + u32 rvz_packed_size = 0; + if constexpr (RVZ) + { + if ((group_data_size & 0x80000000) == 0) + compression_type = WIARVZCompressionType::None; + + group_data_size &= 0x7FFFFFFF; + + rvz_packed_size = Common::swap32(group.rvz_packed_size); + } + if (group_data_size == 0) { std::memset(*out_ptr, 0, bytes_to_read); @@ -471,8 +486,11 @@ bool WIARVZFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, else { const u64 group_offset_in_file = static_cast(Common::swap32(group.data_offset)) << 2; - Chunk& chunk = ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, - exception_lists, RVZ, group_offset_in_data); + + Chunk& chunk = + ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, compression_type, + exception_lists, rvz_packed_size, group_offset_in_data); + if (!chunk.Read(offset_in_group, bytes_to_read, *out_ptr)) { m_cached_chunk_offset = std::numeric_limits::max(); // Invalidate the cache @@ -501,20 +519,22 @@ bool WIARVZFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, template typename WIARVZFileReader::Chunk& WIARVZFileReader::ReadCompressedData(u64 offset_in_file, u64 compressed_size, - u64 decompressed_size, u32 exception_lists, bool rvz_pack, - u64 data_offset) + u64 decompressed_size, + WIARVZCompressionType compression_type, + u32 exception_lists, u32 rvz_packed_size, u64 data_offset) { if (offset_in_file == m_cached_chunk_offset) return m_cached_chunk; std::unique_ptr decompressor; - switch (m_compression_type) + switch (compression_type) { case WIARVZCompressionType::None: decompressor = std::make_unique(); break; case WIARVZCompressionType::Purge: - decompressor = std::make_unique(decompressed_size); + decompressor = std::make_unique(rvz_packed_size == 0 ? decompressed_size : + rvz_packed_size); break; case WIARVZCompressionType::Bzip2: decompressor = std::make_unique(); @@ -532,11 +552,11 @@ WIARVZFileReader::ReadCompressedData(u64 offset_in_file, u64 compressed_siz break; } - const bool compressed_exception_lists = m_compression_type > WIARVZCompressionType::Purge; + const bool compressed_exception_lists = compression_type > WIARVZCompressionType::Purge; m_cached_chunk = Chunk(&m_file, offset_in_file, compressed_size, decompressed_size, exception_lists, - compressed_exception_lists, rvz_pack, data_offset, std::move(decompressor)); + compressed_exception_lists, rvz_packed_size, data_offset, std::move(decompressor)); m_cached_chunk_offset = offset_in_file; return m_cached_chunk; } @@ -561,10 +581,10 @@ WIARVZFileReader::Chunk::Chunk() = default; template WIARVZFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size, u32 exception_lists, - bool compressed_exception_lists, bool rvz_pack, u64 data_offset, - std::unique_ptr decompressor) + bool compressed_exception_lists, u32 rvz_packed_size, + u64 data_offset, std::unique_ptr decompressor) : m_file(file), m_offset_in_file(offset_in_file), m_exception_lists(exception_lists), - m_compressed_exception_lists(compressed_exception_lists), m_rvz_pack(rvz_pack), + m_compressed_exception_lists(compressed_exception_lists), m_rvz_packed_size(rvz_packed_size), m_data_offset(data_offset), m_decompressor(std::move(decompressor)) { constexpr size_t MAX_SIZE_PER_EXCEPTION_LIST = @@ -655,7 +675,7 @@ bool WIARVZFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) return false; } - if (m_rvz_pack && m_exception_lists == 0) + if (m_rvz_packed_size != 0 && m_exception_lists == 0) { if (!Decompress()) return false; @@ -691,10 +711,8 @@ bool WIARVZFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) template bool WIARVZFileReader::Chunk::Decompress() { - if (m_rvz_pack && m_exception_lists == 0) + if (m_rvz_packed_size != 0 && m_exception_lists == 0) { - m_rvz_pack = false; - const size_t bytes_to_move = m_out.bytes_written - m_out_bytes_used_for_exceptions; DecompressionBuffer in{std::vector(bytes_to_move), bytes_to_move}; @@ -703,7 +721,9 @@ bool WIARVZFileReader::Chunk::Decompress() m_out.bytes_written = m_out_bytes_used_for_exceptions; m_decompressor = std::make_unique(std::move(m_decompressor), std::move(in), - m_data_offset); + m_data_offset, m_rvz_packed_size); + + m_rvz_packed_size = 0; } return m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read); @@ -1069,8 +1089,8 @@ static bool AllSame(const u8* begin, const u8* end) template static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, size_t chunks, - u64 total_size, u64 data_offset, u64 in_offset, bool allow_junk_reuse, - bool compression, const FileSystem* file_system) + u64 total_size, u64 data_offset, u64 in_offset, bool multipart, + bool allow_junk_reuse, bool compression, const FileSystem* file_system) { using Seed = std::array; struct JunkInfo @@ -1148,6 +1168,11 @@ static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chun const bool store_junk_efficiently = allow_junk_reuse || !entry.reuse_id; + // TODO: It would be possible to support skipping RVZ packing even when the chunk size is larger + // than 2 MiB (multipart == true), but it would be more effort than it's worth since Dolphin's + // converter doesn't expose chunk sizes larger than 2 MiB to the user anyway + bool first_loop_iteration = !multipart; + while (current_offset < end_offset) { u64 next_junk_start = end_offset; @@ -1165,6 +1190,18 @@ static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chun } } + if (first_loop_iteration) + { + if (next_junk_start == end_offset) + { + // Storing this chunk without RVZ packing would be inefficient, so store it without + PushBack(&entry.main_data, in + in_offset + current_offset, in + in_offset + end_offset); + break; + } + + first_loop_iteration = false; + } + const u64 non_junk_bytes = next_junk_start - current_offset; if (non_junk_bytes > 0) { @@ -1174,6 +1211,7 @@ static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chun PushBack(&entry.main_data, ptr, ptr + non_junk_bytes); current_offset += non_junk_bytes; + entry.rvz_packed_size += sizeof(u32) + non_junk_bytes; } const u64 junk_bytes = next_junk_end - current_offset; @@ -1183,6 +1221,7 @@ static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chun PushBack(&entry.main_data, *seed); current_offset += junk_bytes; + entry.rvz_packed_size += sizeof(u32) + SEED_SIZE; } } } @@ -1192,7 +1231,8 @@ template static void RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset, bool allow_junk_reuse, bool compression, const FileSystem* file_system) { - RVZPack(in, out, size, 1, size, data_offset, 0, allow_junk_reuse, compression, file_system); + RVZPack(in, out, size, 1, size, data_offset, 0, false, allow_junk_reuse, compression, + file_system); } template @@ -1381,7 +1421,7 @@ WIARVZFileReader::ProcessAndCompress(CompressThreadState* state, CompressPa RVZPack(state->decryption_buffer[0].data(), output_entries.data() + first_chunk, bytes_per_chunk, chunks, total_size, data_offset, write_offset_of_group, - allow_junk_reuse, compression, file_system); + groups > 1, allow_junk_reuse, compression, file_system); } else { @@ -1462,9 +1502,19 @@ WIARVZFileReader::ProcessAndCompress(CompressThreadState* state, CompressPa { entry.exception_lists.clear(); entry.main_data.clear(); + if constexpr (RVZ) + { + entry.rvz_packed_size = 0; + entry.compressed = false; + } continue; } + const auto pad_exception_lists = [&entry]() { + while (entry.exception_lists.size() % 4 != 0) + entry.exception_lists.push_back(0); + }; + if (state->compressor) { if (!state->compressor->Start()) @@ -1480,16 +1530,11 @@ WIARVZFileReader::ProcessAndCompress(CompressThreadState* state, CompressPa { return ConversionResultCode::InternalError; } - - entry.exception_lists.clear(); } else { if (!compressed_exception_lists) - { - while (entry.exception_lists.size() % 4 != 0) - entry.exception_lists.push_back(0); - } + pad_exception_lists(); if (state->compressor) { @@ -1510,13 +1555,30 @@ WIARVZFileReader::ProcessAndCompress(CompressThreadState* state, CompressPa return ConversionResultCode::InternalError; } - if (state->compressor) + bool compressed = !!state->compressor; + if constexpr (RVZ) + { + size_t uncompressed_size = entry.main_data.size(); + if (compressed_exception_lists) + uncompressed_size += Common::AlignUp(entry.exception_lists.size(), 4); + + compressed = state->compressor && state->compressor->GetSize() < uncompressed_size; + entry.compressed = compressed; + + if (!compressed) + pad_exception_lists(); + } + + if (compressed) { const u8* data = state->compressor->GetData(); const size_t size = state->compressor->GetSize(); entry.main_data.resize(size); std::copy(data, data + size, entry.main_data.data()); + + if (compressed_exception_lists) + entry.exception_lists.clear(); } } @@ -1540,21 +1602,26 @@ ConversionResultCode WIARVZFileReader::Output(std::vector> 2 > std::numeric_limits::max()) return ConversionResultCode::InternalError; ASSERT((*bytes_written & 3) == 0); group_entry->data_offset = Common::swap32(static_cast(*bytes_written >> 2)); - group_entry->data_size = Common::swap32(static_cast(data_size)); + + u32 data_size = static_cast(entry.exception_lists.size() + entry.main_data.size()); + if constexpr (RVZ) + { + data_size = (data_size & 0x7FFFFFFF) | (static_cast(entry.compressed) << 31); + group_entry->rvz_packed_size = Common::swap32(static_cast(entry.rvz_packed_size)); + } + group_entry->data_size = Common::swap32(data_size); if (!outfile->WriteArray(entry.exception_lists.data(), entry.exception_lists.size())) return ConversionResultCode::WriteFailed; if (!outfile->WriteArray(entry.main_data.data(), entry.main_data.size())) return ConversionResultCode::WriteFailed; - *bytes_written += data_size; + *bytes_written += entry.exception_lists.size() + entry.main_data.size(); if (entry.reuse_id) { @@ -1659,10 +1726,18 @@ WIARVZFileReader::Convert(BlobReader* infile, const VolumeDisc* infile_volu // Conservative estimate for how much space will be taken up by headers. // The compression methods None and Purge have very predictable overhead, // and the other methods are able to compress group entries well - const u64 headers_size_upper_bound = - Common::AlignUp(sizeof(WIAHeader1) + sizeof(WIAHeader2) + partition_entries_size + - raw_data_entries_size + group_entries_size + 0x100, - VolumeWii::BLOCK_TOTAL_SIZE); + const u64 headers_size_upper_bound = [&] { + u64 upper_bound = sizeof(WIAHeader1) + sizeof(WIAHeader2) + partition_entries_size + + raw_data_entries_size + 0x100; + + // RVZ's added data in GroupEntry usually compresses well + if (RVZ && compression_type > WIARVZCompressionType::Purge) + upper_bound += group_entries_size / 2; + else + upper_bound += group_entries_size; + + return Common::AlignUp(upper_bound, VolumeWii::BLOCK_TOTAL_SIZE); + }(); std::vector buffer; diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index d7e3573e7cb9..458b05e8e7c6 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -138,12 +138,22 @@ class WIARVZFileReader : public BlobReader }; static_assert(sizeof(RawDataEntry) == 0x18, "Wrong size for WIA raw data entry"); - struct GroupEntry + struct WIAGroupEntry { u32 data_offset; // >> 2 u32 data_size; }; - static_assert(sizeof(GroupEntry) == 0x08, "Wrong size for WIA group entry"); + static_assert(sizeof(WIAGroupEntry) == 0x08, "Wrong size for WIA group entry"); + + struct RVZGroupEntry + { + u32 data_offset; // >> 2 + u32 data_size; + u32 rvz_packed_size; + }; + static_assert(sizeof(RVZGroupEntry) == 0x0c, "Wrong size for RVZ group entry"); + + using GroupEntry = std::conditional_t; struct HashExceptionEntry { @@ -172,8 +182,8 @@ class WIARVZFileReader : public BlobReader public: Chunk(); Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size, - u32 exception_lists, bool compressed_exception_lists, bool rvz_pack, u64 data_offset, - std::unique_ptr decompressor); + u32 exception_lists, bool compressed_exception_lists, u32 rvz_packed_size, + u64 data_offset, std::unique_ptr decompressor); bool Read(u64 offset, u64 size, u8* out_ptr); @@ -205,7 +215,7 @@ class WIARVZFileReader : public BlobReader size_t m_in_bytes_used_for_exceptions = 0; u32 m_exception_lists = 0; bool m_compressed_exception_lists = false; - bool m_rvz_pack = false; + u32 m_rvz_packed_size = 0; u64 m_data_offset = 0; }; @@ -217,7 +227,8 @@ class WIARVZFileReader : public BlobReader u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups, u32 exception_lists); Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size, - u32 exception_lists = 0, bool rvz_pack = false, u64 data_offset = 0); + WIARVZCompressionType compression_type, u32 exception_lists = 0, + u32 rvz_packed_size = 0, u64 data_offset = 0); static bool ApplyHashExceptions(const std::vector& exception_list, VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]); @@ -273,14 +284,27 @@ class WIARVZFileReader : public BlobReader size_t group_index; }; - struct OutputParametersEntry + struct WIAOutputParametersEntry + { + std::vector exception_lists; + std::vector main_data; + std::optional reuse_id; + std::optional reused_group; + }; + + struct RVZOutputParametersEntry { std::vector exception_lists; std::vector main_data; std::optional reuse_id; std::optional reused_group; + size_t rvz_packed_size = 0; + bool compressed = false; }; + using OutputParametersEntry = + std::conditional_t; + struct OutputParameters { std::vector entries; @@ -355,9 +379,9 @@ class WIARVZFileReader : public BlobReader static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000; static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000; - static constexpr u32 RVZ_VERSION = 0x00020000; - static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00020000; - static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00020000; + static constexpr u32 RVZ_VERSION = 0x00030000; + static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00030000; + static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00030000; }; using WIAFileReader = WIARVZFileReader; diff --git a/Source/Core/DiscIO/WIACompression.cpp b/Source/Core/DiscIO/WIACompression.cpp index c15380a3437a..d6d3acae2bae 100644 --- a/Source/Core/DiscIO/WIACompression.cpp +++ b/Source/Core/DiscIO/WIACompression.cpp @@ -292,10 +292,18 @@ bool ZstdDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBu } RVZPackDecompressor::RVZPackDecompressor(std::unique_ptr decompressor, - DecompressionBuffer decompressed, u64 data_offset) + DecompressionBuffer decompressed, u64 data_offset, + u32 rvz_packed_size) : m_decompressor(std::move(decompressor)), m_decompressed(std::move(decompressed)), - m_data_offset(data_offset) + m_data_offset(data_offset), m_rvz_packed_size(rvz_packed_size) { + m_bytes_read = m_decompressed.bytes_written; +} + +bool RVZPackDecompressor::IncrementBytesRead(size_t x) +{ + m_bytes_read += x; + return m_bytes_read <= m_rvz_packed_size; } std::optional RVZPackDecompressor::ReadToDecompressed(const DecompressionBuffer& in, @@ -308,9 +316,14 @@ std::optional RVZPackDecompressor::ReadToDecompressed(const DecompressionB if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read) { + const size_t prev_bytes_written = m_decompressed.bytes_written; + if (!m_decompressor->Decompress(in, &m_decompressed, in_bytes_read)) return false; + if (!IncrementBytesRead(m_decompressed.bytes_written - prev_bytes_written)) + return false; + if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read) return true; } @@ -395,6 +408,10 @@ bool RVZPackDecompressor::Decompress(const DecompressionBuffer& in, Decompressio out->data.resize(old_out_size); bytes_to_write = out->bytes_written - prev_out_bytes_written; + + if (!IncrementBytesRead(bytes_to_write)) + return false; + if (bytes_to_write == 0) return true; } @@ -417,8 +434,8 @@ bool RVZPackDecompressor::Decompress(const DecompressionBuffer& in, Decompressio bool RVZPackDecompressor::Done() const { - return m_size == 0 && m_decompressed.bytes_written == m_decompressed_bytes_read && - m_decompressor->Done(); + return m_size == 0 && m_rvz_packed_size == m_bytes_read && + m_decompressed.bytes_written == m_decompressed_bytes_read && m_decompressor->Done(); } Compressor::~Compressor() = default; diff --git a/Source/Core/DiscIO/WIACompression.h b/Source/Core/DiscIO/WIACompression.h index 5015802802c3..37e8cf3dc3f3 100644 --- a/Source/Core/DiscIO/WIACompression.h +++ b/Source/Core/DiscIO/WIACompression.h @@ -122,7 +122,7 @@ class RVZPackDecompressor final : public Decompressor { public: RVZPackDecompressor(std::unique_ptr decompressor, DecompressionBuffer decompressed, - u64 data_offset); + u64 data_offset, u32 rvz_packed_size); bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out, size_t* in_bytes_read) override; @@ -130,13 +130,16 @@ class RVZPackDecompressor final : public Decompressor bool Done() const override; private: + bool IncrementBytesRead(size_t x); std::optional ReadToDecompressed(const DecompressionBuffer& in, size_t* in_bytes_read, size_t decompressed_bytes_read, size_t bytes_to_read); std::unique_ptr m_decompressor; DecompressionBuffer m_decompressed; size_t m_decompressed_bytes_read = 0; + size_t m_bytes_read; u64 m_data_offset; + u32 m_rvz_packed_size; u32 m_size = 0; bool m_junk; diff --git a/docs/WIA.md b/docs/WIA.md index 16688d5bddea..aa8120b6d278 100644 --- a/docs/WIA.md +++ b/docs/WIA.md @@ -178,11 +178,24 @@ RVZ is a file format which is closely based on WIA. The differences are as follo * Chunk sizes smaller than 2 MiB are supported. The following applies when using a chunk size smaller than 2 MiB: * The chunk size must be at least 32 KiB and must be a power of two. (Just like with WIA, sizes larger than 2 MiB do not have to be a power of two, they just have to be an integer multiple of 2 MiB.) * For Wii partition data, each chunk contains one `wia_except_list_t` which contains exceptions for that chunk (and no other chunks). Offset 0 refers to the first hash of the current chunk, not the first hash of the full 2 MiB of data. -* An encoding scheme which is described below is used to store pseudorandom padding data losslessly. +* The `wia_group_t` struct has been expanded. See the `rvz_group_t` section below. +* Pseudorandom padding data is stored losslessly using an encoding scheme described in the *RVZ packing* section below. + +## `rvz_group_t` + +Compared to `wia_group_t`, `rvz_group_t` changes the meaning of the most significant bit of `data_size` and adds one additional attribute. + +"Compressed data" below means the data as it is stored in the file. When compression is disabled, this "compressed data" is actually not compressed. + +|Type and name|Description| +|--|--| +|`u32 data_off4`|The offset in the file where the compressed data is stored, divided by 4.| +|`u32 data_size`|The most significant bit is 1 if the data is compressed using the compression method indicated in `wia_disc_t`, and 0 if it is not compressed. The lower 31 bits are the size of the compressed data, including any `wia_except_list_t` structs. The lower 31 bits being 0 is a special case meaning that every byte of the decompressed and unpacked data is `0x00` and the `wia_except_list_t` structs (if there are supposed to be any) contain 0 exceptions.| +|`u32 rvz_packed_size`|The size after decompressing but before decoding the RVZ packing. If this is 0, RVZ packing is not used for this group.| ## RVZ packing -The RVZ packing encoding scheme is applied to all `wia_group_t` data, with any bzip2/LZMA/Zstandard compression being applied on top of it. (In other words, when reading an RVZ file, bzip2/LZMA/Zstandard decompression is done before decoding the RVZ packing.) RVZ packed data can be decoded as follows: +The RVZ packing encoding scheme can be applied to `wia_group_t` data, with any bzip2/LZMA/Zstandard compression being applied on top of it. (In other words, when reading an RVZ file, bzip2/LZMA/Zstandard decompression is done before decoding the RVZ packing.) RVZ packed data can be decoded as follows: 1. Read 4 bytes of data and interpret it as a 32-bit unsigned big endian integer. Call this `size`. 2. If the most significant bit of `size` is not set, read `size` bytes and output them unchanged. If the most significant bit of `size` is set, unset the most significant bit of `size`, then read 68 bytes of PRNG seed data and output `size` bytes using the PRNG algorithm described below. From 660d81a10b4a8ff9a61631e69fb93e9010bccbc0 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 21 Jun 2020 12:38:03 +0200 Subject: [PATCH 36/36] RVZ: Bump version number to 1.0 --- Source/Core/DiscIO/WIABlob.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 458b05e8e7c6..789a3e7dd58a 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -379,7 +379,7 @@ class WIARVZFileReader : public BlobReader static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000; static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000; - static constexpr u32 RVZ_VERSION = 0x00030000; + static constexpr u32 RVZ_VERSION = 0x01000000; static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00030000; static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00030000; };