Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #8538 from JosJuice/wia
Add support for the WIA and RVZ disc image formats
  • Loading branch information
Tilka committed Jun 21, 2020
2 parents 03e0d2c + 660d81a commit 9982251
Show file tree
Hide file tree
Showing 25 changed files with 4,334 additions and 98 deletions.
Expand Up @@ -22,7 +22,7 @@
public final class FileBrowserHelper
{
public static final HashSet<String> GAME_EXTENSIONS = new HashSet<>(Arrays.asList(
"gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wad", "dol", "elf", "dff"));
"gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "rvz", "wad", "dol", "elf", "dff"));

public static final HashSet<String> RAW_EXTENSION = new HashSet<>(Collections.singletonList(
"raw"));
Expand Down
2 changes: 1 addition & 1 deletion Source/Core/Core/Boot/Boot.cpp
Expand Up @@ -159,7 +159,7 @@ BootParameters::GenerateFromFile(std::vector<std::string> paths,
paths.clear();

static const std::unordered_set<std::string> disc_image_extensions = {
{".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".dol", ".elf"}};
{".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".rvz", ".dol", ".elf"}};
if (disc_image_extensions.find(extension) != disc_image_extensions.end() || is_drive)
{
std::unique_ptr<DiscIO::VolumeDisc> disc = DiscIO::CreateDisc(path);
Expand Down
5 changes: 5 additions & 0 deletions Source/Core/DiscIO/Blob.cpp
Expand Up @@ -20,6 +20,7 @@
#include "DiscIO/DriveBlob.h"
#include "DiscIO/FileBlob.h"
#include "DiscIO/TGCBlob.h"
#include "DiscIO/WIABlob.h"
#include "DiscIO/WbfsBlob.h"

namespace DiscIO
Expand Down Expand Up @@ -205,6 +206,10 @@ std::unique_ptr<BlobReader> CreateBlobReader(const std::string& filename)
return TGCFileReader::Create(std::move(file));
case WBFS_MAGIC:
return WbfsFileReader::Create(std::move(file), filename);
case WIA_MAGIC:
return WIAFileReader::Create(std::move(file), filename);
case RVZ_MAGIC:
return RVZFileReader::Create(std::move(file), filename);
default:
if (auto directory_blob = DirectoryBlobReader::Create(filename))
return std::move(directory_blob);
Expand Down
10 changes: 9 additions & 1 deletion Source/Core/DiscIO/Blob.h
Expand Up @@ -25,6 +25,8 @@

namespace DiscIO
{
enum class WIARVZCompressionType : u32;

// Increment CACHE_REVISION (GameFileCache.cpp) if the enum below is modified
enum class BlobType
{
Expand All @@ -34,7 +36,9 @@ enum class BlobType
GCZ,
CISO,
WBFS,
TGC
TGC,
WIA,
RVZ,
};

class BlobReader
Expand Down Expand Up @@ -172,5 +176,9 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path,
bool ConvertToPlain(BlobReader* infile, const std::string& infile_path,
const std::string& outfile_path, CompressCB callback = nullptr,
void* arg = nullptr);
bool ConvertToWIAOrRVZ(BlobReader* infile, const std::string& infile_path,
const std::string& outfile_path, bool rvz,
WIARVZCompressionType compression_type, int compression_level,
int chunk_size, CompressCB callback = nullptr, void* arg = nullptr);

} // namespace DiscIO
11 changes: 11 additions & 0 deletions Source/Core/DiscIO/CMakeLists.txt
Expand Up @@ -21,6 +21,8 @@ add_library(discio
FileSystemGCWii.h
Filesystem.cpp
Filesystem.h
LaggedFibonacciGenerator.cpp
LaggedFibonacciGenerator.h
MultithreadedCompressor.h
NANDImporter.cpp
NANDImporter.h
Expand All @@ -42,13 +44,22 @@ add_library(discio
VolumeWii.h
WbfsBlob.cpp
WbfsBlob.h
WIABlob.cpp
WIABlob.h
WIACompression.cpp
WIACompression.h
WiiEncryptionCache.cpp
WiiEncryptionCache.h
WiiSaveBanner.cpp
WiiSaveBanner.h
)

target_link_libraries(discio
PUBLIC
BZip2::BZip2
LibLZMA::LibLZMA
zstd

PRIVATE
minizip
pugixml
Expand Down
15 changes: 15 additions & 0 deletions Source/Core/DiscIO/DiscIO.vcxproj
Expand Up @@ -55,6 +55,7 @@
<ClCompile Include="FileBlob.cpp" />
<ClCompile Include="Filesystem.cpp" />
<ClCompile Include="FileSystemGCWii.cpp" />
<ClCompile Include="LaggedFibonacciGenerator.cpp" />
<ClCompile Include="NANDImporter.cpp" />
<ClCompile Include="ScrubbedBlob.cpp" />
<ClCompile Include="TGCBlob.cpp" />
Expand All @@ -65,6 +66,8 @@
<ClCompile Include="VolumeWad.cpp" />
<ClCompile Include="VolumeWii.cpp" />
<ClCompile Include="WbfsBlob.cpp" />
<ClCompile Include="WIABlob.cpp" />
<ClCompile Include="WIACompression.cpp" />
<ClCompile Include="WiiEncryptionCache.cpp" />
<ClCompile Include="WiiSaveBanner.cpp" />
</ItemGroup>
Expand All @@ -80,6 +83,7 @@
<ClInclude Include="FileBlob.h" />
<ClInclude Include="Filesystem.h" />
<ClInclude Include="FileSystemGCWii.h" />
<ClInclude Include="LaggedFibonacciGenerator.h" />
<ClInclude Include="MultithreadedCompressor.h" />
<ClInclude Include="NANDImporter.h" />
<ClInclude Include="ScrubbedBlob.h" />
Expand All @@ -91,6 +95,8 @@
<ClInclude Include="VolumeWad.h" />
<ClInclude Include="VolumeWii.h" />
<ClInclude Include="WbfsBlob.h" />
<ClInclude Include="WIABlob.h" />
<ClInclude Include="WIACompression.h" />
<ClInclude Include="WiiEncryptionCache.h" />
<ClInclude Include="WiiSaveBanner.h" />
</ItemGroup>
Expand All @@ -110,6 +116,15 @@
<ProjectReference Include="$(ExternalsDir)pugixml\pugixml.vcxproj">
<Project>{38fee76f-f347-484b-949c-b4649381cffb}</Project>
</ProjectReference>
<ProjectReference Include="$(ExternalsDir)bzip2\bzip2.vcxproj">
<Project>{055a775f-b4f5-4970-9240-f6cf7661f37b}</Project>
</ProjectReference>
<ProjectReference Include="$(ExternalsDir)liblzma\liblzma.vcxproj">
<Project>{1d8c51d2-ffa4-418e-b183-9f42b6a6717e}</Project>
</ProjectReference>
<ProjectReference Include="$(ExternalsDir)zstd\zstd.vcxproj">
<Project>{1bea10f3-80ce-4bc4-9331-5769372cdf99}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
Expand Down
18 changes: 18 additions & 0 deletions Source/Core/DiscIO/DiscIO.vcxproj.filters
Expand Up @@ -90,6 +90,15 @@
<ClCompile Include="ScrubbedBlob.cpp">
<Filter>Volume\Blob</Filter>
</ClCompile>
<ClCompile Include="WIABlob.cpp">
<Filter>Volume\Blob</Filter>
</ClCompile>
<ClCompile Include="LaggedFibonacciGenerator.cpp">
<Filter>Volume\Blob</Filter>
</ClCompile>
<ClCompile Include="WIACompression.cpp">
<Filter>Volume\Blob</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="DiscScrubber.h">
Expand Down Expand Up @@ -164,6 +173,15 @@
<ClInclude Include="MultithreadedCompressor.h">
<Filter>Volume\Blob</Filter>
</ClInclude>
<ClInclude Include="WIABlob.h">
<Filter>Volume\Blob</Filter>
</ClInclude>
<ClInclude Include="LaggedFibonacciGenerator.h">
<Filter>Volume\Blob</Filter>
</ClInclude>
<ClInclude Include="WIACompression.h">
<Filter>Volume\Blob</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Text Include="CMakeLists.txt" />
Expand Down
212 changes: 212 additions & 0 deletions Source/Core/DiscIO/LaggedFibonacciGenerator.cpp
@@ -0,0 +1,212 @@
// This file is under the public domain.

#include "DiscIO/LaggedFibonacciGenerator.h"

#include <algorithm>
#include <cstddef>
#include <cstring>

#include "Common/Align.h"
#include "Common/Assert.h"
#include "Common/CommonTypes.h"
#include "Common/Swap.h"

namespace DiscIO
{
void LaggedFibonacciGenerator::SetSeed(const u32 seed[SEED_SIZE])
{
SetSeed(reinterpret_cast<const u8*>(seed));
}

void LaggedFibonacciGenerator::SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)])
{
m_position_bytes = 0;

for (size_t i = 0; i < SEED_SIZE; ++i)
m_buffer[i] = Common::swap32(seed + i * sizeof(u32));

Initialize(false);
}

size_t LaggedFibonacciGenerator::GetSeed(const u8* data, size_t size, size_t data_offset,
u32 seed_out[SEED_SIZE])
{
if ((reinterpret_cast<uintptr_t>(data) - data_offset) % alignof(u32) != 0)
{
ASSERT(false);
return 0;
}

// For code simplicity, only include whole u32 words when regenerating the seed. It would be
// possible to get rid of this restriction and use a few additional bytes, but it's probably more
// effort than it's worth considering that junk data often starts or ends on 4-byte offsets.
const size_t bytes_to_skip = Common::AlignUp(data_offset, sizeof(u32)) - data_offset;
const u32* u32_data = reinterpret_cast<const u32*>(data + bytes_to_skip);
const size_t u32_size = (size - bytes_to_skip) / sizeof(u32);
const size_t u32_data_offset = (data_offset + bytes_to_skip) / sizeof(u32);

LaggedFibonacciGenerator lfg;
if (!GetSeed(u32_data, u32_size, u32_data_offset, &lfg, seed_out))
return false;

lfg.m_position_bytes = data_offset % (LFG_K * sizeof(u32));

const u8* end = data + size;
size_t reconstructed_bytes = 0;
while (data < end && lfg.GetByte() == *data)
{
++reconstructed_bytes;
++data;
}
return reconstructed_bytes;
}

bool LaggedFibonacciGenerator::GetSeed(const u32* data, size_t size, size_t data_offset,
LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE])
{
if (size < LFG_K)
return false;

// If the data doesn't look like something we can regenerate, return early to save time
if (!std::all_of(data, data + LFG_K, [](u32 x) {
return (Common::swap32(x) & 0x00C00000) == (Common::swap32(x) >> 2 & 0x00C00000);
}))
{
return false;
}

const size_t data_offset_mod_k = data_offset % LFG_K;
const size_t data_offset_div_k = data_offset / LFG_K;

std::copy(data, data + LFG_K - data_offset_mod_k, lfg->m_buffer.data() + data_offset_mod_k);
std::copy(data + LFG_K - data_offset_mod_k, data + LFG_K, lfg->m_buffer.data());

lfg->Backward(0, data_offset_mod_k);

for (size_t i = 0; i < data_offset_div_k; ++i)
lfg->Backward();

if (!lfg->Reinitialize(seed_out))
return false;

for (size_t i = 0; i < data_offset_div_k; ++i)
lfg->Forward();

return true;
}

void LaggedFibonacciGenerator::GetBytes(size_t count, u8* out)
{
while (count > 0)
{
const size_t length = std::min(count, LFG_K * sizeof(u32) - m_position_bytes);

std::memcpy(out, reinterpret_cast<u8*>(m_buffer.data()) + m_position_bytes, length);

m_position_bytes += length;
count -= length;
out += length;

if (m_position_bytes == LFG_K * sizeof(u32))
{
Forward();
m_position_bytes = 0;
}
}
}

u8 LaggedFibonacciGenerator::GetByte()
{
const u8 result = reinterpret_cast<u8*>(m_buffer.data())[m_position_bytes];

++m_position_bytes;

if (m_position_bytes == LFG_K * sizeof(u32))
{
Forward();
m_position_bytes = 0;
}

return result;
}

void LaggedFibonacciGenerator::Forward(size_t count)
{
m_position_bytes += count;
while (m_position_bytes >= LFG_K * sizeof(u32))
{
Forward();
m_position_bytes -= LFG_K * sizeof(u32);
}
}

void LaggedFibonacciGenerator::Forward()
{
for (size_t i = 0; i < LFG_J; ++i)
m_buffer[i] ^= m_buffer[i + LFG_K - LFG_J];

for (size_t i = LFG_J; i < LFG_K; ++i)
m_buffer[i] ^= m_buffer[i - LFG_J];
}

void LaggedFibonacciGenerator::Backward(size_t start_word, size_t end_word)
{
const size_t loop_end = std::max(LFG_J, start_word);
for (size_t i = std::min(end_word, LFG_K); i > loop_end; --i)
m_buffer[i - 1] ^= m_buffer[i - 1 - LFG_J];

for (size_t i = std::min(end_word, LFG_J); i > start_word; --i)
m_buffer[i - 1] ^= m_buffer[i - 1 + LFG_K - LFG_J];
}

bool LaggedFibonacciGenerator::Reinitialize(u32 seed_out[SEED_SIZE])
{
for (size_t i = 0; i < 4; ++i)
Backward();

for (u32& x : m_buffer)
x = Common::swap32(x);

// Reconstruct the bits which are missing due to the output code shifting by 18 instead of 16.
// Unfortunately we can't reconstruct bits 16 and 17 (counting LSB as 0) for the first word,
// but the observable result (when shifting by 18 instead of 16) is not affected by this.
for (size_t i = 0; i < SEED_SIZE; ++i)
{
m_buffer[i] = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000) |
((m_buffer[i + 16] ^ m_buffer[i + 15]) << 9 & 0x00030000);
}

for (size_t i = 0; i < SEED_SIZE; ++i)
seed_out[i] = Common::swap32(m_buffer[i]);

return Initialize(true);
}

bool LaggedFibonacciGenerator::Initialize(bool check_existing_data)
{
for (size_t i = SEED_SIZE; i < LFG_K; ++i)
{
const u32 calculated = (m_buffer[i - 17] << 23) ^ (m_buffer[i - 16] >> 9) ^ m_buffer[i - 1];

if (check_existing_data)
{
const u32 actual = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000);
if ((calculated & 0xFFFCFFFF) != actual)
return false;
}

m_buffer[i] = calculated;
}

// Instead of doing the "shift by 18 instead of 16" oddity when actually outputting the data,
// we can do the shifting (and byteswapping) at this point to make the output code simpler.
for (u32& x : m_buffer)
x = Common::swap32((x & 0xFF00FFFF) | ((x >> 2) & 0x00FF0000));

for (size_t i = 0; i < 4; ++i)
Forward();

return true;
}

} // namespace DiscIO

0 comments on commit 9982251

Please sign in to comment.