Add support for the WIA and RVZ disc image formats

Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java

@@ -22,7 +22,7 @@
 public final class FileBrowserHelper
 {
   public static final HashSet<String> GAME_EXTENSIONS = new HashSet<>(Arrays.asList(
-          "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wad", "dol", "elf", "dff"));
+          "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "rvz", "wad", "dol", "elf", "dff"));
 
   public static final HashSet<String> RAW_EXTENSION = new HashSet<>(Collections.singletonList(
           "raw"));

Source/Core/Core/Boot/Boot.cpp

@@ -159,7 +159,7 @@ BootParameters::GenerateFromFile(std::vector<std::string> paths,
     paths.clear();
 
   static const std::unordered_set<std::string> disc_image_extensions = {
-      {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".dol", ".elf"}};
+      {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".rvz", ".dol", ".elf"}};
   if (disc_image_extensions.find(extension) != disc_image_extensions.end() || is_drive)
   {
     std::unique_ptr<DiscIO::VolumeDisc> disc = DiscIO::CreateDisc(path);

Source/Core/DiscIO/Blob.cpp

@@ -20,6 +20,7 @@
 #include "DiscIO/DriveBlob.h"
 #include "DiscIO/FileBlob.h"
 #include "DiscIO/TGCBlob.h"
+#include "DiscIO/WIABlob.h"
 #include "DiscIO/WbfsBlob.h"
 
 namespace DiscIO
@@ -205,6 +206,10 @@ std::unique_ptr<BlobReader> CreateBlobReader(const std::string& filename)
     return TGCFileReader::Create(std::move(file));
   case WBFS_MAGIC:
     return WbfsFileReader::Create(std::move(file), filename);
+  case WIA_MAGIC:
+    return WIAFileReader::Create(std::move(file), filename);
+  case RVZ_MAGIC:
+    return RVZFileReader::Create(std::move(file), filename);
   default:
     if (auto directory_blob = DirectoryBlobReader::Create(filename))
       return std::move(directory_blob);

Source/Core/DiscIO/Blob.h

@@ -25,6 +25,8 @@
 
 namespace DiscIO
 {
+enum class WIARVZCompressionType : u32;
+
 // Increment CACHE_REVISION (GameFileCache.cpp) if the enum below is modified
 enum class BlobType
 {
@@ -34,7 +36,9 @@ enum class BlobType
   GCZ,
   CISO,
   WBFS,
-  TGC
+  TGC,
+  WIA,
+  RVZ,
 };
 
 class BlobReader
@@ -172,5 +176,9 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path,
 bool ConvertToPlain(BlobReader* infile, const std::string& infile_path,
                     const std::string& outfile_path, CompressCB callback = nullptr,
                     void* arg = nullptr);
+bool ConvertToWIAOrRVZ(BlobReader* infile, const std::string& infile_path,
+                       const std::string& outfile_path, bool rvz,
+                       WIARVZCompressionType compression_type, int compression_level,
+                       int chunk_size, CompressCB callback = nullptr, void* arg = nullptr);
 
 }  // namespace DiscIO

Source/Core/DiscIO/CMakeLists.txt

@@ -21,6 +21,8 @@ add_library(discio
   FileSystemGCWii.h
   Filesystem.cpp
   Filesystem.h
+  LaggedFibonacciGenerator.cpp
+  LaggedFibonacciGenerator.h
   MultithreadedCompressor.h
   NANDImporter.cpp
   NANDImporter.h
@@ -42,13 +44,22 @@ add_library(discio
   VolumeWii.h
   WbfsBlob.cpp
   WbfsBlob.h
+  WIABlob.cpp
+  WIABlob.h
+  WIACompression.cpp
+  WIACompression.h
   WiiEncryptionCache.cpp
   WiiEncryptionCache.h
   WiiSaveBanner.cpp
   WiiSaveBanner.h
 )
 
 target_link_libraries(discio
+PUBLIC
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  zstd
+
 PRIVATE
   minizip
   pugixml

Source/Core/DiscIO/DiscIO.vcxproj

@@ -55,6 +55,7 @@
     <ClCompile Include="FileBlob.cpp" />
     <ClCompile Include="Filesystem.cpp" />
     <ClCompile Include="FileSystemGCWii.cpp" />
+    <ClCompile Include="LaggedFibonacciGenerator.cpp" />
     <ClCompile Include="NANDImporter.cpp" />
     <ClCompile Include="ScrubbedBlob.cpp" />
     <ClCompile Include="TGCBlob.cpp" />
@@ -65,6 +66,8 @@
     <ClCompile Include="VolumeWad.cpp" />
     <ClCompile Include="VolumeWii.cpp" />
     <ClCompile Include="WbfsBlob.cpp" />
+    <ClCompile Include="WIABlob.cpp" />
+    <ClCompile Include="WIACompression.cpp" />
     <ClCompile Include="WiiEncryptionCache.cpp" />
     <ClCompile Include="WiiSaveBanner.cpp" />
   </ItemGroup>
@@ -80,6 +83,7 @@
     <ClInclude Include="FileBlob.h" />
     <ClInclude Include="Filesystem.h" />
     <ClInclude Include="FileSystemGCWii.h" />
+    <ClInclude Include="LaggedFibonacciGenerator.h" />
     <ClInclude Include="MultithreadedCompressor.h" />
     <ClInclude Include="NANDImporter.h" />
     <ClInclude Include="ScrubbedBlob.h" />
@@ -91,6 +95,8 @@
     <ClInclude Include="VolumeWad.h" />
     <ClInclude Include="VolumeWii.h" />
     <ClInclude Include="WbfsBlob.h" />
+    <ClInclude Include="WIABlob.h" />
+    <ClInclude Include="WIACompression.h" />
     <ClInclude Include="WiiEncryptionCache.h" />
     <ClInclude Include="WiiSaveBanner.h" />
   </ItemGroup>
@@ -110,6 +116,15 @@
     <ProjectReference Include="$(ExternalsDir)pugixml\pugixml.vcxproj">
       <Project>{38fee76f-f347-484b-949c-b4649381cffb}</Project>
     </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)bzip2\bzip2.vcxproj">
+      <Project>{055a775f-b4f5-4970-9240-f6cf7661f37b}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)liblzma\liblzma.vcxproj">
+      <Project>{1d8c51d2-ffa4-418e-b183-9f42b6a6717e}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)zstd\zstd.vcxproj">
+      <Project>{1bea10f3-80ce-4bc4-9331-5769372cdf99}</Project>
+    </ProjectReference>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

Source/Core/DiscIO/DiscIO.vcxproj.filters

@@ -90,6 +90,15 @@
     <ClCompile Include="ScrubbedBlob.cpp">
       <Filter>Volume\Blob</Filter>
     </ClCompile>
+    <ClCompile Include="WIABlob.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
+    <ClCompile Include="LaggedFibonacciGenerator.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
+    <ClCompile Include="WIACompression.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="DiscScrubber.h">
@@ -164,6 +173,15 @@
     <ClInclude Include="MultithreadedCompressor.h">
       <Filter>Volume\Blob</Filter>
     </ClInclude>
+    <ClInclude Include="WIABlob.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
+    <ClInclude Include="LaggedFibonacciGenerator.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
+    <ClInclude Include="WIACompression.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Text Include="CMakeLists.txt" />

Source/Core/DiscIO/LaggedFibonacciGenerator.cpp

@@ -0,0 +1,212 @@
+// This file is under the public domain.
+
+#include "DiscIO/LaggedFibonacciGenerator.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+
+#include "Common/Align.h"
+#include "Common/Assert.h"
+#include "Common/CommonTypes.h"
+#include "Common/Swap.h"
+
+namespace DiscIO
+{
+void LaggedFibonacciGenerator::SetSeed(const u32 seed[SEED_SIZE])
+{
+  SetSeed(reinterpret_cast<const u8*>(seed));
+}
+
+void LaggedFibonacciGenerator::SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)])
+{
+  m_position_bytes = 0;
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    m_buffer[i] = Common::swap32(seed + i * sizeof(u32));
+
+  Initialize(false);
+}
+
+size_t LaggedFibonacciGenerator::GetSeed(const u8* data, size_t size, size_t data_offset,
+                                         u32 seed_out[SEED_SIZE])
+{
+  if ((reinterpret_cast<uintptr_t>(data) - data_offset) % alignof(u32) != 0)
+  {
+    ASSERT(false);
+    return 0;
+  }
+
+  // For code simplicity, only include whole u32 words when regenerating the seed. It would be
+  // possible to get rid of this restriction and use a few additional bytes, but it's probably more
+  // effort than it's worth considering that junk data often starts or ends on 4-byte offsets.
+  const size_t bytes_to_skip = Common::AlignUp(data_offset, sizeof(u32)) - data_offset;
+  const u32* u32_data = reinterpret_cast<const u32*>(data + bytes_to_skip);
+  const size_t u32_size = (size - bytes_to_skip) / sizeof(u32);
+  const size_t u32_data_offset = (data_offset + bytes_to_skip) / sizeof(u32);
+
+  LaggedFibonacciGenerator lfg;
+  if (!GetSeed(u32_data, u32_size, u32_data_offset, &lfg, seed_out))
+    return false;
+
+  lfg.m_position_bytes = data_offset % (LFG_K * sizeof(u32));
+
+  const u8* end = data + size;
+  size_t reconstructed_bytes = 0;
+  while (data < end && lfg.GetByte() == *data)
+  {
+    ++reconstructed_bytes;
+    ++data;
+  }
+  return reconstructed_bytes;
+}
+
+bool LaggedFibonacciGenerator::GetSeed(const u32* data, size_t size, size_t data_offset,
+                                       LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE])
+{
+  if (size < LFG_K)
+    return false;
+
+  // If the data doesn't look like something we can regenerate, return early to save time
+  if (!std::all_of(data, data + LFG_K, [](u32 x) {
+        return (Common::swap32(x) & 0x00C00000) == (Common::swap32(x) >> 2 & 0x00C00000);
+      }))
+  {
+    return false;
+  }
+
+  const size_t data_offset_mod_k = data_offset % LFG_K;
+  const size_t data_offset_div_k = data_offset / LFG_K;
+
+  std::copy(data, data + LFG_K - data_offset_mod_k, lfg->m_buffer.data() + data_offset_mod_k);
+  std::copy(data + LFG_K - data_offset_mod_k, data + LFG_K, lfg->m_buffer.data());
+
+  lfg->Backward(0, data_offset_mod_k);
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Backward();
+
+  if (!lfg->Reinitialize(seed_out))
+    return false;
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Forward();
+
+  return true;
+}
+
+void LaggedFibonacciGenerator::GetBytes(size_t count, u8* out)
+{
+  while (count > 0)
+  {
+    const size_t length = std::min(count, LFG_K * sizeof(u32) - m_position_bytes);
+
+    std::memcpy(out, reinterpret_cast<u8*>(m_buffer.data()) + m_position_bytes, length);
+
+    m_position_bytes += length;
+    count -= length;
+    out += length;
+
+    if (m_position_bytes == LFG_K * sizeof(u32))
+    {
+      Forward();
+      m_position_bytes = 0;
+    }
+  }
+}
+
+u8 LaggedFibonacciGenerator::GetByte()
+{
+  const u8 result = reinterpret_cast<u8*>(m_buffer.data())[m_position_bytes];
+
+  ++m_position_bytes;
+
+  if (m_position_bytes == LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes = 0;
+  }
+
+  return result;
+}
+
+void LaggedFibonacciGenerator::Forward(size_t count)
+{
+  m_position_bytes += count;
+  while (m_position_bytes >= LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes -= LFG_K * sizeof(u32);
+  }
+}
+
+void LaggedFibonacciGenerator::Forward()
+{
+  for (size_t i = 0; i < LFG_J; ++i)
+    m_buffer[i] ^= m_buffer[i + LFG_K - LFG_J];
+
+  for (size_t i = LFG_J; i < LFG_K; ++i)
+    m_buffer[i] ^= m_buffer[i - LFG_J];
+}
+
+void LaggedFibonacciGenerator::Backward(size_t start_word, size_t end_word)
+{
+  const size_t loop_end = std::max(LFG_J, start_word);
+  for (size_t i = std::min(end_word, LFG_K); i > loop_end; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 - LFG_J];
+
+  for (size_t i = std::min(end_word, LFG_J); i > start_word; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 + LFG_K - LFG_J];
+}
+
+bool LaggedFibonacciGenerator::Reinitialize(u32 seed_out[SEED_SIZE])
+{
+  for (size_t i = 0; i < 4; ++i)
+    Backward();
+
+  for (u32& x : m_buffer)
+    x = Common::swap32(x);
+
+  // Reconstruct the bits which are missing due to the output code shifting by 18 instead of 16.
+  // Unfortunately we can't reconstruct bits 16 and 17 (counting LSB as 0) for the first word,
+  // but the observable result (when shifting by 18 instead of 16) is not affected by this.
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+  {
+    m_buffer[i] = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000) |
+                  ((m_buffer[i + 16] ^ m_buffer[i + 15]) << 9 & 0x00030000);
+  }
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    seed_out[i] = Common::swap32(m_buffer[i]);
+
+  return Initialize(true);
+}
+
+bool LaggedFibonacciGenerator::Initialize(bool check_existing_data)
+{
+  for (size_t i = SEED_SIZE; i < LFG_K; ++i)
+  {
+    const u32 calculated = (m_buffer[i - 17] << 23) ^ (m_buffer[i - 16] >> 9) ^ m_buffer[i - 1];
+
+    if (check_existing_data)
+    {
+      const u32 actual = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000);
+      if ((calculated & 0xFFFCFFFF) != actual)
+        return false;
+    }
+
+    m_buffer[i] = calculated;
+  }
+
+  // Instead of doing the "shift by 18 instead of 16" oddity when actually outputting the data,
+  // we can do the shifting (and byteswapping) at this point to make the output code simpler.
+  for (u32& x : m_buffer)
+    x = Common::swap32((x & 0xFF00FFFF) | ((x >> 2) & 0x00FF0000));
+
+  for (size_t i = 0; i < 4; ++i)
+    Forward();
+
+  return true;
+}
+
+}  // namespace DiscIO