Merge pull request #10271 from Pokechu22/hash.h-merge

Merge CRC32.h into Hash.h and remove MD5.h

Source/Core/Common/CMakeLists.txt

@@ -25,8 +25,6 @@ add_library(common
   Config/Layer.cpp
   Config/Layer.h
   CPUDetect.h
-  CRC32.cpp
-  CRC32.h
   Crypto/AES.cpp
   Crypto/AES.h
   Crypto/bn.cpp
@@ -83,8 +81,6 @@ add_library(common
   MathUtil.h
   Matrix.cpp
   Matrix.h
-  MD5.cpp
-  MD5.h
   MemArena.h
   MemoryUtil.cpp
   MemoryUtil.h

Source/Core/Common/Hash.cpp

@@ -5,6 +5,8 @@
 
 #include <algorithm>
 #include <cstring>
+#include <zlib.h>
+
 #include "Common/BitUtils.h"
 #include "Common/CPUDetect.h"
 #include "Common/CommonFuncs.h"
@@ -529,4 +531,28 @@ void SetHash64Function()
     ptrHashFunction = &GetMurmurHash3;
   }
 }
+
+u32 ComputeCRC32(std::string_view data)
+{
+  return ComputeCRC32(reinterpret_cast<const u8*>(data.data()), static_cast<u32>(data.size()));
+}
+
+u32 ComputeCRC32(const u8* ptr, u32 length)
+{
+  return UpdateCRC32(StartCRC32(), ptr, length);
+}
+
+u32 StartCRC32()
+{
+  return crc32(0L, Z_NULL, 0);
+}
+
+u32 UpdateCRC32(u32 crc, const u8* ptr, u32 length)
+{
+  static_assert(std::is_same_v<const u8*, const Bytef*>);
+  static_assert(std::is_same_v<u32, uInt>);
+  // Use zlib's crc32 implementation to compute the hash
+  // crc32_z (which takes a size_t) would be better, but it isn't available on Android
+  return crc32(crc, ptr, length);
+}
 }  // namespace Common

Source/Core/Common/Hash.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <cstddef>
+#include <string_view>
 
 #include "Common/CommonTypes.h"
 
@@ -14,4 +15,9 @@ u32 HashAdler32(const u8* data, size_t len);         // Fairly accurate, slightl
 u32 HashEctor(const u8* ptr, size_t length);         // JUNK. DO NOT USE FOR NEW THINGS
 u64 GetHash64(const u8* src, u32 len, u32 samples);
 void SetHash64Function();
+
+u32 ComputeCRC32(std::string_view data);
+u32 ComputeCRC32(const u8* ptr, u32 length);
+u32 StartCRC32();
+u32 UpdateCRC32(u32 crc, const u8* ptr, u32 length);
 }  // namespace Common

Source/Core/Core/Boot/Boot.cpp

@@ -22,11 +22,11 @@ namespace fs = std::filesystem;
 
 #include "Common/Align.h"
 #include "Common/CDUtils.h"
-#include "Common/CRC32.h"
 #include "Common/CommonPaths.h"
 #include "Common/CommonTypes.h"
 #include "Common/Config/Config.h"
 #include "Common/FileUtil.h"
+#include "Common/Hash.h"
 #include "Common/IOFile.h"
 #include "Common/Logging/Log.h"
 #include "Common/MsgHandler.h"

Source/Core/Core/HW/WiimoteEmu/MotionPlus.cpp

@@ -8,10 +8,10 @@
 #include <iterator>
 
 #include <mbedtls/bignum.h>
-#include <zlib.h>
 
 #include "Common/BitUtils.h"
 #include "Common/ChunkFile.h"
+#include "Common/Hash.h"
 #include "Common/Logging/Log.h"
 #include "Common/MathUtil.h"
 #include "Common/MsgHandler.h"
@@ -146,9 +146,9 @@ void MotionPlus::Reset()
 void MotionPlus::CalibrationData::UpdateChecksum()
 {
   // Checksum is crc32 of all data other than the checksum itself.
-  auto crc_result = crc32(0, Z_NULL, 0);
-  crc_result = crc32(crc_result, reinterpret_cast<const Bytef*>(this), 0xe);
-  crc_result = crc32(crc_result, reinterpret_cast<const Bytef*>(this) + 0x10, 0xe);
+  u32 crc_result = Common::StartCRC32();
+  crc_result = Common::UpdateCRC32(crc_result, reinterpret_cast<const u8*>(this), 0xe);
+  crc_result = Common::UpdateCRC32(crc_result, reinterpret_cast<const u8*>(this) + 0x10, 0xe);
 
   crc32_lsb = u16(crc_result);
   crc32_msb = u16(crc_result >> 16);

Source/Core/Core/NetPlayClient.cpp

@@ -7,6 +7,7 @@
 #include <cstddef>
 #include <cstring>
 #include <fstream>
+#include <functional>
 #include <memory>
 #include <mutex>
 #include <sstream>
@@ -23,7 +24,6 @@
 #include "Common/ENetUtil.h"
 #include "Common/FileUtil.h"
 #include "Common/Logging/Log.h"
-#include "Common/MD5.h"
 #include "Common/MsgHandler.h"
 #include "Common/NandPaths.h"
 #include "Common/QoSSession.h"
@@ -62,6 +62,7 @@
 #include "Core/NetPlayCommon.h"
 #include "Core/PowerPC/PowerPC.h"
 #include "Core/SyncIdentifier.h"
+#include "DiscIO/Blob.h"
 
 #include "InputCommon/ControllerEmu/ControlGroup/Attachments.h"
 #include "InputCommon/GCAdapter.h"
@@ -2462,6 +2463,39 @@ bool NetPlayClient::DoAllPlayersHaveGame()
   });
 }
 
+static std::string MD5Sum(const std::string& file_path, std::function<bool(int)> report_progress)
+{
+  std::vector<u8> data(8 * 1024 * 1024);
+  u64 read_offset = 0;
+  mbedtls_md5_context ctx;
+
+  std::unique_ptr<DiscIO::BlobReader> file(DiscIO::CreateBlobReader(file_path));
+  u64 game_size = file->GetDataSize();
+
+  mbedtls_md5_starts_ret(&ctx);
+
+  while (read_offset < game_size)
+  {
+    size_t read_size = std::min(static_cast<u64>(data.size()), game_size - read_offset);
+    if (!file->Read(read_offset, read_size, data.data()))
+      return "";
+
+    mbedtls_md5_update_ret(&ctx, data.data(), read_size);
+    read_offset += read_size;
+
+    int progress =
+        static_cast<int>(static_cast<float>(read_offset) / static_cast<float>(game_size) * 100);
+    if (!report_progress(progress))
+      return "";
+  }
+
+  std::array<u8, 16> output;
+  mbedtls_md5_finish_ret(&ctx, output.data());
+
+  // Convert to hex
+  return fmt::format("{:02x}", fmt::join(output, ""));
+}
+
 void NetPlayClient::ComputeMD5(const SyncIdentifier& sync_identifier)
 {
   if (m_should_compute_MD5)
@@ -2488,7 +2522,7 @@ void NetPlayClient::ComputeMD5(const SyncIdentifier& sync_identifier)
   if (m_MD5_thread.joinable())
     m_MD5_thread.join();
   m_MD5_thread = std::thread([this, file]() {
-    std::string sum = MD5::MD5Sum(file, [&](int progress) {
+    std::string sum = MD5Sum(file, [&](int progress) {
       sf::Packet packet;
       packet << MessageID::MD5Progress;
       packet << progress;

Source/Core/DiscIO/VolumeVerifier.cpp

@@ -16,13 +16,13 @@
 #include <mbedtls/sha1.h>
 #include <pugixml.hpp>
 #include <unzip.h>
-#include <zlib.h>
 
 #include "Common/Align.h"
 #include "Common/Assert.h"
 #include "Common/CommonPaths.h"
 #include "Common/CommonTypes.h"
 #include "Common/FileUtil.h"
+#include "Common/Hash.h"
 #include "Common/HttpRequest.h"
 #include "Common/IOFile.h"
 #include "Common/Logging/Log.h"
@@ -1041,7 +1041,7 @@ void VolumeVerifier::SetUpHashing()
             [](const GroupToVerify& a, const GroupToVerify& b) { return a.offset < b.offset; });
 
   if (m_hashes_to_calculate.crc32)
-    m_crc32_context = crc32(0, nullptr, 0);
+    m_crc32_context = Common::StartCRC32();
 
   if (m_hashes_to_calculate.md5)
   {
@@ -1171,9 +1171,8 @@ void VolumeVerifier::Process()
     if (m_hashes_to_calculate.crc32)
     {
       m_crc32_future = std::async(std::launch::async, [this, byte_increment] {
-        // It would be nice to use crc32_z here instead of crc32, but it isn't available on Android
         m_crc32_context =
-            crc32(m_crc32_context, m_data.data(), static_cast<unsigned int>(byte_increment));
+            Common::UpdateCRC32(m_crc32_context, m_data.data(), static_cast<u32>(byte_increment));
       });
     }
 

Source/Core/DiscIO/VolumeVerifier.h

@@ -172,7 +172,7 @@ class VolumeVerifier final
 
   Hashes<bool> m_hashes_to_calculate{};
   bool m_calculating_any_hash = false;
-  unsigned long m_crc32_context = 0;
+  u32 m_crc32_context = 0;
   mbedtls_md5_context m_md5_context{};
   mbedtls_sha1_context m_sha1_context{};
 

Source/Core/DolphinLib.props

@@ -21,7 +21,6 @@
     <ClInclude Include="Common\BitUtils.h" />
     <ClInclude Include="Common\BlockingLoop.h" />
     <ClInclude Include="Common\CDUtils.h" />
-    <ClInclude Include="Common\CRC32.h" />
     <ClInclude Include="Common\ChunkFile.h" />
     <ClInclude Include="Common\CodeBlock.h" />
     <ClInclude Include="Common\ColorUtil.h" />
@@ -124,7 +123,6 @@
     <ClInclude Include="Common\Logging\LogManager.h" />
     <ClInclude Include="Common\MathUtil.h" />
     <ClInclude Include="Common\Matrix.h" />
-    <ClInclude Include="Common\MD5.h" />
     <ClInclude Include="Common\MemArena.h" />
     <ClInclude Include="Common\MemoryUtil.h" />
     <ClInclude Include="Common\MinizipUtil.h" />
@@ -687,7 +685,6 @@
     <ClCompile Include="AudioCommon\WaveFile.cpp" />
     <ClCompile Include="Common\Analytics.cpp" />
     <ClCompile Include="Common\CDUtils.cpp" />
-    <ClCompile Include="Common\CRC32.cpp" />
     <ClCompile Include="Common\ColorUtil.cpp" />
     <ClCompile Include="Common\CommonFuncs.cpp" />
     <ClCompile Include="Common\CompatPatches.cpp" />
@@ -725,7 +722,6 @@
     <ClCompile Include="Common\Logging\LogManager.cpp" />
     <ClCompile Include="Common\MathUtil.cpp" />
     <ClCompile Include="Common\Matrix.cpp" />
-    <ClCompile Include="Common\MD5.cpp" />
     <ClCompile Include="Common\MemArenaWin.cpp" />
     <ClCompile Include="Common\MemoryUtil.cpp" />
     <ClCompile Include="Common\MsgHandler.cpp" />