Skip to content
Permalink
Browse files

VolumeVerifier: Multithreading

  • Loading branch information...
JosJuice committed Aug 6, 2019
1 parent f754a1a commit 34fb608dd6b4137ecc98e1c03228165a27eb800a
Showing with 113 additions and 39 deletions.
  1. +101 −39 Source/Core/DiscIO/VolumeVerifier.cpp
  2. +12 −0 Source/Core/DiscIO/VolumeVerifier.h
@@ -6,7 +6,9 @@

#include <algorithm>
#include <cinttypes>
#include <future>
#include <limits>
#include <mutex>
#include <optional>
#include <string>
#include <unordered_set>
@@ -683,6 +685,34 @@ void VolumeVerifier::SetUpHashing()
}
}

void VolumeVerifier::WaitForAsyncOperations() const
{
if (m_crc32_future.valid())
m_crc32_future.wait();
if (m_md5_future.valid())
m_md5_future.wait();
if (m_sha1_future.valid())
m_sha1_future.wait();
if (m_content_future.valid())
m_content_future.wait();
if (m_block_future.valid())
m_block_future.wait();
}

bool VolumeVerifier::ReadChunkAndWaitForAsyncOperations(u64 bytes_to_read)
{
std::vector<u8> data(bytes_to_read);
{
std::lock_guard lk(m_volume_mutex);
if (!m_volume.Read(m_progress, bytes_to_read, data.data(), PARTITION_NONE))
return false;
}

WaitForAsyncOperations();
m_data = std::move(data);
return true;
}

void VolumeVerifier::Process()
{
ASSERT(m_started);
@@ -718,13 +748,8 @@ void VolumeVerifier::Process()
}
bytes_to_read = std::min(bytes_to_read, m_max_progress - m_progress);

bool read_succeeded = false;
std::vector<u8> data(bytes_to_read);
if (m_calculating_any_hash || content_read || block_read)
{
if (m_volume.Read(m_progress, bytes_to_read, data.data(), PARTITION_NONE))
read_succeeded = true;
}
const bool is_data_needed = m_calculating_any_hash || content_read || block_read;
const bool read_succeeded = is_data_needed && ReadChunkAndWaitForAsyncOperations(bytes_to_read);

if (m_calculating_any_hash)
{
@@ -736,55 +761,90 @@ void VolumeVerifier::Process()
{
if (m_hashes_to_calculate.crc32)
{
// It would be nice to use crc32_z here instead of crc32, but it isn't available on Android
m_crc32_context =
crc32(m_crc32_context, data.data(), static_cast<unsigned int>(bytes_to_read));
m_crc32_future = std::async(std::launch::async, [this] {
// Would be nice to use crc32_z here instead of crc32, but it isn't available on Android
m_crc32_context =
crc32(m_crc32_context, m_data.data(), static_cast<unsigned int>(m_data.size()));
});
}

if (m_hashes_to_calculate.md5)
mbedtls_md5_update_ret(&m_md5_context, data.data(), bytes_to_read);
{
m_md5_future = std::async(std::launch::async, [this] {
mbedtls_md5_update_ret(&m_md5_context, m_data.data(), m_data.size());
});
}

if (m_hashes_to_calculate.sha1)
mbedtls_sha1_update_ret(&m_sha1_context, data.data(), bytes_to_read);
{
m_sha1_future = std::async(std::launch::async, [this] {
mbedtls_sha1_update_ret(&m_sha1_context, m_data.data(), m_data.size());
});
}
}
}

if (content_read)
{
if (!read_succeeded || !m_volume.CheckContentIntegrity(content, data, m_ticket))
{
AddProblem(
Severity::High,
StringFromFormat(Common::GetStringT("Content %08x is corrupt.").c_str(), content.id));
}
m_content_future = std::async(std::launch::async, [this, read_succeeded, content] {
if (!read_succeeded || !m_volume.CheckContentIntegrity(content, m_data, m_ticket))
{
AddProblem(
Severity::High,
StringFromFormat(Common::GetStringT("Content %08x is corrupt.").c_str(), content.id));
}
});

m_content_index++;
}

while (m_block_index < m_blocks.size() &&
m_blocks[m_block_index].offset < m_progress + bytes_to_read)
if (m_block_index < m_blocks.size() &&
m_blocks[m_block_index].offset < m_progress + bytes_to_read)
{
const bool success = m_blocks[m_block_index].offset == m_progress ?
read_succeeded && m_volume.CheckBlockIntegrity(
m_blocks[m_block_index].block_index, data,
m_blocks[m_block_index].partition) :
m_volume.CheckBlockIntegrity(m_blocks[m_block_index].block_index,
m_blocks[m_block_index].partition);
if (!success)
m_md5_future = std::async(
std::launch::async,
[this, read_succeeded, bytes_to_read](size_t block_index, u64 progress) {
while (block_index < m_blocks.size() &&
m_blocks[block_index].offset < progress + bytes_to_read)
{
bool success;
if (m_blocks[block_index].offset == progress)
{
success = read_succeeded &&
m_volume.CheckBlockIntegrity(m_blocks[block_index].block_index, m_data,
m_blocks[block_index].partition);
}
else
{
std::lock_guard lk(m_volume_mutex);
success = m_volume.CheckBlockIntegrity(m_blocks[block_index].block_index,
m_blocks[block_index].partition);
}

if (!success)
{
const u64 offset = m_blocks[block_index].offset;
if (m_scrubber.CanBlockBeScrubbed(offset))
{
WARN_LOG(DISCIO, "Integrity check failed for unused block at 0x%" PRIx64, offset);
m_unused_block_errors[m_blocks[block_index].partition]++;
}
else
{
WARN_LOG(DISCIO, "Integrity check failed for block at 0x%" PRIx64, offset);
m_block_errors[m_blocks[block_index].partition]++;
}
}
block_index++;
}
},
m_block_index, m_progress);

while (m_block_index < m_blocks.size() &&
m_blocks[m_block_index].offset < m_progress + bytes_to_read)
{
const u64 offset = m_blocks[m_block_index].offset;
if (m_scrubber.CanBlockBeScrubbed(offset))
{
WARN_LOG(DISCIO, "Integrity check failed for unused block at 0x%" PRIx64, offset);
m_unused_block_errors[m_blocks[m_block_index].partition]++;
}
else
{
WARN_LOG(DISCIO, "Integrity check failed for block at 0x%" PRIx64, offset);
m_block_errors[m_blocks[m_block_index].partition]++;
}
m_block_index++;
}
m_block_index++;
}

m_progress += bytes_to_read;
@@ -806,6 +866,8 @@ void VolumeVerifier::Finish()
return;
m_done = true;

WaitForAsyncOperations();

ASSERT(m_content_index == m_content_offsets.size());
ASSERT(m_block_index == m_blocks.size());

@@ -4,7 +4,9 @@

#pragma once

#include <future>
#include <map>
#include <mutex>
#include <optional>
#include <string>
#include <vector>
@@ -98,6 +100,8 @@ class VolumeVerifier final
u64 GetBiggestUsedOffset(const FileInfo& file_info) const;
void CheckMisc();
void SetUpHashing();
void WaitForAsyncOperations() const;
bool ReadChunkAndWaitForAsyncOperations(u64 bytes_to_read);

void AddProblem(Severity severity, std::string text);

@@ -113,6 +117,14 @@ class VolumeVerifier final
mbedtls_md5_context m_md5_context;
mbedtls_sha1_context m_sha1_context;

std::vector<u8> m_data;
std::mutex m_volume_mutex;
std::future<void> m_crc32_future;
std::future<void> m_md5_future;
std::future<void> m_sha1_future;
std::future<void> m_content_future;
std::future<void> m_block_future;

DiscScrubber m_scrubber;
IOS::ES::TicketReader m_ticket;
std::vector<u64> m_content_offsets;

0 comments on commit 34fb608

Please sign in to comment.
You can’t perform that action at this time.