Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #9572 from JosJuice/volumeverifier-align-group 
VolumeVerifier: Align partition reads to groups
  • Loading branch information
@JosJuice
JosJuice committed Mar 22, 2021
2 parents 57c9c9e + c0eb954 commit cb9a4da
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 54 deletions.
2 changes: 1 addition & 1 deletion Source/Core/DiscIO/Volume.h
View file
Expand Up @@ -126,7 +126,7 @@ class Volume
virtual bool IsNKit() const = 0;
virtual bool SupportsIntegrityCheck() const { return false; }
virtual bool CheckH3TableIntegrity(const Partition& partition) const { return false; }
virtual bool CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encrypted_data,
virtual bool CheckBlockIntegrity(u64 block_index, const u8* encrypted_data,
const Partition& partition) const
{
return false;
Expand Down
101 changes: 61 additions & 40 deletions Source/Core/DiscIO/VolumeVerifier.cpp
View file
Expand Up @@ -357,7 +357,7 @@ RedumpVerifier::Result RedumpVerifier::Finish(const Hashes<std::vector<u8>>& has
return {Status::Unknown, Common::GetStringT("Unknown disc")};
}

constexpr u64 BLOCK_SIZE = 0x20000;
constexpr u64 DEFAULT_READ_SIZE = 0x20000; // Arbitrary value

VolumeVerifier::VolumeVerifier(const Volume& volume, bool redump_verification,
Hashes<bool> hashes_to_calculate)
Expand Down Expand Up @@ -585,13 +585,25 @@ bool VolumeVerifier::CheckPartition(const Partition& partition)
// Prepare for hash verification in the Process step
if (m_volume.SupportsIntegrityCheck())
{
u64 offset = m_volume.PartitionOffsetToRawOffset(0, partition);
const std::optional<u64> size =
m_volume.ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE);
const u64 end_offset = offset + size.value_or(0);
const u64 data_size =
m_volume.ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE).value_or(0);
const size_t blocks = static_cast<size_t>(data_size / VolumeWii::BLOCK_TOTAL_SIZE);

if (data_size % VolumeWii::BLOCK_TOTAL_SIZE != 0)
{
std::string text = Common::FmtFormatT(
"The data size for the {0} partition is not evenly divisible by the block size.", name);
AddProblem(Severity::Low, std::move(text));
}

for (size_t i = 0; offset < end_offset; ++i, offset += VolumeWii::BLOCK_TOTAL_SIZE)
m_blocks.emplace_back(BlockToVerify{partition, offset, i});
u64 offset = m_volume.PartitionOffsetToRawOffset(0, partition);
for (size_t block_index = 0; block_index < blocks;
block_index += VolumeWii::BLOCKS_PER_GROUP, offset += VolumeWii::GROUP_TOTAL_SIZE)
{
m_groups.emplace_back(
GroupToVerify{partition, offset, block_index,
std::min(block_index + VolumeWii::BLOCKS_PER_GROUP, blocks)});
}

m_block_errors.emplace(partition, 0);
}
Expand Down Expand Up @@ -753,7 +765,7 @@ void VolumeVerifier::CheckVolumeSize()
}
}

if (m_content_index != m_content_offsets.size() || m_block_index != m_blocks.size() ||
if (m_content_index != m_content_offsets.size() || m_group_index != m_groups.size() ||
(volume_size_roughly_known && m_biggest_referenced_offset > volume_size))
{
const bool second_layer_missing = is_disc && volume_size_roughly_known &&
Expand Down Expand Up @@ -1020,8 +1032,8 @@ void VolumeVerifier::SetUpHashing()
m_scrubber.SetupScrub(&m_volume);
}

std::sort(m_blocks.begin(), m_blocks.end(),
[](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; });
std::sort(m_groups.begin(), m_groups.end(),
[](const GroupToVerify& a, const GroupToVerify& b) { return a.offset < b.offset; });

if (m_hashes_to_calculate.crc32)
m_crc32_context = crc32(0, nullptr, 0);
Expand Down Expand Up @@ -1049,8 +1061,8 @@ void VolumeVerifier::WaitForAsyncOperations() const
m_sha1_future.wait();
if (m_content_future.valid())
m_content_future.wait();
if (m_block_future.valid())
m_block_future.wait();
if (m_group_future.valid())
m_group_future.wait();
}

bool VolumeVerifier::ReadChunkAndWaitForAsyncOperations(u64 bytes_to_read)
Expand Down Expand Up @@ -1086,8 +1098,8 @@ void VolumeVerifier::Process()

IOS::ES::Content content{};
bool content_read = false;
bool block_read = false;
u64 bytes_to_read = BLOCK_SIZE;
bool group_read = false;
u64 bytes_to_read = DEFAULT_READ_SIZE;
u64 excess_bytes = 0;
if (m_content_index < m_content_offsets.size() &&
m_content_offsets[m_content_index] == m_progress)
Expand All @@ -1107,20 +1119,22 @@ void VolumeVerifier::Process()
{
bytes_to_read = std::min(bytes_to_read, m_content_offsets[m_content_index] - m_progress);
}
else if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset == m_progress)
else if (m_group_index < m_groups.size() && m_groups[m_group_index].offset == m_progress)
{
bytes_to_read = VolumeWii::BLOCK_TOTAL_SIZE;
block_read = true;
const size_t blocks =
m_groups[m_group_index].block_index_end - m_groups[m_group_index].block_index_start;
bytes_to_read = VolumeWii::BLOCK_TOTAL_SIZE * blocks;
group_read = true;

if (m_block_index + 1 < m_blocks.size() &&
m_blocks[m_block_index + 1].offset < m_progress + bytes_to_read)
if (m_group_index + 1 < m_groups.size() &&
m_groups[m_group_index + 1].offset < m_progress + bytes_to_read)
{
excess_bytes = m_progress + bytes_to_read - m_blocks[m_block_index + 1].offset;
excess_bytes = m_progress + bytes_to_read - m_groups[m_group_index + 1].offset;
}
}
else if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset > m_progress)
else if (m_group_index < m_groups.size() && m_groups[m_group_index].offset > m_progress)
{
bytes_to_read = std::min(bytes_to_read, m_blocks[m_block_index].offset - m_progress);
bytes_to_read = std::min(bytes_to_read, m_groups[m_group_index].offset - m_progress);
}

if (m_progress + bytes_to_read > m_max_progress)
Expand All @@ -1133,7 +1147,7 @@ void VolumeVerifier::Process()
excess_bytes -= bytes_over_max;
}

const bool is_data_needed = m_calculating_any_hash || content_read || block_read;
const bool is_data_needed = m_calculating_any_hash || content_read || group_read;
const bool read_succeeded = is_data_needed && ReadChunkAndWaitForAsyncOperations(bytes_to_read);

if (!read_succeeded)
Expand Down Expand Up @@ -1185,33 +1199,40 @@ void VolumeVerifier::Process()
m_content_index++;
}

if (block_read)
if (group_read)
{
m_block_future = std::async(std::launch::async, [this, read_succeeded,
block_index = m_block_index] {
const BlockToVerify& block = m_blocks[block_index];
if (read_succeeded &&
m_volume.CheckBlockIntegrity(block.block_index, m_data, block.partition))
m_group_future = std::async(std::launch::async, [this, read_succeeded,
group_index = m_group_index] {
const GroupToVerify& group = m_groups[group_index];
u64 offset_in_group = 0;
for (u64 block_index = group.block_index_start; block_index < group.block_index_end;
++block_index, offset_in_group += VolumeWii::BLOCK_TOTAL_SIZE)
{
m_biggest_verified_offset =
std::max(m_biggest_verified_offset, block.offset + VolumeWii::BLOCK_TOTAL_SIZE);
}
else
{
if (m_scrubber.CanBlockBeScrubbed(block.offset))
const u64 block_offset = group.offset + offset_in_group;

if (read_succeeded && m_volume.CheckBlockIntegrity(
block_index, m_data.data() + offset_in_group, group.partition))
{
WARN_LOG_FMT(DISCIO, "Integrity check failed for unused block at {:#x}", block.offset);
m_unused_block_errors[block.partition]++;
m_biggest_verified_offset =
std::max(m_biggest_verified_offset, block_offset + VolumeWii::BLOCK_TOTAL_SIZE);
}
else
{
WARN_LOG_FMT(DISCIO, "Integrity check failed for block at {:#x}", block.offset);
m_block_errors[block.partition]++;
if (m_scrubber.CanBlockBeScrubbed(block_offset))
{
WARN_LOG_FMT(DISCIO, "Integrity check failed for unused block at {:#x}", block_offset);
m_unused_block_errors[group.partition]++;
}
else
{
WARN_LOG_FMT(DISCIO, "Integrity check failed for block at {:#x}", block_offset);
m_block_errors[group.partition]++;
}
}
}
});

m_block_index++;
m_group_index++;
}

m_progress += byte_increment;
Expand Down
11 changes: 6 additions & 5 deletions Source/Core/DiscIO/VolumeVerifier.h
View file
Expand Up @@ -135,11 +135,12 @@ class VolumeVerifier final
const Result& GetResult() const;

private:
struct BlockToVerify
struct GroupToVerify
{
Partition partition;
u64 offset;
u64 block_index;
size_t block_index_start;
size_t block_index_end;
};

std::vector<Partition> CheckPartitions();
Expand Down Expand Up @@ -182,14 +183,14 @@ class VolumeVerifier final
std::future<void> m_md5_future;
std::future<void> m_sha1_future;
std::future<void> m_content_future;
std::future<void> m_block_future;
std::future<void> m_group_future;

DiscScrubber m_scrubber;
IOS::ES::TicketReader m_ticket;
std::vector<u64> m_content_offsets;
u16 m_content_index = 0;
std::vector<BlockToVerify> m_blocks;
size_t m_block_index = 0; // Index in m_blocks, not index in a specific partition
std::vector<GroupToVerify> m_groups;
size_t m_group_index = 0; // Index in m_groups, not index in a specific partition
std::map<Partition, size_t> m_block_errors;
std::map<Partition, size_t> m_unused_block_errors;

Expand Down
11 changes: 4 additions & 7 deletions Source/Core/DiscIO/VolumeWii.cpp
View file
Expand Up @@ -412,12 +412,9 @@ bool VolumeWii::CheckH3TableIntegrity(const Partition& partition) const
return h3_table_sha1 == contents[0].sha1;
}

bool VolumeWii::CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encrypted_data,
bool VolumeWii::CheckBlockIntegrity(u64 block_index, const u8* encrypted_data,
const Partition& partition) const
{
if (encrypted_data.size() != BLOCK_TOTAL_SIZE)
return false;

auto it = m_partitions.find(partition);
if (it == m_partitions.end())
return false;
Expand All @@ -431,10 +428,10 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encr
return false;

HashBlock hashes;
DecryptBlockHashes(encrypted_data.data(), &hashes, aes_context);
DecryptBlockHashes(encrypted_data, &hashes, aes_context);

u8 cluster_data[BLOCK_DATA_SIZE];
DecryptBlockData(encrypted_data.data(), cluster_data, aes_context);
DecryptBlockData(encrypted_data, cluster_data, aes_context);

for (u32 hash_index = 0; hash_index < 31; ++hash_index)
{
Expand Down Expand Up @@ -474,7 +471,7 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition)
std::vector<u8> cluster(BLOCK_TOTAL_SIZE);
if (!m_reader->Read(cluster_offset, cluster.size(), cluster.data()))
return false;
return CheckBlockIntegrity(block_index, cluster, partition);
return CheckBlockIntegrity(block_index, cluster.data(), partition);
}

bool VolumeWii::HashGroup(const std::array<u8, BLOCK_DATA_SIZE> in[BLOCKS_PER_GROUP],
Expand Down
2 changes: 1 addition & 1 deletion Source/Core/DiscIO/VolumeWii.h
View file
Expand Up @@ -82,7 +82,7 @@ class VolumeWii : public VolumeDisc
bool IsDatelDisc() const override;
bool SupportsIntegrityCheck() const override { return m_encrypted; }
bool CheckH3TableIntegrity(const Partition& partition) const override;
bool CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encrypted_data,
bool CheckBlockIntegrity(u64 block_index, const u8* encrypted_data,
const Partition& partition) const override;
bool CheckBlockIntegrity(u64 block_index, const Partition& partition) const override;

Expand Down

0 comments on commit cb9a4da

Please sign in to comment.