From c21328dfdf6c2c98634892cc1ea191ff233df548 Mon Sep 17 00:00:00 2001 From: Gliniak Date: Tue, 21 Jun 2022 11:05:32 +0200 Subject: [PATCH] [VFS] Separation of STFS, SVOD into different entities --- src/xenia/emulator.cc | 16 +- .../vfs/devices/stfs_container_device.cc | 858 ------------------ src/xenia/vfs/devices/stfs_container_device.h | 148 --- src/xenia/vfs/devices/stfs_xbox.h | 42 +- .../vfs/devices/xcontent_container_device.cc | 165 ++++ .../vfs/devices/xcontent_container_device.h | 98 ++ ...r_entry.cc => xcontent_container_entry.cc} | 24 +- ...ner_entry.h => xcontent_container_entry.h} | 26 +- ...ner_file.cc => xcontent_container_file.cc} | 17 +- ...ainer_file.h => xcontent_container_file.h} | 18 +- .../xcontent_devices/stfs_container_device.cc | 324 +++++++ .../xcontent_devices/stfs_container_device.h | 95 ++ .../xcontent_devices/svod_container_device.cc | 417 +++++++++ .../xcontent_devices/svod_container_device.h | 77 ++ src/xenia/vfs/vfs_dump.cc | 7 +- 15 files changed, 1269 insertions(+), 1063 deletions(-) delete mode 100644 src/xenia/vfs/devices/stfs_container_device.cc delete mode 100644 src/xenia/vfs/devices/stfs_container_device.h create mode 100644 src/xenia/vfs/devices/xcontent_container_device.cc create mode 100644 src/xenia/vfs/devices/xcontent_container_device.h rename src/xenia/vfs/devices/{stfs_container_entry.cc => xcontent_container_entry.cc} (51%) rename src/xenia/vfs/devices/{stfs_container_entry.h => xcontent_container_entry.h} (63%) rename src/xenia/vfs/devices/{stfs_container_file.cc => xcontent_container_file.cc} (77%) rename src/xenia/vfs/devices/{stfs_container_file.h => xcontent_container_file.h} (68%) create mode 100644 src/xenia/vfs/devices/xcontent_devices/stfs_container_device.cc create mode 100644 src/xenia/vfs/devices/xcontent_devices/stfs_container_device.h create mode 100644 src/xenia/vfs/devices/xcontent_devices/svod_container_device.cc create mode 100644 src/xenia/vfs/devices/xcontent_devices/svod_container_device.h diff --git a/src/xenia/emulator.cc b/src/xenia/emulator.cc index cca28982f4..0ba0758b98 100644 --- a/src/xenia/emulator.cc +++ b/src/xenia/emulator.cc @@ -2,7 +2,7 @@ ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** - * Copyright 2021 Ben Vanik. All rights reserved. * + * Copyright 2023 Ben Vanik. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ @@ -45,11 +45,12 @@ #include "xenia/ui/imgui_drawer.h" #include "xenia/ui/window.h" #include "xenia/ui/windowed_app_context.h" +#include "xenia/vfs/device.h" #include "xenia/vfs/devices/disc_image_device.h" #include "xenia/vfs/devices/host_path_device.h" #include "xenia/vfs/devices/null_device.h" -#include "xenia/vfs/devices/stfs_container_device.h" #include "xenia/vfs/virtual_file_system.h" +#include "xenia/vfs/devices/xcontent_container_device.h" #if XE_ARCH_AMD64 #include "xenia/cpu/backend/x64/x64_backend.h" @@ -351,16 +352,21 @@ X_STATUS Emulator::LaunchDiscImage(const std::filesystem::path& path) { X_STATUS Emulator::LaunchStfsContainer(const std::filesystem::path& path) { auto mount_path = "\\Device\\Cdrom0"; + auto device = + vfs::XContentContainerDevice::CreateContentDevice(mount_path, path); + if (!device) { + xe::FatalError("Cannot create XContent (STFS, SVOD) device."); + return X_STATUS_NO_SUCH_FILE; + } // Register the container in the virtual filesystem. - auto device = std::make_unique(mount_path, path); if (!device->Initialize()) { xe::FatalError( - "Unable to mount STFS container; file not found or corrupt."); + "Unable to initialize XContent container; file not found or corrupt."); return X_STATUS_NO_SUCH_FILE; } if (!file_system_->RegisterDevice(std::move(device))) { - xe::FatalError("Unable to register STFS container."); + xe::FatalError("Unable to register XContent container."); return X_STATUS_NO_SUCH_FILE; } diff --git a/src/xenia/vfs/devices/stfs_container_device.cc b/src/xenia/vfs/devices/stfs_container_device.cc deleted file mode 100644 index 1381786322..0000000000 --- a/src/xenia/vfs/devices/stfs_container_device.cc +++ /dev/null @@ -1,858 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2020 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#include "xenia/vfs/devices/stfs_container_device.h" - -#include -#include -#include - -#include "xenia/base/logging.h" -#include "xenia/base/math.h" -#include "xenia/vfs/devices/stfs_container_entry.h" - -#if XE_PLATFORM_WIN32 -#include "xenia/base/platform_win.h" -#define timegm _mkgmtime -#endif - -namespace xe { -namespace vfs { - -// Convert FAT timestamp to 100-nanosecond intervals since January 1, 1601 (UTC) -uint64_t decode_fat_timestamp(uint32_t date, uint32_t time) { - struct tm tm = {0}; - // 80 is the difference between 1980 (FAT) and 1900 (tm); - tm.tm_year = ((0xFE00 & date) >> 9) + 80; - tm.tm_mon = (0x01E0 & date) >> 5; - tm.tm_mday = (0x001F & date) >> 0; - tm.tm_hour = (0xF800 & time) >> 11; - tm.tm_min = (0x07E0 & time) >> 5; - tm.tm_sec = (0x001F & time) << 1; // the value stored in 2-seconds intervals - tm.tm_isdst = 0; - time_t timet = timegm(&tm); - if (timet == -1) { - return 0; - } - // 11644473600LL is a difference between 1970 and 1601 - return (timet + 11644473600LL) * 10000000; -} - -StfsContainerDevice::StfsContainerDevice(const std::string_view mount_path, - const std::filesystem::path& host_path) - : Device(mount_path), - name_("STFS"), - host_path_(host_path), - files_total_size_(), - svod_base_offset_(), - header_(), - svod_layout_(), - blocks_per_hash_table_(1), - block_step{0, 0} {} - -StfsContainerDevice::~StfsContainerDevice() { CloseFiles(); } - -bool StfsContainerDevice::Initialize() { - // Resolve a valid STFS file if a directory is given. - if (std::filesystem::is_directory(host_path_) && - !ResolveFromFolder(host_path_)) { - XELOGE("Could not resolve an STFS container given path {}", - xe::path_to_utf8(host_path_)); - return false; - } - - if (!std::filesystem::exists(host_path_)) { - XELOGE("Path to STFS container does not exist: {}", - xe::path_to_utf8(host_path_)); - return false; - } - - // Open the data file(s) - auto open_result = OpenFiles(); - if (open_result != Error::kSuccess) { - XELOGE("Failed to open STFS container: {}", open_result); - return false; - } - - switch (header_.metadata.volume_type) { - case XContentVolumeType::kStfs: - return ReadSTFS() == Error::kSuccess; - break; - case XContentVolumeType::kSvod: - return ReadSVOD() == Error::kSuccess; - default: - XELOGE("Unknown XContent volume type: {}", - xe::byte_swap(uint32_t(header_.metadata.volume_type.value))); - return false; - } -} - -StfsContainerDevice::Error StfsContainerDevice::OpenFiles() { - // Map the file containing the STFS Header and read it. - XELOGI("Loading STFS header file: {}", xe::path_to_utf8(host_path_)); - - auto header_file = xe::filesystem::OpenFile(host_path_, "rb"); - if (!header_file) { - XELOGE("Error opening STFS header file."); - return Error::kErrorReadError; - } - - auto header_result = ReadHeaderAndVerify(header_file); - if (header_result != Error::kSuccess) { - XELOGE("Error reading STFS header: {}", header_result); - fclose(header_file); - files_total_size_ = 0; - return header_result; - } - - // If the STFS package is a single file, the header is self contained and - // we don't need to map any extra files. - // NOTE: data_file_count is 0 for STFS and 1 for SVOD - if (header_.metadata.data_file_count <= 1) { - XELOGI("STFS container is a single file."); - files_.emplace(std::make_pair(0, header_file)); - return Error::kSuccess; - } - - // If the STFS package is multi-file, it is an SVOD system. We need to map - // the files in the .data folder and can discard the header. - auto data_fragment_path = host_path_; - data_fragment_path += ".data"; - if (!std::filesystem::exists(data_fragment_path)) { - XELOGE("STFS container is multi-file, but path {} does not exist.", - xe::path_to_utf8(data_fragment_path)); - return Error::kErrorFileMismatch; - } - - // Ensure data fragment files are sorted - auto fragment_files = filesystem::ListFiles(data_fragment_path); - std::sort(fragment_files.begin(), fragment_files.end(), - [](filesystem::FileInfo& left, filesystem::FileInfo& right) { - return left.name < right.name; - }); - - if (fragment_files.size() != header_.metadata.data_file_count) { - XELOGE("SVOD expecting {} data fragments, but {} are present.", - header_.metadata.data_file_count, fragment_files.size()); - return Error::kErrorFileMismatch; - } - - for (size_t i = 0; i < fragment_files.size(); i++) { - auto& fragment = fragment_files.at(i); - auto path = fragment.path / fragment.name; - auto file = xe::filesystem::OpenFile(path, "rb"); - if (!file) { - XELOGI("Failed to map SVOD file {}.", xe::path_to_utf8(path)); - CloseFiles(); - return Error::kErrorReadError; - } - - xe::filesystem::Seek(file, 0L, SEEK_END); - files_total_size_ += xe::filesystem::Tell(file); - // no need to seek back, any reads from this file will seek first anyway - files_.emplace(std::make_pair(i, file)); - } - XELOGI("SVOD successfully mapped {} files.", fragment_files.size()); - return Error::kSuccess; -} - -void StfsContainerDevice::CloseFiles() { - for (auto& file : files_) { - fclose(file.second); - } - files_.clear(); - files_total_size_ = 0; -} - -void StfsContainerDevice::Dump(StringBuffer* string_buffer) { - auto global_lock = global_critical_region_.Acquire(); - root_entry_->Dump(string_buffer, 0); -} - -Entry* StfsContainerDevice::ResolvePath(const std::string_view path) { - // The filesystem will have stripped our prefix off already, so the path will - // be in the form: - // some\PATH.foo - XELOGFS("StfsContainerDevice::ResolvePath({})", path); - return root_entry_->ResolvePath(path); -} - -StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify( - FILE* header_file) { - // Check size of the file is enough to store an STFS header - xe::filesystem::Seek(header_file, 0L, SEEK_END); - files_total_size_ = xe::filesystem::Tell(header_file); - xe::filesystem::Seek(header_file, 0L, SEEK_SET); - - if (sizeof(StfsHeader) > files_total_size_) { - return Error::kErrorTooSmall; - } - - // Read header & check signature - if (fread(&header_, sizeof(StfsHeader), 1, header_file) != 1) { - return Error::kErrorReadError; - } - - if (!header_.header.is_magic_valid()) { - // Unexpected format. - return Error::kErrorFileMismatch; - } - - // Pre-calculate some values used in block number calculations - if (header_.metadata.volume_type == XContentVolumeType::kStfs) { - blocks_per_hash_table_ = - header_.metadata.volume_descriptor.stfs.flags.bits.read_only_format ? 1 - : 2; - - block_step[0] = kBlocksPerHashLevel[0] + blocks_per_hash_table_; - block_step[1] = kBlocksPerHashLevel[1] + - ((kBlocksPerHashLevel[0] + 1) * blocks_per_hash_table_); - } - - return Error::kSuccess; -} - -StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() { - // SVOD Systems can have different layouts. The root block is - // denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in - // the first "actual" data fragment of the system. - auto& svod_header = files_.at(0); - const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA"; - - uint8_t magic_buf[20]; - size_t magic_offset; - - // Check for EDGF layout - if (header_.metadata.volume_descriptor.svod.features.bits - .enhanced_gdf_layout) { - // The STFS header has specified that this SVOD system uses the EGDF layout. - // We can expect the magic block to be located immediately after the hash - // blocks. We also offset block address calculation by 0x1000 by shifting - // block indices by +0x2. - xe::filesystem::Seek(svod_header, 0x2000, SEEK_SET); - if (fread(magic_buf, 1, countof(magic_buf), svod_header) != - countof(magic_buf)) { - XELOGE("ReadSVOD failed to read SVOD magic at 0x2000"); - return Error::kErrorReadError; - } - - if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) { - svod_base_offset_ = 0x0000; - magic_offset = 0x2000; - svod_layout_ = SvodLayoutType::kEnhancedGDF; - XELOGI("SVOD uses an EGDF layout. Magic block present at 0x2000."); - } else { - XELOGE("SVOD uses an EGDF layout, but the magic block was not found."); - return Error::kErrorFileMismatch; - } - } else { - xe::filesystem::Seek(svod_header, 0x12000, SEEK_SET); - if (fread(magic_buf, 1, countof(magic_buf), svod_header) != - countof(magic_buf)) { - XELOGE("ReadSVOD failed to read SVOD magic at 0x12000"); - return Error::kErrorReadError; - } - if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) { - // If the SVOD's magic block is at 0x12000, it is likely using an XSF - // layout. This is usually due to converting the game using a third-party - // tool, as most of them use a nulled XSF as a template. - - svod_base_offset_ = 0x10000; - magic_offset = 0x12000; - - // Check for XSF Header - const char* XSF_MAGIC = "XSF"; - xe::filesystem::Seek(svod_header, 0x2000, SEEK_SET); - if (fread(magic_buf, 1, 3, svod_header) != 3) { - XELOGE("ReadSVOD failed to read SVOD XSF magic at 0x2000"); - return Error::kErrorReadError; - } - if (std::memcmp(magic_buf, XSF_MAGIC, 3) == 0) { - svod_layout_ = SvodLayoutType::kXSF; - XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000."); - XELOGI("Game was likely converted using a third-party tool."); - } else { - svod_layout_ = SvodLayoutType::kUnknown; - XELOGI("SVOD appears to use an XSF layout, but no header is present."); - XELOGI("SVOD magic block found at 0x12000"); - } - } else { - xe::filesystem::Seek(svod_header, 0xD000, SEEK_SET); - if (fread(magic_buf, 1, countof(magic_buf), svod_header) != - countof(magic_buf)) { - XELOGE("ReadSVOD failed to read SVOD magic at 0xD000"); - return Error::kErrorReadError; - } - if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) { - // If the SVOD's magic block is at 0xD000, it most likely means that it - // is a single-file system. The STFS Header is 0xB000 bytes , and the - // remaining 0x2000 is from hash tables. In most cases, these will be - // STFS, not SVOD. - - svod_base_offset_ = 0xB000; - magic_offset = 0xD000; - - // Check for single file system - if (header_.metadata.data_file_count == 1) { - svod_layout_ = SvodLayoutType::kSingleFile; - XELOGI("SVOD is a single file. Magic block present at 0xD000."); - } else { - svod_layout_ = SvodLayoutType::kUnknown; - XELOGE( - "SVOD is not a single file, but the magic block was found at " - "0xD000."); - } - } else { - XELOGE("Could not locate SVOD magic block."); - return Error::kErrorReadError; - } - } - } - - // Parse the root directory - xe::filesystem::Seek(svod_header, magic_offset + 0x14, SEEK_SET); - - struct { - uint32_t block; - uint32_t size; - uint32_t creation_date; - uint32_t creation_time; - } root_data; - static_assert_size(root_data, 0x10); - - if (fread(&root_data, sizeof(root_data), 1, svod_header) != 1) { - XELOGE("ReadSVOD failed to read root block data at 0x{X}", - magic_offset + 0x14); - return Error::kErrorReadError; - } - - uint64_t root_creation_timestamp = - decode_fat_timestamp(root_data.creation_date, root_data.creation_time); - - auto root_entry = new StfsContainerEntry(this, nullptr, "", &files_); - root_entry->attributes_ = kFileAttributeDirectory; - root_entry->access_timestamp_ = root_creation_timestamp; - root_entry->create_timestamp_ = root_creation_timestamp; - root_entry->write_timestamp_ = root_creation_timestamp; - root_entry_ = std::unique_ptr(root_entry); - - // Traverse all child entries - return ReadEntrySVOD(root_data.block, 0, root_entry); -} - -StfsContainerDevice::Error StfsContainerDevice::ReadEntrySVOD( - uint32_t block, uint32_t ordinal, StfsContainerEntry* parent) { - // For games with a large amount of files, the ordinal offset can overrun - // the current block and potentially hit a hash block. - size_t ordinal_offset = ordinal * 0x4; - size_t block_offset = ordinal_offset / 0x800; - size_t true_ordinal_offset = ordinal_offset % 0x800; - - // Calculate the file & address of the block - size_t entry_address, entry_file; - BlockToOffsetSVOD(block + block_offset, &entry_address, &entry_file); - entry_address += true_ordinal_offset; - - // Read directory entry - auto& file = files_.at(entry_file); - xe::filesystem::Seek(file, entry_address, SEEK_SET); - -#pragma pack(push, 1) - struct { - uint16_t node_l; - uint16_t node_r; - uint32_t data_block; - uint32_t length; - uint8_t attributes; - uint8_t name_length; - } dir_entry; - static_assert_size(dir_entry, 0xE); -#pragma pack(pop) - - if (fread(&dir_entry, sizeof(dir_entry), 1, file) != 1) { - XELOGE("ReadEntrySVOD failed to read directory entry at 0x{X}", - entry_address); - return Error::kErrorReadError; - } - - auto name_buffer = std::make_unique(dir_entry.name_length); - if (fread(name_buffer.get(), 1, dir_entry.name_length, file) != - dir_entry.name_length) { - XELOGE("ReadEntrySVOD failed to read directory entry name at 0x{X}", - entry_address); - return Error::kErrorReadError; - } - - auto name = std::string(name_buffer.get(), dir_entry.name_length); - - // Read the left node - if (dir_entry.node_l) { - auto node_result = ReadEntrySVOD(block, dir_entry.node_l, parent); - if (node_result != Error::kSuccess) { - return node_result; - } - } - - // Read file & address of block's data - size_t data_address, data_file; - BlockToOffsetSVOD(dir_entry.data_block, &data_address, &data_file); - - // Create the entry - // NOTE: SVOD entries don't have timestamps for individual files, which can - // cause issues when decrypting games. Using the root entry's timestamp - // solves this issues. - auto entry = StfsContainerEntry::Create(this, parent, name, &files_); - if (dir_entry.attributes & kFileAttributeDirectory) { - // Entry is a directory - entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly; - entry->data_offset_ = 0; - entry->data_size_ = 0; - entry->block_ = block; - entry->access_timestamp_ = root_entry_->create_timestamp(); - entry->create_timestamp_ = root_entry_->create_timestamp(); - entry->write_timestamp_ = root_entry_->create_timestamp(); - - if (dir_entry.length) { - // If length is greater than 0, traverse the directory's children - auto directory_result = - ReadEntrySVOD(dir_entry.data_block, 0, entry.get()); - if (directory_result != Error::kSuccess) { - return directory_result; - } - } - } else { - // Entry is a file - entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly; - entry->size_ = dir_entry.length; - entry->allocation_size_ = xe::round_up(dir_entry.length, kBlockSize); - entry->data_offset_ = data_address; - entry->data_size_ = dir_entry.length; - entry->block_ = dir_entry.data_block; - entry->access_timestamp_ = root_entry_->create_timestamp(); - entry->create_timestamp_ = root_entry_->create_timestamp(); - entry->write_timestamp_ = root_entry_->create_timestamp(); - - // Fill in all block records, sector by sector. - if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) { - uint32_t block_index = dir_entry.data_block; - size_t remaining_size = xe::round_up(dir_entry.length, 0x800); - - size_t last_record = -1; - size_t last_offset = -1; - while (remaining_size) { - const size_t BLOCK_SIZE = 0x800; - - size_t offset, file_index; - BlockToOffsetSVOD(block_index, &offset, &file_index); - - block_index++; - remaining_size -= BLOCK_SIZE; - - if (offset - last_offset == 0x800) { - // Consecutive, so append to last entry. - entry->block_list_[last_record].length += BLOCK_SIZE; - last_offset = offset; - continue; - } - - entry->block_list_.push_back({file_index, offset, BLOCK_SIZE}); - last_record = entry->block_list_.size() - 1; - last_offset = offset; - } - } - } - - parent->children_.emplace_back(std::move(entry)); - - // Read the right node. - if (dir_entry.node_r) { - auto node_result = ReadEntrySVOD(block, dir_entry.node_r, parent); - if (node_result != Error::kSuccess) { - return node_result; - } - } - - return Error::kSuccess; -} - -void StfsContainerDevice::BlockToOffsetSVOD(size_t block, size_t* out_address, - size_t* out_file_index) { - // SVOD Systems use hash blocks for integrity checks. These hash blocks - // cause blocks to be discontinuous in memory, and must be accounted for. - // - Each data block is 0x800 bytes in length - // - Every group of 0x198 data blocks is preceded a Level0 hash table. - // Level0 tables contain 0xCC hashes, each representing two data blocks. - // The total size of each Level0 hash table is 0x1000 bytes in length. - // - Every 0xA1C4 Level0 hash tables is preceded by a Level1 hash table. - // Level1 tables contain 0xCB hashes, each representing two Level0 hashes. - // The total size of each Level1 hash table is 0x1000 bytes in length. - // - Files are split into fragments of 0xA290000 bytes in length, - // consisting of 0x14388 data blocks, 0xCB Level0 hash tables, and 0x1 - // Level1 hash table. - - const size_t BLOCK_SIZE = 0x800; - const size_t HASH_BLOCK_SIZE = 0x1000; - const size_t BLOCKS_PER_L0_HASH = 0x198; - const size_t HASHES_PER_L1_HASH = 0xA1C4; - const size_t BLOCKS_PER_FILE = 0x14388; - const size_t MAX_FILE_SIZE = 0xA290000; - const size_t BLOCK_OFFSET = - header_.metadata.volume_descriptor.svod.start_data_block(); - - // Resolve the true block address and file index - size_t true_block = block - (BLOCK_OFFSET * 2); - if (svod_layout_ == SvodLayoutType::kEnhancedGDF) { - // EGDF has an 0x1000 byte offset, which is two blocks - true_block += 0x2; - } - - size_t file_block = true_block % BLOCKS_PER_FILE; - size_t file_index = true_block / BLOCKS_PER_FILE; - size_t offset = 0; - - // Calculate offset caused by Level0 Hash Tables - size_t level0_table_count = (file_block / BLOCKS_PER_L0_HASH) + 1; - offset += level0_table_count * HASH_BLOCK_SIZE; - - // Calculate offset caused by Level1 Hash Tables - size_t level1_table_count = (level0_table_count / HASHES_PER_L1_HASH) + 1; - offset += level1_table_count * HASH_BLOCK_SIZE; - - // For single-file SVOD layouts, include the size of the header in the offset. - if (svod_layout_ == SvodLayoutType::kSingleFile) { - offset += svod_base_offset_; - } - - size_t block_address = (file_block * BLOCK_SIZE) + offset; - - // If the offset causes the block address to overrun the file, round it. - if (block_address >= MAX_FILE_SIZE) { - file_index += 1; - block_address %= MAX_FILE_SIZE; - block_address += 0x2000; - } - - *out_address = block_address; - *out_file_index = file_index; -} - -StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() { - auto& file = files_.at(0); - - auto root_entry = new StfsContainerEntry(this, nullptr, "", &files_); - root_entry->attributes_ = kFileAttributeDirectory; - root_entry_ = std::unique_ptr(root_entry); - - std::vector all_entries; - - // Load all listings. - StfsDirectoryBlock directory; - - auto& descriptor = header_.metadata.volume_descriptor.stfs; - uint32_t table_block_index = descriptor.file_table_block_number(); - size_t n = 0; - for (n = 0; n < descriptor.file_table_block_count; n++) { - auto offset = BlockToOffsetSTFS(table_block_index); - xe::filesystem::Seek(file, offset, SEEK_SET); - - if (fread(&directory, sizeof(StfsDirectoryBlock), 1, file) != 1) { - XELOGE("ReadSTFS failed to read directory block at 0x{X}", offset); - return Error::kErrorReadError; - } - - for (size_t m = 0; m < kEntriesPerDirectoryBlock; m++) { - auto& dir_entry = directory.entries[m]; - - if (dir_entry.name[0] == 0) { - // Done. - break; - } - - StfsContainerEntry* parent_entry = nullptr; - if (dir_entry.directory_index == 0xFFFF) { - parent_entry = root_entry; - } else { - parent_entry = all_entries[dir_entry.directory_index]; - } - - std::string name(reinterpret_cast(dir_entry.name), - dir_entry.flags.name_length & 0x3F); - auto entry = - StfsContainerEntry::Create(this, parent_entry, name, &files_); - - if (dir_entry.flags.directory) { - entry->attributes_ = kFileAttributeDirectory; - } else { - entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly; - entry->data_offset_ = BlockToOffsetSTFS(dir_entry.start_block_number()); - entry->data_size_ = dir_entry.length; - } - entry->size_ = dir_entry.length; - entry->allocation_size_ = xe::round_up(dir_entry.length, kBlockSize); - - entry->create_timestamp_ = - decode_fat_timestamp(dir_entry.create_date, dir_entry.create_time); - entry->write_timestamp_ = decode_fat_timestamp(dir_entry.modified_date, - dir_entry.modified_time); - entry->access_timestamp_ = entry->write_timestamp_; - - all_entries.push_back(entry.get()); - - // Fill in all block records. - // It's easier to do this now and just look them up later, at the cost - // of some memory. Nasty chain walk. - // TODO(benvanik): optimize if flags.contiguous is set. - if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) { - uint32_t block_index = dir_entry.start_block_number(); - size_t remaining_size = dir_entry.length; - while (remaining_size && block_index != kEndOfChain) { - size_t block_size = - std::min(static_cast(kBlockSize), remaining_size); - size_t offset = BlockToOffsetSTFS(block_index); - entry->block_list_.push_back({0, offset, block_size}); - remaining_size -= block_size; - auto block_hash = GetBlockHash(block_index); - block_index = block_hash->level0_next_block(); - } - - if (remaining_size) { - // Loop above must have exited prematurely, bad hash tables? - XELOGW( - "STFS file {} only found {} bytes for file, expected {} ({} " - "bytes missing)", - name, dir_entry.length - remaining_size, dir_entry.length, - remaining_size); - assert_always(); - } - - // Check that the number of blocks retrieved from hash entries matches - // the block count read from the file entry - if (entry->block_list_.size() != dir_entry.allocated_data_blocks()) { - XELOGW( - "STFS failed to read correct block-chain for entry {}, read {} " - "blocks, expected {}", - entry->name_, entry->block_list_.size(), - dir_entry.allocated_data_blocks()); - assert_always(); - } - } - - parent_entry->children_.emplace_back(std::move(entry)); - } - - auto block_hash = GetBlockHash(table_block_index); - table_block_index = block_hash->level0_next_block(); - if (table_block_index == kEndOfChain) { - break; - } - } - - if (n + 1 != descriptor.file_table_block_count) { - XELOGW("STFS read {} file table blocks, but STFS headers expected {}!", - n + 1, descriptor.file_table_block_count); - assert_always(); - } - - return Error::kSuccess; -} - -size_t StfsContainerDevice::BlockToOffsetSTFS(uint64_t block_index) const { - // For every level there is a hash table - // Level 0: hash table of next 170 blocks - // Level 1: hash table of next 170 hash tables - // Level 2: hash table of next 170 level 1 hash tables - // And so on... - uint64_t base = kBlocksPerHashLevel[0]; - uint64_t block = block_index; - for (uint32_t i = 0; i < 3; i++) { - block += ((block_index + base) / base) * blocks_per_hash_table_; - if (block_index < base) { - break; - } - - base *= kBlocksPerHashLevel[0]; - } - - return xe::round_up(header_.header.header_size, kBlockSize) + (block << 12); -} - -uint32_t StfsContainerDevice::BlockToHashBlockNumberSTFS( - uint32_t block_index, uint32_t hash_level) const { - uint32_t block = 0; - if (hash_level == 0) { - if (block_index < kBlocksPerHashLevel[0]) { - return 0; - } - - block = (block_index / kBlocksPerHashLevel[0]) * block_step[0]; - block += - ((block_index / kBlocksPerHashLevel[1]) + 1) * blocks_per_hash_table_; - - if (block_index < kBlocksPerHashLevel[1]) { - return block; - } - - return block + blocks_per_hash_table_; - } - - if (hash_level == 1) { - if (block_index < kBlocksPerHashLevel[1]) { - return block_step[0]; - } - - block = (block_index / kBlocksPerHashLevel[1]) * block_step[1]; - return block + blocks_per_hash_table_; - } - - // Level 2 is always at blockStep1 - return block_step[1]; -} - -size_t StfsContainerDevice::BlockToHashBlockOffsetSTFS( - uint32_t block_index, uint32_t hash_level) const { - uint64_t block = BlockToHashBlockNumberSTFS(block_index, hash_level); - return xe::round_up(header_.header.header_size, kBlockSize) + (block << 12); -} - -const StfsHashEntry* StfsContainerDevice::GetBlockHash(uint32_t block_index) { - auto& file = files_.at(0); - - auto& descriptor = header_.metadata.volume_descriptor.stfs; - - // Offset for selecting the secondary hash block, in packages that have them - uint32_t secondary_table_offset = - descriptor.flags.bits.root_active_index ? kBlockSize : 0; - - auto hash_offset_lv0 = BlockToHashBlockOffsetSTFS(block_index, 0); - if (!cached_hash_tables_.count(hash_offset_lv0)) { - // If this is read_only_format then it doesn't contain secondary blocks, no - // need to check upper hash levels - if (descriptor.flags.bits.read_only_format) { - secondary_table_offset = 0; - } else { - // Not a read-only package, need to check each levels active index flag to - // see if we need to use secondary block or not - - // Check level1 table if package has it - if (descriptor.total_block_count > kBlocksPerHashLevel[0]) { - auto hash_offset_lv1 = BlockToHashBlockOffsetSTFS(block_index, 1); - - if (!cached_hash_tables_.count(hash_offset_lv1)) { - // Check level2 table if package has it - if (descriptor.total_block_count > kBlocksPerHashLevel[1]) { - auto hash_offset_lv2 = BlockToHashBlockOffsetSTFS(block_index, 2); - - if (!cached_hash_tables_.count(hash_offset_lv2)) { - xe::filesystem::Seek( - file, hash_offset_lv2 + secondary_table_offset, SEEK_SET); - - StfsHashTable table_lv2; - if (fread(&table_lv2, sizeof(StfsHashTable), 1, file) != 1) { - XELOGE("GetBlockHash failed to read level2 hash table at 0x{X}", - hash_offset_lv2 + secondary_table_offset); - return nullptr; - } - cached_hash_tables_[hash_offset_lv2] = table_lv2; - } - - auto record = - (block_index / kBlocksPerHashLevel[1]) % kBlocksPerHashLevel[0]; - auto record_data = - &cached_hash_tables_[hash_offset_lv2].entries[record]; - secondary_table_offset = - record_data->levelN_active_index() ? kBlockSize : 0; - } - - xe::filesystem::Seek(file, hash_offset_lv1 + secondary_table_offset, - SEEK_SET); - - StfsHashTable table_lv1; - if (fread(&table_lv1, sizeof(StfsHashTable), 1, file) != 1) { - XELOGE("GetBlockHash failed to read level1 hash table at 0x{X}", - hash_offset_lv1 + secondary_table_offset); - return nullptr; - } - cached_hash_tables_[hash_offset_lv1] = table_lv1; - } - - auto record = - (block_index / kBlocksPerHashLevel[0]) % kBlocksPerHashLevel[0]; - auto record_data = - &cached_hash_tables_[hash_offset_lv1].entries[record]; - secondary_table_offset = - record_data->levelN_active_index() ? kBlockSize : 0; - } - } - - xe::filesystem::Seek(file, hash_offset_lv0 + secondary_table_offset, - SEEK_SET); - - StfsHashTable table_lv0; - if (fread(&table_lv0, sizeof(StfsHashTable), 1, file) != 1) { - XELOGE("GetBlockHash failed to read level0 hash table at 0x{X}", - hash_offset_lv0 + secondary_table_offset); - return nullptr; - } - cached_hash_tables_[hash_offset_lv0] = table_lv0; - } - - auto record = block_index % kBlocksPerHashLevel[0]; - auto record_data = &cached_hash_tables_[hash_offset_lv0].entries[record]; - - return record_data; -} - -XContentPackageType StfsContainerDevice::ReadMagic( - const std::filesystem::path& path) { - auto map = MappedMemory::Open(path, MappedMemory::Mode::kRead, 0, 4); - return XContentPackageType(xe::load_and_swap(map->data())); -} - -bool StfsContainerDevice::ResolveFromFolder(const std::filesystem::path& path) { - // Scan through folders until a file with magic is found - std::queue queue; - - filesystem::FileInfo folder; - filesystem::GetInfo(host_path_, &folder); - queue.push(folder); - - while (!queue.empty()) { - auto current_file = queue.front(); - queue.pop(); - - if (current_file.type == filesystem::FileInfo::Type::kDirectory) { - auto path = current_file.path / current_file.name; - auto child_files = filesystem::ListFiles(path); - for (auto file : child_files) { - queue.push(file); - } - } else { - // Try to read the file's magic - auto path = current_file.path / current_file.name; - auto magic = ReadMagic(path); - - if (magic == XContentPackageType::kCon || - magic == XContentPackageType::kLive || - magic == XContentPackageType::kPirs) { - host_path_ = current_file.path / current_file.name; - XELOGI("STFS Package found: {}", xe::path_to_utf8(host_path_)); - return true; - } - } - } - - if (host_path_ == path) { - // Could not find a suitable container file - return false; - } - return true; -} - -} // namespace vfs -} // namespace xe diff --git a/src/xenia/vfs/devices/stfs_container_device.h b/src/xenia/vfs/devices/stfs_container_device.h deleted file mode 100644 index 504e22cfd0..0000000000 --- a/src/xenia/vfs/devices/stfs_container_device.h +++ /dev/null @@ -1,148 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2020 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_ -#define XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_ - -#include -#include -#include -#include - -#include "xenia/base/math.h" -#include "xenia/base/string_util.h" -#include "xenia/kernel/util/xex2_info.h" -#include "xenia/vfs/device.h" -#include "xenia/vfs/devices/stfs_xbox.h" - -namespace xe { -namespace vfs { - -// https://free60project.github.io/wiki/STFS.html - -class StfsContainerEntry; - -class StfsContainerDevice : public Device { - public: - const static uint32_t kBlockSize = 0x1000; - - StfsContainerDevice(const std::string_view mount_path, - const std::filesystem::path& host_path); - ~StfsContainerDevice() override; - - bool Initialize() override; - - bool is_read_only() const override { - return header_.metadata.volume_type != XContentVolumeType::kStfs || - header_.metadata.volume_descriptor.stfs.flags.bits.read_only_format; - } - - void Dump(StringBuffer* string_buffer) override; - Entry* ResolvePath(const std::string_view path) override; - - const std::string& name() const override { return name_; } - uint32_t attributes() const override { return 0; } - uint32_t component_name_max_length() const override { return 40; } - - uint32_t total_allocation_units() const override { - if (header_.metadata.volume_type == XContentVolumeType::kStfs) { - return header_.metadata.volume_descriptor.stfs.total_block_count; - } - - return uint32_t(data_size() / sectors_per_allocation_unit() / - bytes_per_sector()); - } - uint32_t available_allocation_units() const override { - if (!is_read_only()) { - auto& descriptor = header_.metadata.volume_descriptor.stfs; - return kBlocksPerHashLevel[2] - - (descriptor.total_block_count - descriptor.free_block_count); - } - return 0; - } - uint32_t sectors_per_allocation_unit() const override { return 8; } - uint32_t bytes_per_sector() const override { return 0x200; } - - size_t data_size() const { - if (header_.header.header_size) { - if (header_.metadata.volume_type == XContentVolumeType::kStfs) { - return header_.metadata.volume_descriptor.stfs.total_block_count * - kBlockSize; - } - return files_total_size_ - - xe::round_up(header_.header.header_size, kBlockSize); - } - return files_total_size_ - sizeof(StfsHeader); - } - - private: - const uint32_t kBlocksPerHashLevel[3] = {170, 28900, 4913000}; - const uint32_t kEndOfChain = 0xFFFFFF; - const uint32_t kEntriesPerDirectoryBlock = - kBlockSize / sizeof(StfsDirectoryEntry); - - enum class Error { - kSuccess = 0, - kErrorOutOfMemory = -1, - kErrorReadError = -10, - kErrorFileMismatch = -30, - kErrorDamagedFile = -31, - kErrorTooSmall = -32, - }; - - enum class SvodLayoutType { - kUnknown = 0x0, - kEnhancedGDF = 0x1, - kXSF = 0x2, - kSingleFile = 0x4, - }; - - XContentPackageType ReadMagic(const std::filesystem::path& path); - bool ResolveFromFolder(const std::filesystem::path& path); - - Error OpenFiles(); - void CloseFiles(); - - Error ReadHeaderAndVerify(FILE* header_file); - - Error ReadSVOD(); - Error ReadEntrySVOD(uint32_t sector, uint32_t ordinal, - StfsContainerEntry* parent); - void BlockToOffsetSVOD(size_t sector, size_t* address, size_t* file_index); - - Error ReadSTFS(); - size_t BlockToOffsetSTFS(uint64_t block_index) const; - uint32_t BlockToHashBlockNumberSTFS(uint32_t block_index, - uint32_t hash_level) const; - size_t BlockToHashBlockOffsetSTFS(uint32_t block_index, - uint32_t hash_level) const; - - const StfsHashEntry* GetBlockHash(uint32_t block_index); - - std::string name_; - std::filesystem::path host_path_; - - std::map files_; - size_t files_total_size_; - - size_t svod_base_offset_; - - std::unique_ptr root_entry_; - StfsHeader header_; - SvodLayoutType svod_layout_; - uint32_t blocks_per_hash_table_; - uint32_t block_step[2]; - - std::unordered_map cached_hash_tables_; -}; - -} // namespace vfs -} // namespace xe - -#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_ diff --git a/src/xenia/vfs/devices/stfs_xbox.h b/src/xenia/vfs/devices/stfs_xbox.h index 0cba213937..8763f0b62c 100644 --- a/src/xenia/vfs/devices/stfs_xbox.h +++ b/src/xenia/vfs/devices/stfs_xbox.h @@ -2,7 +2,7 @@ ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** - * Copyright 2021 Ben Vanik. All rights reserved. * + * Copyright 2022 Ben Vanik. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ @@ -13,9 +13,33 @@ #include "xenia/base/string_util.h" #include "xenia/kernel/util/xex2_info.h" +#if XE_PLATFORM_WIN32 +#include "xenia/base/platform_win.h" +#define timegm _mkgmtime +#endif + namespace xe { namespace vfs { +// Convert FAT timestamp to 100-nanosecond intervals since January 1, 1601 (UTC) +inline uint64_t decode_fat_timestamp(uint32_t date, uint32_t time) { + struct tm tm = {0}; + // 80 is the difference between 1980 (FAT) and 1900 (tm); + tm.tm_year = ((0xFE00 & date) >> 9) + 80; + tm.tm_mon = ((0x01E0 & date) >> 5) - 1; + tm.tm_mday = (0x001F & date) >> 0; + tm.tm_hour = (0xF800 & time) >> 11; + tm.tm_min = (0x07E0 & time) >> 5; + tm.tm_sec = (0x001F & time) << 1; // the value stored in 2-seconds intervals + tm.tm_isdst = 0; + time_t timet = timegm(&tm); + if (timet == -1) { + return 0; + } + // 11644473600LL is a difference between 1970 and 1601 + return (timet + 11644473600LL) * 10000000; +} + // Structs used for interchange between Xenia and actual Xbox360 kernel/XAM inline uint32_t load_uint24_be(const uint8_t* p) { @@ -455,13 +479,21 @@ struct XContentHeader { static_assert_size(XContentHeader, 0x344); #pragma pack(pop) -struct StfsHeader { - XContentHeader header; - XContentMetadata metadata; +struct XContentContainerHeader { + XContentHeader content_header; + XContentMetadata content_metadata; // TODO: title/system updates contain more data after XContentMetadata, seems // to affect header.header_size + + bool is_package_readonly() const { + if (content_metadata.volume_type == vfs::XContentVolumeType::kSvod) { + return true; + } + + return content_metadata.volume_descriptor.stfs.flags.bits.read_only_format; + } }; -static_assert_size(StfsHeader, 0x971A); +static_assert_size(XContentContainerHeader, 0x971A); } // namespace vfs } // namespace xe diff --git a/src/xenia/vfs/devices/xcontent_container_device.cc b/src/xenia/vfs/devices/xcontent_container_device.cc new file mode 100644 index 0000000000..49c3ccc5f2 --- /dev/null +++ b/src/xenia/vfs/devices/xcontent_container_device.cc @@ -0,0 +1,165 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2023 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/base/logging.h" +#include "xenia/vfs/devices/xcontent_container_device.h" +#include "xenia/vfs/devices/xcontent_devices/stfs_container_device.h" +#include "xenia/vfs/devices/xcontent_devices/svod_container_device.h" + +namespace xe { +namespace vfs { + +std::unique_ptr XContentContainerDevice::CreateContentDevice( + const std::string_view mount_path, const std::filesystem::path& host_path) { + if (!std::filesystem::exists(host_path)) { + XELOGE("Path to XContent container does not exist: {}", + xe::path_to_utf8(host_path)); + return nullptr; + } + + if (std::filesystem::is_directory(host_path)) { + return nullptr; + } + + FILE* host_file = xe::filesystem::OpenFile(host_path, "rb"); + if (!host_file) { + XELOGE("Error opening XContent file."); + return nullptr; + } + + const uint64_t package_size = std::filesystem::file_size(host_path); + if (package_size < sizeof(XContentContainerHeader)) { + return nullptr; + } + + const auto header = XContentContainerDevice::ReadContainerHeader(host_file); + if (header == nullptr) { + return nullptr; + } + + fclose(host_file); + + if (!header->content_header.is_magic_valid()) { + return nullptr; + } + + switch (header->content_metadata.volume_type) { + case XContentVolumeType::kStfs: + return std::make_unique(mount_path, host_path); + break; + case XContentVolumeType::kSvod: + return std::make_unique(mount_path, host_path); + break; + default: + break; + } + + return nullptr; +} + +XContentContainerDevice::XContentContainerDevice( + const std::string_view mount_path, const std::filesystem::path& host_path) + : Device(mount_path), + name_("XContent"), + host_path_(host_path), + files_total_size_(0), + header_(std::make_unique()) {} + +XContentContainerDevice::~XContentContainerDevice() {} + +bool XContentContainerDevice::Initialize() { + if (!std::filesystem::exists(host_path_)) { + XELOGE("Path to XContent container does not exist: {}", + xe::path_to_utf8(host_path_)); + return false; + } + + if (std::filesystem::is_directory(host_path_)) { + return false; + } + + XELOGI("Loading XContent header file: {}", xe::path_to_utf8(host_path_)); + auto header_file = xe::filesystem::OpenFile(host_path_, "rb"); + if (!header_file) { + XELOGE("Error opening XContent header file."); + return false; + } + + auto header_loading_result = ReadHeaderAndVerify(header_file); + if (header_loading_result != Result::kSuccess) { + XELOGE("Error reading XContent header: {}", header_loading_result); + fclose(header_file); + return false; + } + + SetupContainer(); + + if (LoadHostFiles(header_file) != Result::kSuccess) { + XELOGE("Error loading XContent host files."); + return false; + } + + return Read() == Result::kSuccess; +} + +XContentContainerHeader* XContentContainerDevice::ReadContainerHeader( + FILE* host_file) { + XContentContainerHeader* header = new XContentContainerHeader(); + // Read header & check signature + if (fread(header, sizeof(XContentContainerHeader), 1, host_file) != 1) { + return nullptr; + } + return header; +} + +Entry* XContentContainerDevice::ResolvePath(const std::string_view path) { + // The filesystem will have stripped our prefix off already, so the path will + // be in the form: + // some\PATH.foo + XELOGFS("StfsContainerDevice::ResolvePath({})", path); + return root_entry_->ResolvePath(path); +} + +void XContentContainerDevice::Dump(StringBuffer* string_buffer) { + auto global_lock = global_critical_region_.Acquire(); + root_entry_->Dump(string_buffer, 0); +} + +void XContentContainerDevice::CloseFiles() { + for (auto& file : files_) { + fclose(file.second); + } + files_.clear(); + files_total_size_ = 0; +} + +XContentContainerDevice::Result XContentContainerDevice::ReadHeaderAndVerify( + FILE* header_file) { + files_total_size_ = std::filesystem::file_size(host_path_); + if (files_total_size_ < sizeof(XContentContainerHeader)) { + return Result::kTooSmall; + } + + const XContentContainerHeader* header = ReadContainerHeader(header_file); + if (header == nullptr) { + return Result::kReadError; + } + + std::memcpy(header_.get(), header, sizeof(XContentContainerHeader)); + + if (!header_->content_header.is_magic_valid()) { + // Unexpected format. + return Result::kFileMismatch; + } + + return Result::kSuccess; +} + +} // namespace vfs +} // namespace xe \ No newline at end of file diff --git a/src/xenia/vfs/devices/xcontent_container_device.h b/src/xenia/vfs/devices/xcontent_container_device.h new file mode 100644 index 0000000000..954565c09d --- /dev/null +++ b/src/xenia/vfs/devices/xcontent_container_device.h @@ -0,0 +1,98 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2023 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_VFS_DEVICES_XCONTENT_CONTAINER_DEVICE_H_ +#define XENIA_VFS_DEVICES_XCONTENT_CONTAINER_DEVICE_H_ + +#include +#include +#include + +#include "xenia/base/math.h" +#include "xenia/kernel/util/xex2_info.h" +#include "xenia/vfs/device.h" +#include "xenia/vfs/devices/stfs_xbox.h" + +namespace xe { +namespace vfs { +class XContentContainerDevice : public Device { + public: + const static uint32_t kBlockSize = 0x1000; + + static std::unique_ptr CreateContentDevice( + const std::string_view mount_path, + const std::filesystem::path& host_path); + + ~XContentContainerDevice() override; + + bool Initialize(); + + const std::string& name() const override { return name_; } + uint32_t attributes() const override { return 0; } + + uint32_t sectors_per_allocation_unit() const override { return 8; } + uint32_t bytes_per_sector() const override { return 0x200; } + + size_t data_size() const { + if (header_->content_header.header_size) { + return files_total_size_ - + xe::round_up(header_->content_header.header_size, kBlockSize); + } + return files_total_size_ - sizeof(XContentContainerHeader); + } + + protected: + XContentContainerDevice(const std::string_view mount_path, + const std::filesystem::path& host_path); + enum class Result { + kSuccess = 0, + kOutOfMemory = -1, + kReadError = -10, + kFileMismatch = -30, + kDamagedFile = -31, + kTooSmall = -32, + }; + + virtual Result Read() = 0; + // Load all host files. Usually STFS is only 1 file, meanwhile SVOD is usually multiple file. + virtual Result LoadHostFiles(FILE* header_file) = 0; + // Initialize any container specific fields. + virtual void SetupContainer() { }; + + Entry* ResolvePath(const std::string_view path); + void CloseFiles(); + void Dump(StringBuffer* string_buffer); + Result ReadHeaderAndVerify(FILE* header_file); + + void SetName(std::string name) { name_ = name; } + const std::string& GetName() const { return name_; } + + void SetFilesSize(uint64_t files_size) { files_total_size_ = files_size; } + const uint64_t GetFilesSize() const { return files_total_size_; } + + const std::filesystem::path& GetHostPath() const { return host_path_; } + + const XContentContainerHeader* GetContainerHeader() const { return header_.get(); } + + std::string name_; + std::filesystem::path host_path_; + + std::map files_; + size_t files_total_size_; + std::unique_ptr root_entry_; + std::unique_ptr header_; + + private: + static XContentContainerHeader* ReadContainerHeader(FILE* host_file); +}; + +} // namespace vfs +} // namespace xe + +#endif diff --git a/src/xenia/vfs/devices/stfs_container_entry.cc b/src/xenia/vfs/devices/xcontent_container_entry.cc similarity index 51% rename from src/xenia/vfs/devices/stfs_container_entry.cc rename to src/xenia/vfs/devices/xcontent_container_entry.cc index 1197460cc9..1f264dfec5 100644 --- a/src/xenia/vfs/devices/stfs_container_entry.cc +++ b/src/xenia/vfs/devices/xcontent_container_entry.cc @@ -2,43 +2,43 @@ ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** - * Copyright 2020 Ben Vanik. All rights reserved. * + * Copyright 2023 Ben Vanik. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ -#include "xenia/vfs/devices/stfs_container_entry.h" -#include "xenia/base/math.h" -#include "xenia/vfs/devices/stfs_container_file.h" +#include "xenia/vfs/devices/xcontent_container_entry.h" +#include "xenia/vfs/devices/xcontent_container_file.h" #include namespace xe { namespace vfs { -StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent, - const std::string_view path, - MultiFileHandles* files) +XContentContainerEntry::XContentContainerEntry(Device* device, Entry* parent, + const std::string_view path, + MultiFileHandles* files) : Entry(device, parent, path), files_(files), data_offset_(0), data_size_(0), block_(0) {} -StfsContainerEntry::~StfsContainerEntry() = default; +XContentContainerEntry::~XContentContainerEntry() = default; -std::unique_ptr StfsContainerEntry::Create( +std::unique_ptr XContentContainerEntry::Create( Device* device, Entry* parent, const std::string_view name, MultiFileHandles* files) { auto path = xe::utf8::join_guest_paths(parent->path(), name); auto entry = - std::make_unique(device, parent, path, files); + std::make_unique(device, parent, path, files); return std::move(entry); } -X_STATUS StfsContainerEntry::Open(uint32_t desired_access, File** out_file) { - *out_file = new StfsContainerFile(desired_access, this); +X_STATUS XContentContainerEntry::Open(uint32_t desired_access, + File** out_file) { + *out_file = new XContentContainerFile(desired_access, this); return X_STATUS_SUCCESS; } diff --git a/src/xenia/vfs/devices/stfs_container_entry.h b/src/xenia/vfs/devices/xcontent_container_entry.h similarity index 63% rename from src/xenia/vfs/devices/stfs_container_entry.h rename to src/xenia/vfs/devices/xcontent_container_entry.h index 0990b97ce7..8022a088ab 100644 --- a/src/xenia/vfs/devices/stfs_container_entry.h +++ b/src/xenia/vfs/devices/xcontent_container_entry.h @@ -2,13 +2,13 @@ ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** - * Copyright 2020 Ben Vanik. All rights reserved. * + * Copyright 2023 Ben Vanik. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ -#ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_ -#define XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_ +#ifndef XENIA_VFS_DEVICES_XCONTENT_CONTAINER_ENTRY_H_ +#define XENIA_VFS_DEVICES_XCONTENT_CONTAINER_ENTRY_H_ #include #include @@ -21,18 +21,17 @@ namespace xe { namespace vfs { typedef std::map MultiFileHandles; -class StfsContainerDevice; +class XContentContainerDevice; -class StfsContainerEntry : public Entry { +class XContentContainerEntry : public Entry { public: - StfsContainerEntry(Device* device, Entry* parent, const std::string_view path, - MultiFileHandles* files); - ~StfsContainerEntry() override; + XContentContainerEntry(Device* device, Entry* parent, + const std::string_view path, MultiFileHandles* files); + ~XContentContainerEntry() override; - static std::unique_ptr Create(Device* device, - Entry* parent, - const std::string_view name, - MultiFileHandles* files); + static std::unique_ptr Create( + Device* device, Entry* parent, const std::string_view name, + MultiFileHandles* files); MultiFileHandles* files() const { return files_; } size_t data_offset() const { return data_offset_; } @@ -50,6 +49,7 @@ class StfsContainerEntry : public Entry { private: friend class StfsContainerDevice; + friend class SvodContainerDevice; MultiFileHandles* files_; size_t data_offset_; @@ -61,4 +61,4 @@ class StfsContainerEntry : public Entry { } // namespace vfs } // namespace xe -#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_ \ No newline at end of file +#endif // XENIA_VFS_DEVICES_XCONTENT_CONTAINER_ENTRY_H_ \ No newline at end of file diff --git a/src/xenia/vfs/devices/stfs_container_file.cc b/src/xenia/vfs/devices/xcontent_container_file.cc similarity index 77% rename from src/xenia/vfs/devices/stfs_container_file.cc rename to src/xenia/vfs/devices/xcontent_container_file.cc index 791d791016..6c2b918164 100644 --- a/src/xenia/vfs/devices/stfs_container_file.cc +++ b/src/xenia/vfs/devices/xcontent_container_file.cc @@ -2,31 +2,30 @@ ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** - * Copyright 2014 Ben Vanik. All rights reserved. * + * Copyright 2023 Ben Vanik. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ -#include "xenia/vfs/devices/stfs_container_file.h" - #include #include #include "xenia/base/math.h" -#include "xenia/vfs/devices/stfs_container_entry.h" +#include "xenia/vfs/devices/xcontent_container_entry.h" +#include "xenia/vfs/devices/xcontent_container_file.h" namespace xe { namespace vfs { -StfsContainerFile::StfsContainerFile(uint32_t file_access, - StfsContainerEntry* entry) +XContentContainerFile::XContentContainerFile(uint32_t file_access, + XContentContainerEntry* entry) : File(file_access, entry), entry_(entry) {} -StfsContainerFile::~StfsContainerFile() = default; +XContentContainerFile::~XContentContainerFile() = default; -void StfsContainerFile::Destroy() { delete this; } +void XContentContainerFile::Destroy() { delete this; } -X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length, +X_STATUS XContentContainerFile::ReadSync(void* buffer, size_t buffer_length, size_t byte_offset, size_t* out_bytes_read) { if (byte_offset >= entry_->size()) { diff --git a/src/xenia/vfs/devices/stfs_container_file.h b/src/xenia/vfs/devices/xcontent_container_file.h similarity index 68% rename from src/xenia/vfs/devices/stfs_container_file.h rename to src/xenia/vfs/devices/xcontent_container_file.h index d680dffe52..e2b46bb7af 100644 --- a/src/xenia/vfs/devices/stfs_container_file.h +++ b/src/xenia/vfs/devices/xcontent_container_file.h @@ -2,13 +2,13 @@ ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** - * Copyright 2014 Ben Vanik. All rights reserved. * + * Copyright 2023 Ben Vanik. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ -#ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_FILE_H_ -#define XENIA_VFS_DEVICES_STFS_CONTAINER_FILE_H_ +#ifndef XENIA_VFS_DEVICES_XCONTENT_CONTAINER_FILE_H_ +#define XENIA_VFS_DEVICES_XCONTENT_CONTAINER_FILE_H_ #include "xenia/vfs/file.h" @@ -17,12 +17,12 @@ namespace xe { namespace vfs { -class StfsContainerEntry; +class XContentContainerEntry; -class StfsContainerFile : public File { +class XContentContainerFile : public File { public: - StfsContainerFile(uint32_t file_access, StfsContainerEntry* entry); - ~StfsContainerFile() override; + XContentContainerFile(uint32_t file_access, XContentContainerEntry* entry); + ~XContentContainerFile() override; void Destroy() override; @@ -35,10 +35,10 @@ class StfsContainerFile : public File { X_STATUS SetLength(size_t length) override { return X_STATUS_ACCESS_DENIED; } private: - StfsContainerEntry* entry_; + XContentContainerEntry* entry_; }; } // namespace vfs } // namespace xe -#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_FILE_H_ +#endif // XENIA_VFS_DEVICES_XCONTENT_CONTAINER_FILE_H_ diff --git a/src/xenia/vfs/devices/xcontent_devices/stfs_container_device.cc b/src/xenia/vfs/devices/xcontent_devices/stfs_container_device.cc new file mode 100644 index 0000000000..c241c2c019 --- /dev/null +++ b/src/xenia/vfs/devices/xcontent_devices/stfs_container_device.cc @@ -0,0 +1,324 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2023 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include +#include + +#include "xenia/base/logging.h" +#include "xenia/kernel/xam/content_manager.h" +#include "xenia/vfs/devices/xcontent_container_entry.h" +#include "xenia/vfs/devices/xcontent_devices/stfs_container_device.h" + +namespace xe { +namespace vfs { + +StfsContainerDevice::StfsContainerDevice(const std::string_view mount_path, + const std::filesystem::path& host_path) + : XContentContainerDevice(mount_path, host_path), + blocks_per_hash_table_(1), + block_step_{0, 0} { + SetName("STFS"); +} + +StfsContainerDevice::~StfsContainerDevice() { CloseFiles(); } + +void StfsContainerDevice::SetupContainer() { + // Additional part specific to STFS container. + const XContentContainerHeader* header = GetContainerHeader(); + blocks_per_hash_table_ = header->is_package_readonly() ? 1 : 2; + + block_step_[0] = kBlocksPerHashLevel[0] + blocks_per_hash_table_; + block_step_[1] = kBlocksPerHashLevel[1] + + ((kBlocksPerHashLevel[0] + 1) * blocks_per_hash_table_); +} + +XContentContainerDevice::Result StfsContainerDevice::LoadHostFiles( + FILE* header_file) { + const XContentContainerHeader* header = GetContainerHeader(); + + if (header->content_metadata.data_file_count > 0) { + XELOGW("STFS container is not a single file. Loading might fail!"); + } + + files_.emplace(std::make_pair(0, header_file)); + return Result::kSuccess; +} + +StfsContainerDevice::Result StfsContainerDevice::Read() { + auto& file = files_.at(0); + + auto root_entry = new XContentContainerEntry(this, nullptr, "", &files_); + root_entry->attributes_ = kFileAttributeDirectory; + root_entry_ = std::unique_ptr(root_entry); + + std::vector all_entries; + + // Load all listings. + StfsDirectoryBlock directory; + + auto& descriptor = + GetContainerHeader()->content_metadata.volume_descriptor.stfs; + uint32_t table_block_index = descriptor.file_table_block_number(); + size_t n = 0; + for (n = 0; n < descriptor.file_table_block_count; n++) { + const size_t offset = BlockToOffset(table_block_index); + xe::filesystem::Seek(file, offset, SEEK_SET); + + if (fread(&directory, sizeof(StfsDirectoryBlock), 1, file) != 1) { + XELOGE("ReadSTFS failed to read directory block at 0x{X}", offset); + return Result::kReadError; + } + + for (size_t m = 0; m < kEntriesPerDirectoryBlock; m++) { + const StfsDirectoryEntry& dir_entry = directory.entries[m]; + + if (dir_entry.name[0] == 0) { + // Done. + break; + } + + XContentContainerEntry* parent_entry = + dir_entry.directory_index == 0xFFFF + ? root_entry + : all_entries[dir_entry.directory_index]; + + std::unique_ptr entry = + ReadEntry(parent_entry, &files_, &dir_entry); + all_entries.push_back(entry.get()); + parent_entry->children_.emplace_back(std::move(entry)); + } + + const StfsHashEntry* block_hash = GetBlockHash(table_block_index); + table_block_index = block_hash->level0_next_block(); + if (table_block_index == kEndOfChain) { + break; + } + } + + if (n + 1 != descriptor.file_table_block_count) { + XELOGW("STFS read {} file table blocks, but STFS headers expected {}!", + n + 1, descriptor.file_table_block_count); + assert_always(); + } + + return Result::kSuccess; +} + +std::unique_ptr StfsContainerDevice::ReadEntry( + Entry* parent, MultiFileHandles* files, + const StfsDirectoryEntry* dir_entry) { + std::string name(reinterpret_cast(dir_entry->name), + dir_entry->flags.name_length & 0x3F); + + auto entry = XContentContainerEntry::Create(this, parent, name, &files_); + + if (dir_entry->flags.directory) { + entry->attributes_ = kFileAttributeDirectory; + } else { + entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly; + entry->data_offset_ = BlockToOffset(dir_entry->start_block_number()); + entry->data_size_ = dir_entry->length; + } + entry->size_ = dir_entry->length; + entry->allocation_size_ = xe::round_up(dir_entry->length, kBlockSize); + + entry->create_timestamp_ = + decode_fat_timestamp(dir_entry->create_date, dir_entry->create_time); + entry->write_timestamp_ = + decode_fat_timestamp(dir_entry->modified_date, dir_entry->modified_time); + entry->access_timestamp_ = entry->write_timestamp_; + + // Fill in all block records. + // It's easier to do this now and just look them up later, at the cost + // of some memory. Nasty chain walk. + // TODO(benvanik): optimize if flags.contiguous is set. + if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) { + uint32_t block_index = dir_entry->start_block_number(); + size_t remaining_size = dir_entry->length; + while (remaining_size && block_index != kEndOfChain) { + size_t block_size = + std::min(static_cast(kBlockSize), remaining_size); + size_t offset = BlockToOffset(block_index); + entry->block_list_.push_back({0, offset, block_size}); + remaining_size -= block_size; + auto block_hash = GetBlockHash(block_index); + block_index = block_hash->level0_next_block(); + } + + if (remaining_size) { + // Loop above must have exited prematurely, bad hash tables? + XELOGW( + "STFS file {} only found {} bytes for file, expected {} ({} " + "bytes missing)", + name, dir_entry->length - remaining_size, dir_entry->length, + remaining_size); + assert_always(); + } + + // Check that the number of blocks retrieved from hash entries matches + // the block count read from the file entry + if (entry->block_list_.size() != dir_entry->allocated_data_blocks()) { + XELOGW( + "STFS failed to read correct block-chain for entry {}, read {} " + "blocks, expected {}", + entry->name_, entry->block_list_.size(), + dir_entry->allocated_data_blocks()); + assert_always(); + } + } + + return entry; +} + +size_t StfsContainerDevice::BlockToOffset(uint64_t block_index) const { + // For every level there is a hash table + // Level 0: hash table of next 170 blocks + // Level 1: hash table of next 170 hash tables + // Level 2: hash table of next 170 level 1 hash tables + // And so on... + uint64_t block = block_index; + for (uint32_t i = 0; i < kBlocksHashLevelAmount; i++) { + const uint32_t level_base = kBlocksPerHashLevel[i]; + block += ((block_index + level_base) / level_base) * blocks_per_hash_table_; + if (block_index < level_base) { + break; + } + } + + return xe::round_up(GetContainerHeader()->content_header.header_size, + kBlockSize) + + (block << 12); +} + +uint32_t StfsContainerDevice::BlockToHashBlockNumber( + uint32_t block_index, uint32_t hash_level) const { + if (hash_level == 2) { + return block_step_[1]; + } + + if (block_index < kBlocksPerHashLevel[hash_level]) { + return hash_level == 0 ? 0 : block_step_[hash_level - 1]; + } + + uint32_t block = + (block_index / kBlocksPerHashLevel[hash_level]) * block_step_[hash_level]; + + if (hash_level == 0) { + block += + ((block_index / kBlocksPerHashLevel[1]) + 1) * blocks_per_hash_table_; + + if (block_index < kBlocksPerHashLevel[1]) { + return block; + } + } + + return block + blocks_per_hash_table_; +} + +size_t StfsContainerDevice::BlockToHashBlockOffset(uint32_t block_index, + uint32_t hash_level) const { + const uint64_t block = BlockToHashBlockNumber(block_index, hash_level); + return xe::round_up(header_->content_header.header_size, kBlockSize) + + (block << 12); +} + +const uint8_t StfsContainerDevice::GetAmountOfHashLevelsToCheck( + uint32_t total_block_count) const { + for (uint8_t level = 0; level < kBlocksHashLevelAmount; level++) { + if (total_block_count < kBlocksPerHashLevel[level]) { + return level; + } + } + XELOGE("GetAmountOfHashLevelsToCheck - Invalid total_block_count: {}", + total_block_count); + return 0; +} + +void StfsContainerDevice::UpdateCachedHashTable( + uint32_t block_index, uint8_t hash_level, + uint32_t& secondary_table_offset) { + const size_t hash_offset = BlockToHashBlockOffset(block_index, hash_level); + // Do nothing. It's already there. + if (!cached_hash_tables_.count(hash_offset)) { + auto& file = files_.at(0); + xe::filesystem::Seek(file, hash_offset + secondary_table_offset, SEEK_SET); + StfsHashTable table; + if (fread(&table, sizeof(StfsHashTable), 1, file) != 1) { + XELOGE("GetBlockHash failed to read level{} hash table at 0x{X}", + hash_level, hash_offset + secondary_table_offset); + return; + } + cached_hash_tables_[hash_offset] = table; + } + + uint32_t record = block_index % kBlocksPerHashLevel[0]; + if (hash_level >= 1) { + record = (block_index / kBlocksPerHashLevel[hash_level - 1]) % + kBlocksPerHashLevel[0]; + } + const StfsHashEntry* record_data = + &cached_hash_tables_[hash_offset].entries[record]; + secondary_table_offset = record_data->levelN_active_index() ? kBlockSize : 0; +} + +void StfsContainerDevice::UpdateCachedHashTables( + uint32_t block_index, uint8_t highest_hash_level_to_update, + uint32_t& secondary_table_offset) { + for (int8_t level = highest_hash_level_to_update; level >= 0; level--) { + UpdateCachedHashTable(block_index, level, secondary_table_offset); + } +} + +const StfsHashEntry* StfsContainerDevice::GetBlockHash(uint32_t block_index) { + auto& file = files_.at(0); + + const StfsVolumeDescriptor& descriptor = + header_->content_metadata.volume_descriptor.stfs; + + // Offset for selecting the secondary hash block, in packages that have them + uint32_t secondary_table_offset = + descriptor.flags.bits.root_active_index ? kBlockSize : 0; + + uint8_t hash_levels_to_process = + GetAmountOfHashLevelsToCheck(descriptor.total_block_count); + + if (header_->is_package_readonly()) { + secondary_table_offset = 0; + // Because we have read only package we only need to check first hash level. + hash_levels_to_process = 0; + } + + UpdateCachedHashTables(block_index, hash_levels_to_process, + secondary_table_offset); + + const size_t hash_offset = BlockToHashBlockOffset(block_index, 0); + const uint32_t record = block_index % kBlocksPerHashLevel[0]; + return &cached_hash_tables_[hash_offset].entries[record]; +} + +const uint8_t StfsContainerDevice::GetBlocksPerHashTableFromContainerHeader() + const { + const XContentContainerHeader* header = GetContainerHeader(); + if (!header) { + XELOGE( + "VFS: SetBlocksPerHashTableBasedOnContainerHeader - Missing " + "Container " + "Header!"); + return 0; + } + + if (header->is_package_readonly()) { + return 1; + } + + return 2; +} + +} // namespace vfs +} // namespace xe diff --git a/src/xenia/vfs/devices/xcontent_devices/stfs_container_device.h b/src/xenia/vfs/devices/xcontent_devices/stfs_container_device.h new file mode 100644 index 0000000000..828e00921b --- /dev/null +++ b/src/xenia/vfs/devices/xcontent_devices/stfs_container_device.h @@ -0,0 +1,95 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2023 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_VFS_DEVICES_XCONTENT_DEVICES_STFS_CONTAINER_DEVICE_H_ +#define XENIA_VFS_DEVICES_XCONTENT_DEVICES_STFS_CONTAINER_DEVICE_H_ + +#include +#include + +#include "xenia/base/string_util.h" +#include "xenia/kernel/util/xex2_info.h" +#include "xenia/vfs/device.h" +#include "xenia/vfs/devices/stfs_xbox.h" +#include "xenia/vfs/devices/xcontent_container_device.h" +#include "xenia/vfs/devices/xcontent_container_entry.h" + +namespace xe { +namespace vfs { + +// https://free60project.github.io/wiki/STFS.html + +class StfsContainerDevice : public XContentContainerDevice { + public: + StfsContainerDevice(const std::string_view mount_path, + const std::filesystem::path& host_path); + ~StfsContainerDevice() override; + + bool is_read_only() const override { + return GetContainerHeader() + ->content_metadata.volume_descriptor.stfs.flags.bits.read_only_format; + } + + uint32_t component_name_max_length() const override { return 40; } + + uint32_t total_allocation_units() const override { + return GetContainerHeader() + ->content_metadata.volume_descriptor.stfs.total_block_count; + } + uint32_t available_allocation_units() const override { + if (!is_read_only()) { + auto& descriptor = + GetContainerHeader()->content_metadata.volume_descriptor.stfs; + return kBlocksPerHashLevel[2] - + (descriptor.total_block_count - descriptor.free_block_count); + } + return 0; + } + + private: + static const uint8_t kBlocksHashLevelAmount = 3; + const uint32_t kBlocksPerHashLevel[kBlocksHashLevelAmount] = {170, 28900, + 4913000}; + const uint32_t kEndOfChain = 0xFFFFFF; + const uint32_t kEntriesPerDirectoryBlock = + kBlockSize / sizeof(StfsDirectoryEntry); + + void SetupContainer() override; + Result LoadHostFiles(FILE* header_file) override; + + Result Read() override; + std::unique_ptr ReadEntry( + Entry* parent, MultiFileHandles* files, + const StfsDirectoryEntry* dir_entry); + + size_t BlockToOffset(uint64_t block_index) const; + uint32_t BlockToHashBlockNumber(uint32_t block_index, + uint32_t hash_level) const; + size_t BlockToHashBlockOffset(uint32_t block_index, + uint32_t hash_level) const; + const uint8_t GetAmountOfHashLevelsToCheck(uint32_t total_block_count) const; + + const StfsHashEntry* GetBlockHash(uint32_t block_index); + void UpdateCachedHashTable(uint32_t block_index, uint8_t hash_level, + uint32_t& secondary_table_offset); + void UpdateCachedHashTables(uint32_t block_index, + uint8_t highest_hash_level_to_update, + uint32_t& secondary_table_offset); + const uint8_t GetBlocksPerHashTableFromContainerHeader() const; + + uint8_t blocks_per_hash_table_; + uint32_t block_step_[2]; + + std::unordered_map cached_hash_tables_; +}; + +} // namespace vfs +} // namespace xe + +#endif // XENIA_VFS_DEVICES_XCONTENT_DEVICES_STFS_CONTAINER_DEVICE_H_ diff --git a/src/xenia/vfs/devices/xcontent_devices/svod_container_device.cc b/src/xenia/vfs/devices/xcontent_devices/svod_container_device.cc new file mode 100644 index 0000000000..369ecf10dc --- /dev/null +++ b/src/xenia/vfs/devices/xcontent_devices/svod_container_device.cc @@ -0,0 +1,417 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2023 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include +#include + +#include "xenia/base/logging.h" +#include "xenia/vfs/devices/xcontent_container_device.h" +#include "xenia/vfs/devices/xcontent_container_entry.h" +#include "xenia/vfs/devices/xcontent_devices/svod_container_device.h" + +namespace xe { +namespace vfs { + +SvodContainerDevice::SvodContainerDevice(const std::string_view mount_path, + const std::filesystem::path& host_path) + : XContentContainerDevice(mount_path, host_path), + svod_base_offset_(), + svod_layout_() { + SetName("FATX"); +} + +SvodContainerDevice::~SvodContainerDevice() { CloseFiles(); } + +SvodContainerDevice::Result SvodContainerDevice::LoadHostFiles( + FILE* header_file) { + std::filesystem::path data_fragment_path = host_path_; + data_fragment_path += ".data"; + if (!std::filesystem::exists(data_fragment_path)) { + XELOGE("STFS container is multi-file, but path {} does not exist.", + xe::path_to_utf8(data_fragment_path)); + return Result::kFileMismatch; + } + + // Ensure data fragment files are sorted + auto fragment_files = filesystem::ListFiles(data_fragment_path); + std::sort(fragment_files.begin(), fragment_files.end(), + [](filesystem::FileInfo& left, filesystem::FileInfo& right) { + return left.name < right.name; + }); + + if (fragment_files.size() != header_->content_metadata.data_file_count) { + XELOGE("SVOD expecting {} data fragments, but {} are present.", + header_->content_metadata.data_file_count, fragment_files.size()); + return Result::kFileMismatch; + } + + for (size_t i = 0; i < fragment_files.size(); i++) { + auto& fragment = fragment_files.at(i); + auto path = fragment.path / fragment.name; + auto file = xe::filesystem::OpenFile(path, "rb"); + if (!file) { + XELOGI("Failed to map SVOD file {}.", xe::path_to_utf8(path)); + CloseFiles(); + return Result::kReadError; + } + + xe::filesystem::Seek(file, 0L, SEEK_END); + files_total_size_ += xe::filesystem::Tell(file); + // no need to seek back, any reads from this file will seek first anyway + files_.emplace(std::make_pair(i, file)); + } + XELOGI("SVOD successfully mapped {} files.", fragment_files.size()); + return Result::kSuccess; +} + +XContentContainerDevice::Result SvodContainerDevice::Read() { + // SVOD Systems can have different layouts. The root block is + // denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in + // the first "actual" data fragment of the system. + auto& svod_header = files_.at(0); + + size_t magic_offset; + SetLayout(svod_header, magic_offset); + + // Parse the root directory + xe::filesystem::Seek(svod_header, magic_offset + 0x14, SEEK_SET); + + struct { + uint32_t block; + uint32_t size; + uint32_t creation_date; + uint32_t creation_time; + } root_data; + static_assert_size(root_data, 0x10); + + if (fread(&root_data, sizeof(root_data), 1, svod_header) != 1) { + XELOGE("ReadSVOD failed to read root block data at 0x{X}", + magic_offset + 0x14); + return Result::kReadError; + } + + const uint64_t root_creation_timestamp = + decode_fat_timestamp(root_data.creation_date, root_data.creation_time); + + auto root_entry = new XContentContainerEntry(this, nullptr, "", &files_); + root_entry->attributes_ = kFileAttributeDirectory; + root_entry->access_timestamp_ = root_creation_timestamp; + root_entry->create_timestamp_ = root_creation_timestamp; + root_entry->write_timestamp_ = root_creation_timestamp; + root_entry_ = std::unique_ptr(root_entry); + + // Traverse all child entries + return ReadEntry(root_data.block, 0, root_entry); +} + +SvodContainerDevice::Result SvodContainerDevice::ReadEntry( + uint32_t block, uint32_t ordinal, XContentContainerEntry* parent) { + // For games with a large amount of files, the ordinal offset can overrun + // the current block and potentially hit a hash block. + size_t ordinal_offset = ordinal * 0x4; + size_t block_offset = ordinal_offset / 0x800; + size_t true_ordinal_offset = ordinal_offset % 0x800; + + // Calculate the file & address of the block + size_t entry_address, entry_file; + BlockToOffset(block + block_offset, &entry_address, &entry_file); + entry_address += true_ordinal_offset; + + // Read directory entry + auto& file = files_.at(entry_file); + xe::filesystem::Seek(file, entry_address, SEEK_SET); + +#pragma pack(push, 1) + struct { + uint16_t node_l; + uint16_t node_r; + uint32_t data_block; + uint32_t length; + uint8_t attributes; + uint8_t name_length; + } dir_entry; + static_assert_size(dir_entry, 0xE); +#pragma pack(pop) + + if (fread(&dir_entry, sizeof(dir_entry), 1, file) != 1) { + XELOGE("ReadEntrySVOD failed to read directory entry at 0x{X}", + entry_address); + return Result::kReadError; + } + + auto name_buffer = std::make_unique(dir_entry.name_length); + if (fread(name_buffer.get(), 1, dir_entry.name_length, file) != + dir_entry.name_length) { + XELOGE("ReadEntrySVOD failed to read directory entry name at 0x{X}", + entry_address); + return Result::kReadError; + } + + auto name = std::string(name_buffer.get(), dir_entry.name_length); + + // Read the left node + if (dir_entry.node_l) { + auto node_result = ReadEntry(block, dir_entry.node_l, parent); + if (node_result != Result::kSuccess) { + return node_result; + } + } + + // Read file & address of block's data + size_t data_address, data_file; + BlockToOffset(dir_entry.data_block, &data_address, &data_file); + + // Create the entry + // NOTE: SVOD entries don't have timestamps for individual files, which can + // cause issues when decrypting games. Using the root entry's timestamp + // solves this issues. + auto entry = XContentContainerEntry::Create(this, parent, name, &files_); + if (dir_entry.attributes & kFileAttributeDirectory) { + // Entry is a directory + entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly; + entry->data_offset_ = 0; + entry->data_size_ = 0; + entry->block_ = block; + entry->access_timestamp_ = root_entry_->create_timestamp(); + entry->create_timestamp_ = root_entry_->create_timestamp(); + entry->write_timestamp_ = root_entry_->create_timestamp(); + + if (dir_entry.length) { + // If length is greater than 0, traverse the directory's children + auto directory_result = ReadEntry(dir_entry.data_block, 0, entry.get()); + if (directory_result != Result::kSuccess) { + return directory_result; + } + } + } else { + // Entry is a file + entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly; + entry->size_ = dir_entry.length; + entry->allocation_size_ = xe::round_up(dir_entry.length, kBlockSize); + entry->data_offset_ = data_address; + entry->data_size_ = dir_entry.length; + entry->block_ = dir_entry.data_block; + entry->access_timestamp_ = root_entry_->create_timestamp(); + entry->create_timestamp_ = root_entry_->create_timestamp(); + entry->write_timestamp_ = root_entry_->create_timestamp(); + + // Fill in all block records, sector by sector. + if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) { + uint32_t block_index = dir_entry.data_block; + size_t remaining_size = xe::round_up(dir_entry.length, 0x800); + + size_t last_record = -1; + size_t last_offset = -1; + while (remaining_size) { + const size_t BLOCK_SIZE = 0x800; + + size_t offset, file_index; + BlockToOffset(block_index, &offset, &file_index); + + block_index++; + remaining_size -= BLOCK_SIZE; + + if (offset - last_offset == BLOCK_SIZE) { + // Consecutive, so append to last entry. + entry->block_list_[last_record].length += BLOCK_SIZE; + last_offset = offset; + continue; + } + + entry->block_list_.push_back({file_index, offset, BLOCK_SIZE}); + last_record = entry->block_list_.size() - 1; + last_offset = offset; + } + } + } + + parent->children_.emplace_back(std::move(entry)); + + // Read the right node. + if (dir_entry.node_r) { + auto node_result = ReadEntry(block, dir_entry.node_r, parent); + if (node_result != Result::kSuccess) { + return node_result; + } + } + + return Result::kSuccess; +} + +XContentContainerDevice::Result SvodContainerDevice::SetLayout( + FILE* header, size_t& magic_offset) { + if (IsEDGFLayout()) { + return SetEDGFLayout(header, magic_offset); + } + + if (IsXSFLayout(header)) { + return SetXSFLayout(header, magic_offset); + } + + return SetNormalLayout(header, magic_offset); +} + +XContentContainerDevice::Result SvodContainerDevice::SetEDGFLayout( + FILE* header, size_t& magic_offset) { + uint8_t magic_buf[20]; + xe::filesystem::Seek(header, 0x2000, SEEK_SET); + if (fread(magic_buf, 1, countof(magic_buf), header) != countof(magic_buf)) { + XELOGE("ReadSVOD failed to read SVOD magic at 0x2000"); + return Result::kReadError; + } + + if (std::memcmp(magic_buf, MEDIA_MAGIC, countof(magic_buf)) != 0) { + XELOGE("SVOD uses an EGDF layout, but the magic block was not found."); + return Result::kFileMismatch; + } + + svod_base_offset_ = 0x0000; + magic_offset = 0x2000; + svod_layout_ = SvodLayoutType::kEnhancedGDF; + XELOGI("SVOD uses an EGDF layout. Magic block present at 0x2000."); + return Result::kSuccess; +} + +const bool SvodContainerDevice::IsXSFLayout(FILE* header) const { + uint8_t magic_buf[20]; + xe::filesystem::Seek(header, 0x12000, SEEK_SET); + + if (fread(magic_buf, 1, countof(magic_buf), header) != countof(magic_buf)) { + XELOGE("ReadSVOD failed to read SVOD magic at 0x12000"); + return false; + } + + return std::memcmp(magic_buf, MEDIA_MAGIC, countof(magic_buf)) == 0; +} + +XContentContainerDevice::Result SvodContainerDevice::SetXSFLayout( + FILE* header, size_t& magic_offset) { + uint8_t magic_buf[20]; + const char* XSF_MAGIC = "XSF"; + + xe::filesystem::Seek(header, 0x2000, SEEK_SET); + if (fread(magic_buf, 1, 3, header) != 3) { + XELOGE("ReadSVOD failed to read SVOD XSF magic at 0x2000"); + return Result::kReadError; + } + + svod_base_offset_ = 0x10000; + magic_offset = 0x12000; + + if (std::memcmp(magic_buf, XSF_MAGIC, 3) != 0) { + svod_layout_ = SvodLayoutType::kUnknown; + XELOGI("SVOD appears to use an XSF layout, but no header is present."); + XELOGI("SVOD magic block found at 0x12000"); + return Result::kSuccess; + } + + svod_layout_ = SvodLayoutType::kXSF; + XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000."); + XELOGI("Game was likely converted using a third-party tool."); + return Result::kSuccess; +} + +XContentContainerDevice::Result SvodContainerDevice::SetNormalLayout( + FILE* header, size_t& magic_offset) { + uint8_t magic_buf[20]; + + xe::filesystem::Seek(header, 0xD000, SEEK_SET); + if (fread(magic_buf, 1, countof(magic_buf), header) != countof(magic_buf)) { + XELOGE("ReadSVOD failed to read SVOD magic at 0xD000"); + return Result::kReadError; + } + + if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) != 0) { + XELOGE("Could not locate SVOD magic block."); + return Result::kReadError; + } + + // If the SVOD's magic block is at 0xD000, it most likely means that it + // is a single-file system. The STFS Header is 0xB000 bytes and the + // remaining 0x2000 is from hash tables. In most cases, these will be + // STFS, not SVOD. + svod_base_offset_ = 0xB000; + magic_offset = 0xD000; + + // Check for single file system + if (header_->content_metadata.data_file_count == 1) { + svod_layout_ = SvodLayoutType::kSingleFile; + XELOGI("SVOD is a single file. Magic block present at 0xD000."); + } else { + svod_layout_ = SvodLayoutType::kUnknown; + XELOGE( + "SVOD is not a single file, but the magic block was found at " + "0xD000."); + } + return Result::kSuccess; +} + +void SvodContainerDevice::BlockToOffset(size_t block, size_t* out_address, + size_t* out_file_index) const { + // SVOD Systems use hash blocks for integrity checks. These hash blocks + // cause blocks to be discontinuous in memory, and must be accounted for. + // - Each data block is 0x800 bytes in length + // - Every group of 0x198 data blocks is preceded a Level0 hash table. + // Level0 tables contain 0xCC hashes, each representing two data blocks. + // The total size of each Level0 hash table is 0x1000 bytes in length. + // - Every 0xA1C4 Level0 hash tables is preceded by a Level1 hash table. + // Level1 tables contain 0xCB hashes, each representing two Level0 hashes. + // The total size of each Level1 hash table is 0x1000 bytes in length. + // - Files are split into fragments of 0xA290000 bytes in length, + // consisting of 0x14388 data blocks, 0xCB Level0 hash tables, and 0x1 + // Level1 hash table. + + const size_t BLOCK_SIZE = 0x800; + const size_t HASH_BLOCK_SIZE = 0x1000; + const size_t BLOCKS_PER_L0_HASH = 0x198; + const size_t HASHES_PER_L1_HASH = 0xA1C4; + const size_t BLOCKS_PER_FILE = 0x14388; + const size_t MAX_FILE_SIZE = 0xA290000; + const size_t BLOCK_OFFSET = + header_->content_metadata.volume_descriptor.svod.start_data_block(); + + // Resolve the true block address and file index + size_t true_block = block - (BLOCK_OFFSET * 2); + if (svod_layout_ == SvodLayoutType::kEnhancedGDF) { + // EGDF has an 0x1000 byte offset, which is two blocks + true_block += 0x2; + } + + size_t file_block = true_block % BLOCKS_PER_FILE; + size_t file_index = true_block / BLOCKS_PER_FILE; + size_t offset = 0; + + // Calculate offset caused by Level0 Hash Tables + size_t level0_table_count = (file_block / BLOCKS_PER_L0_HASH) + 1; + offset += level0_table_count * HASH_BLOCK_SIZE; + + // Calculate offset caused by Level1 Hash Tables + size_t level1_table_count = (level0_table_count / HASHES_PER_L1_HASH) + 1; + offset += level1_table_count * HASH_BLOCK_SIZE; + + // For single-file SVOD layouts, include the size of the header in the offset. + if (svod_layout_ == SvodLayoutType::kSingleFile) { + offset += svod_base_offset_; + } + + size_t block_address = (file_block * BLOCK_SIZE) + offset; + + // If the offset causes the block address to overrun the file, round it. + if (block_address >= MAX_FILE_SIZE) { + file_index += 1; + block_address %= MAX_FILE_SIZE; + block_address += 0x2000; + } + + *out_address = block_address; + *out_file_index = file_index; +} + +} // namespace vfs +} // namespace xe diff --git a/src/xenia/vfs/devices/xcontent_devices/svod_container_device.h b/src/xenia/vfs/devices/xcontent_devices/svod_container_device.h new file mode 100644 index 0000000000..2c389f6a9e --- /dev/null +++ b/src/xenia/vfs/devices/xcontent_devices/svod_container_device.h @@ -0,0 +1,77 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2023 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_VFS_DEVICES_XCONTENT_DEVICES_SVOD_CONTAINER_DEVICE_H_ +#define XENIA_VFS_DEVICES_XCONTENT_DEVICES_SVOD_CONTAINER_DEVICE_H_ + +#include +#include +#include +#include + +#include "xenia/base/string_util.h" +#include "xenia/kernel/util/xex2_info.h" +#include "xenia/vfs/device.h" +#include "xenia/vfs/devices/stfs_xbox.h" +#include "xenia/vfs/devices/xcontent_container_device.h" +#include "xenia/vfs/devices/xcontent_container_entry.h" + +namespace xe { +namespace vfs { +class SvodContainerDevice : public XContentContainerDevice { + public: + SvodContainerDevice(const std::string_view mount_path, + const std::filesystem::path& host_path); + ~SvodContainerDevice() override; + + bool is_read_only() const override { return true; } + + uint32_t component_name_max_length() const override { return 255; } + + uint32_t total_allocation_units() const override { + return uint32_t(data_size() / sectors_per_allocation_unit() / + bytes_per_sector()); + } + uint32_t available_allocation_units() const override { return 0; } + + private: + enum class SvodLayoutType { + kUnknown = 0x0, + kEnhancedGDF = 0x1, + kXSF = 0x2, + kSingleFile = 0x4, + }; + const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA"; + + Result LoadHostFiles(FILE* header_file) override; + + Result Read() override; + Result ReadEntry(uint32_t sector, uint32_t ordinal, + XContentContainerEntry* parent); + void BlockToOffset(size_t sector, size_t* address, size_t* file_index) const; + + Result SetLayout(FILE* header, size_t& magic_offset); + Result SetEDGFLayout(FILE* header, size_t& magic_offset); + Result SetXSFLayout(FILE* header, size_t& magic_offset); + Result SetNormalLayout(FILE* header, size_t& magic_offset); + + const bool IsEDGFLayout() const { + return header_->content_metadata.volume_descriptor.svod.features.bits + .enhanced_gdf_layout; + } + const bool IsXSFLayout(FILE* header) const; + + size_t svod_base_offset_; + SvodLayoutType svod_layout_; +}; + +} // namespace vfs +} // namespace xe + +#endif // XENIA_VFS_DEVICES_XCONTENT_DEVICES_SVOD_CONTAINER_DEVICE_H_ diff --git a/src/xenia/vfs/vfs_dump.cc b/src/xenia/vfs/vfs_dump.cc index f2416b59a6..6551f1598e 100644 --- a/src/xenia/vfs/vfs_dump.cc +++ b/src/xenia/vfs/vfs_dump.cc @@ -17,7 +17,7 @@ #include "xenia/base/logging.h" #include "xenia/base/math.h" -#include "xenia/vfs/devices/stfs_container_device.h" +#include "xenia/vfs/devices/xcontent_container_device.h" #include "xenia/vfs/file.h" namespace xe { @@ -38,10 +38,9 @@ int vfs_dump_main(const std::vector& args) { } std::filesystem::path base_path = cvars::dump_path; - std::unique_ptr device; + std::unique_ptr device = + vfs::XContentContainerDevice::CreateContentDevice("", cvars::source); - // TODO: Flags specifying the type of device. - device = std::make_unique("", cvars::source); if (!device->Initialize()) { XELOGE("Failed to initialize device"); return 1;