Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Package installation fixes #13205

Merged
merged 4 commits into from Jan 11, 2023
Merged

Package installation fixes #13205

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a8887a4
package_reader: fix some warnings
Megamouse Jan 10, 2023
bfc85b1
Qt: fix some package install cancellation issues
Megamouse Jan 10, 2023
f20e5eb
Qt: more package install fixes
Megamouse Jan 11, 2023
a291869
Qt: Allow users to create shortcuts for existing bootable entries eve…
Megamouse Jan 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
196 changes: 144 additions & 52 deletions rpcs3/Crypto/unpkg.cpp
View file
Open in desktop
Expand Up @@ -515,7 +515,7 @@ bool package_reader::read_param_sfo()

std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry));

for (const auto& entry : entries)
for (const PKGEntry& entry : entries)
{
if (entry.name_size > 256)
{
Expand Down Expand Up @@ -567,11 +567,9 @@ bool package_reader::read_param_sfo()

return true;
}
else
{
pkg_log.error("Failed to create temporary PARAM.SFO file");
return false;
}

pkg_log.error("Failed to create temporary PARAM.SFO file");
return false;
}

return false;
Expand Down Expand Up @@ -621,7 +619,7 @@ package_error package_reader::check_target_app_version() const
if (!target_app_ver.empty())
{
// We are unable to compare anything with the target app version
pkg_log.error("A target app version is required (%s), but no PARAM.SFO was found for %s", target_app_ver, title_id);
pkg_log.error("A target app version is required (%s), but no PARAM.SFO was found for %s. (path='%s', error=%s)", target_app_ver, title_id, sfo_path, fs::g_tls_error);
return package_error::app_version;
}

Expand Down Expand Up @@ -691,20 +689,16 @@ package_error package_reader::check_target_app_version() const
return package_error::app_version;
}

bool package_reader::fill_data(std::map<std::string, install_entry*>& all_install_entries)
bool package_reader::set_install_path()
{
if (!m_is_valid)
{
return false;
}

m_install_entries.clear();
m_install_path.clear();
m_bootable_file_path.clear();
m_entry_indexer = 0;
m_written_bytes = 0;

// Get full path and create the directory
// Get full path
std::string dir = rpcs3::utils::get_hdd0_dir();

// Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType
Expand Down Expand Up @@ -741,13 +735,27 @@ bool package_reader::fill_data(std::map<std::string, install_entry*>& all_instal
// If false, an existing directory is being overwritten: cannot cancel the operation
m_was_null = !fs::is_dir(dir);

if (!fs::create_path(dir))
m_install_path = dir;
return true;
}

bool package_reader::fill_data(std::map<std::string, install_entry*>& all_install_entries)
{
if (!m_is_valid)
{
pkg_log.error("Could not create the installation directory %s", dir);
return false;
}

m_install_path = dir;
if (!fs::create_path(m_install_path))
{
pkg_log.error("Could not create the installation directory %s (error=%s)", m_install_path, fs::g_tls_error);
return false;
}

m_install_entries.clear();
m_bootable_file_path.clear();
m_entry_indexer = 0;
m_written_bytes = 0;

if (!decrypt_data())
{
Expand All @@ -774,13 +782,15 @@ bool package_reader::fill_data(std::map<std::string, install_entry*>& all_instal
decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data());

const std::string name{reinterpret_cast<char*>(m_bufs.back().get()), entry.name_size};
std::string path = dir + vfs::escape(name);
std::string path = m_install_path + vfs::escape(name);

const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);

(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);

switch (const u8 entry_type = entry.type & 0xff)
const u8 entry_type = entry.type & 0xff;

switch (entry_type)
{
case PKG_FILE_ENTRY_FOLDER:
case 0x12:
Expand All @@ -807,7 +817,14 @@ bool package_reader::fill_data(std::map<std::string, install_entry*>& all_instal
const std::string true_path = std::filesystem::weakly_canonical(std::filesystem::u8path(path)).string();
auto map_ptr = &*all_install_entries.try_emplace(true_path).first;

m_install_entries.push_back({ map_ptr, name, entry.file_offset, entry.file_size, entry.type, entry.pad });
m_install_entries.push_back({
.weak_reference = map_ptr,
.name = name,
.file_offset = entry.file_offset,
.file_size = entry.file_size,
.type = entry.type,
.pad = entry.pad
});

if (map_ptr->second && !(entry.type & PKG_FILE_ENTRY_OVERWRITE))
{
Expand All @@ -824,7 +841,7 @@ bool package_reader::fill_data(std::map<std::string, install_entry*>& all_instal

if (num_failures != 0)
{
pkg_log.error("Package installation failed: %s", dir);
pkg_log.error("Package installation failed: %s", m_install_path);
return false;
}

Expand All @@ -833,20 +850,28 @@ bool package_reader::fill_data(std::map<std::string, install_entry*>& all_instal

fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false);

usz package_reader::extract_worker(thread_key thread_data_key, std::map<std::string, install_entry*>& all_install_entries)
void package_reader::extract_worker(thread_key thread_data_key)
{
usz num_failures = 0;

while (true)
while (m_num_failures == 0 && !m_aborted)
{
const usz maybe_index = m_entry_indexer++;
// Make sure m_entry_indexer does not exceed m_install_entries
const usz index = m_entry_indexer.fetch_op([this](usz& v)
{
if (v < m_install_entries.size())
{
v++;
return true;
}

return false;
}).first;

if (maybe_index >= m_install_entries.size())
if (index >= m_install_entries.size())
{
break;
}

const install_entry& entry = ::at32(m_install_entries, maybe_index);
const install_entry& entry = ::at32(m_install_entries, index);

if (!entry.is_dominating())
{
Expand Down Expand Up @@ -927,7 +952,7 @@ usz package_reader::extract_worker(thread_key thread_data_key, std::map<std::str
out = DecryptEDAT(out, name, 1, reinterpret_cast<u8*>(&m_header.klicensee), true);
if (!out || !fs::write_file(path, fs::rewrite, static_cast<fs::container_stream<std::vector<u8>>*>(out.release().get())->obj))
{
num_failures++;
m_num_failures++;
pkg_log.error("Failed to create file %s", path);
break;
}
Expand All @@ -952,71 +977,121 @@ usz package_reader::extract_worker(thread_key thread_data_key, std::map<std::str
}
else
{
num_failures++;
m_num_failures++;
}
}
else
{
num_failures++;
m_num_failures++;
pkg_log.error("Failed to create file %s", path);
}

break;
}
default:
{
num_failures++;
m_num_failures++;
pkg_log.error("Unknown PKG entry type (0x%x) %s", entry.type, name);
break;
}
}
}

return num_failures;
}

bool package_reader::extract_data(std::deque<package_reader>& readers, std::deque<std::string>& bootable_paths)
package_error package_reader::extract_data(std::deque<package_reader>& readers, std::deque<std::string>& bootable_paths)
{
std::map<std::string, install_entry*> all_install_entries;
package_error error = package_error::no_error;
usz num_failures = 0;

for (auto& reader : readers)
// Set paths first in order to know if the install dir was empty before starting any installations.
// This will also allow us to remove all the new packages in one path at once if any of them fail.
for (package_reader& reader : readers)
{
if (!reader.fill_data(all_install_entries))
reader.m_result = result::not_started;

if (!reader.set_install_path())
{
return false;
error = package_error::other;
reader.m_result = result::error;
break;
}
}

usz num_failures = 0;

for (auto& reader : readers)
for (package_reader& reader : readers)
{
// Use a seperate map for each reader. We need to check if the target app version exists for each package in sequence.
std::map<std::string, install_entry*> all_install_entries;

if (error == package_error::no_error)
{
// Check if this package is allowed to be installed on top of the existing data
error = reader.check_target_app_version();
}

if (error == package_error::no_error)
{
reader.m_result = result::started;

// Parse the files to be installed and create all paths.
if (!reader.fill_data(all_install_entries))
{
error = package_error::other;
}
}

reader.m_bufs.resize(std::min<usz>(utils::get_thread_count(), reader.m_install_entries.size()));
reader.m_num_failures = error == package_error::no_error ? 0 : 1;

atomic_t<usz> thread_indexer = 0;

named_thread_group workers("PKG Installer "sv, std::max<usz>(reader.m_bufs.size(), 1) - 1, [&]()
named_thread_group workers("PKG Installer "sv, std::max<u32>(::narrow<u32>(reader.m_bufs.size()), 1) - 1, [&]()
{
num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries);
reader.extract_worker(thread_key{thread_indexer++});
});

num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries);
reader.extract_worker(thread_key{thread_indexer++});
workers.join();

num_failures += reader.m_num_failures;

reader.m_bufs.clear();
reader.m_bufs.shrink_to_fit();

if (num_failures)
// We don't count this package as aborted if all entries were processed.
if (reader.m_num_failures || (reader.m_aborted && reader.m_entry_indexer < reader.m_install_entries.size()))
{
if (reader.m_was_null)
// Clear boot path. We don't want to propagate potentially broken paths to the caller.
reader.m_bootable_file_path.clear();

bool cleaned = reader.m_was_null;

if (reader.m_was_null && fs::is_dir(reader.m_install_path))
{
fs::remove_all(reader.m_install_path, true);
pkg_log.notice("Removing partial installation ('%s')", reader.m_install_path);

if (!fs::remove_all(reader.m_install_path, true))
{
pkg_log.notice("Failed to remove partial installation ('%s') (error=%s)", reader.m_install_path, fs::g_tls_error);
cleaned = false;
}
}

pkg_log.success("Package failed to install ('%s')", reader.m_install_path);
if (reader.m_num_failures)
{
pkg_log.error("Package failed to install ('%s')", reader.m_install_path);
reader.m_result = cleaned ? result::error_cleaned : result::error;
}
else
{
pkg_log.warning("Package installation aborted ('%s')", reader.m_install_path);
reader.m_result = cleaned ? result::aborted_cleaned : result::aborted;
}
break;
}
else if (reader.get_progress(1) != 1)

reader.m_result = result::success;

if (reader.get_progress(1) != 1)
{
pkg_log.warning("Missing %d bytes from PKG total files size.", reader.m_header.data_size - reader.m_written_bytes);
reader.m_written_bytes = reader.m_header.data_size; // Mark as completed anyway
Expand All @@ -1026,13 +1101,18 @@ bool package_reader::extract_data(std::deque<package_reader>& readers, std::dequ
bootable_paths.emplace_back(std::move(reader.m_bootable_file_path));
}

return num_failures == 0;
if (num_failures > 0)
{
error = package_error::other;
}

return error;
}

void package_reader::archive_seek(const s64 new_offset, const fs::seek_mode damode)
{
if (m_file) m_file.seek(new_offset, damode);
};
}

u64 package_reader::archive_read(void* data_ptr, const u64 num_bytes)
{
Expand Down Expand Up @@ -1128,4 +1208,16 @@ std::span<const char> package_reader::decrypt(u64 offset, u64 size, const uchar*

// Return the amount of data written in buf
return data_span;
};
}

int package_reader::get_progress(int maximum) const
{
const usz wr = m_written_bytes;

return wr >= m_header.data_size ? maximum : ::narrow<int>(wr * maximum / m_header.data_size);
}

void package_reader::abort_extract()
{
m_aborted = true;
}