Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rsx: Shaders cache fixes and optimizations #9530

Merged
merged 3 commits into from Jan 2, 2021
Merged

rsx: Shaders cache fixes and optimizations #9530

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f56ec3e
rsx: Shaders cache loading and saving bugfixes
elad335 Jan 2, 2021
f49dee9
rsx: Fix shader cache of 2 or less pipelines
elad335 Jan 2, 2021
e6f1bb3
rsx: Make FP shader cache load lock-free
elad335 Jan 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 12 additions & 3 deletions Utilities/File.h
View file
Open in desktop
Expand Up @@ -733,9 +733,18 @@ namespace fs
// Always use write flag, remove read flag
if (fs::file f{path, mode + fs::write - fs::read})
{
// Write args sequentially
(f.write(args), ...);
return true;
if constexpr (sizeof...(args) == 2u && (std::is_pointer_v<Args> || ...))
{
// Specialization for [const void*, usz] args
f.write(args...);
return true;
}
else
{
// Write args sequentially
(f.write(args), ...);
return true;
}
}

return false;
Expand Down
100 changes: 58 additions & 42 deletions rpcs3/Emu/RSX/rsx_cache.h
View file
Open in desktop
Expand Up @@ -423,8 +423,7 @@ namespace rsx
std::string version_prefix;
std::string root_path;
std::string pipeline_class_name;
std::mutex fpd_mutex;
std::unordered_map<u64, std::vector<u8>> fragment_program_data;
lf_fifo<std::unique_ptr<u8[]>, 100> fragment_program_data;
kd-11 marked this conversation as resolved.
Show resolved Hide resolved

backend_storage& m_storage;

Expand All @@ -447,20 +446,34 @@ namespace rsx
fs::dir_entry tmp = entries[pos];

const auto filename = directory_path + "/" + tmp.name;
std::vector<u8> bytes;
fs::file f(filename);

if (!f)
{
// Unexpected error, but avoid crash
continue;
}

if (f.size() != sizeof(pipeline_data))
{
rsx_log.error("Removing cached pipeline object %s since it's not binary compatible with the current shader cache", tmp.name.c_str());
fs::remove_file(filename);
continue;
}
f.read<u8>(bytes, f.size());

auto entry = unpack(*reinterpret_cast<pipeline_data*>(bytes.data()));
pipeline_data pdata{};
f.read(&pdata, f.size());

auto entry = unpack(pdata);

if (std::get<1>(entry).data.empty() || !std::get<2>(entry).ucode_length)
{
continue;
}

m_storage.preload_programs(std::get<1>(entry), std::get<2>(entry));

unpacked[unpacked.push_begin()] = entry;
unpacked[unpacked.push_begin()] = std::move(entry);
}
// Do not account for an extra shader that was never processed
processed--;
Expand Down Expand Up @@ -573,29 +586,28 @@ namespace rsx

std::string directory_path = root_path + "/pipelines/" + pipeline_class_name + "/" + version_prefix;

if (!fs::is_dir(directory_path))
fs::dir root = fs::dir(directory_path);

if (!root)
{
fs::create_path(directory_path);
fs::create_path(root_path + "/raw");

return;
}

fs::dir root = fs::dir(directory_path);

u32 entry_count = 0;
std::vector<fs::dir_entry> entries;
for (auto It = root.begin(); It != root.end(); ++It, entry_count++)
{
fs::dir_entry tmp = *It;

if (tmp.name == "." || tmp.name == "..")
for (auto&& tmp : root)
{
if (tmp.is_directory)
continue;

entries.push_back(tmp);
}

if ((entry_count = ::size32(entries)) <= 2)
u32 entry_count = ::size32(entries);

if (!entry_count)
return;

root.rewind();
Expand Down Expand Up @@ -643,17 +655,20 @@ namespace rsx
}

pipeline_data data = pack(pipeline, vp, fp);

std::string fp_name = root_path + "/raw/" + fmt::format("%llX.fp", data.fragment_program_hash);
std::string vp_name = root_path + "/raw/" + fmt::format("%llX.vp", data.vertex_program_hash);

if (!fs::is_file(fp_name))
// Writeback to cache either if file does not exist or it is invalid (unexpected size)
// Note: fs::write_file is not atomic, if the process is terminated in the middle an empty file is created
if (fs::stat_t s{}; !fs::stat(fp_name, s) || s.size != fp.ucode_length)
{
fs::file(fp_name, fs::rewrite).write(fp.get_data(), fp.ucode_length);
fs::write_file(fp_name, fs::rewrite, fp.get_data(), fp.ucode_length);
}

if (!fs::is_file(vp_name))
if (fs::stat_t s{}; !fs::stat(vp_name, s) || s.size != vp.data.size() * sizeof(u32))
{
fs::file(vp_name, fs::rewrite).write<u32>(vp.data);
fs::write_file(vp_name, fs::rewrite, vp.data);
}

u64 state_hash = 0;
Expand All @@ -669,50 +684,51 @@ namespace rsx
state_hash ^= rpcs3::hash_base<u16>(data.fp_shadow_textures);
state_hash ^= rpcs3::hash_base<u16>(data.fp_redirected_textures);

std::string pipeline_file_name = fmt::format("%llX+%llX+%llX+%llX.bin", data.vertex_program_hash, data.fragment_program_hash, data.pipeline_storage_hash, state_hash);
std::string pipeline_path = root_path + "/pipelines/" + pipeline_class_name + "/" + version_prefix + "/" + pipeline_file_name;
fs::file(pipeline_path, fs::rewrite).write(&data, sizeof(pipeline_data));
const std::string pipeline_file_name = fmt::format("%llX+%llX+%llX+%llX.bin", data.vertex_program_hash, data.fragment_program_hash, data.pipeline_storage_hash, state_hash);
const std::string pipeline_path = root_path + "/pipelines/" + pipeline_class_name + "/" + version_prefix + "/" + pipeline_file_name;
fs::write_file(pipeline_path, fs::rewrite, &data, sizeof(data));
}

RSXVertexProgram load_vp_raw(u64 program_hash)
{
std::vector<u32> data;
std::string filename = fmt::format("%llX.vp", program_hash);
RSXVertexProgram vp = {};

fs::file f(root_path + "/raw/" + filename);
f.read<u32>(data, f.size() / sizeof(u32));
fs::file f(fmt::format("%s/raw/%llX.vp", root_path, program_hash));
if (f) f.read(vp.data, f.size() / sizeof(u32));

RSXVertexProgram vp = {};
vp.data = data;
vp.skip_vertex_input_check = true;

return vp;
}

RSXFragmentProgram load_fp_raw(u64 program_hash)
{
std::vector<u8> data;
std::string filename = fmt::format("%llX.fp", program_hash);

fs::file f(root_path + "/raw/" + filename);
f.read<u8>(data, f.size());
fs::file f(fmt::format("%s/raw/%llX.fp", root_path, program_hash));

RSXFragmentProgram fp = {};

elad335 marked this conversation as resolved.
Show resolved Hide resolved
const u32 size = fp.ucode_length = f ? ::size32(f) : 0;

if (!size)
{
std::lock_guard<std::mutex> lock(fpd_mutex);
fragment_program_data[program_hash] = data;
fp.data = fragment_program_data[program_hash].data();
return fp;
}
fp.ucode_length = ::size32(data);

auto buf = std::make_unique<u8[]>(size);
fp.data = buf.get();
f.read(buf.get(), size);
fragment_program_data[fragment_program_data.push_begin()] = std::move(buf);
return fp;
}

std::tuple<pipeline_storage_type, RSXVertexProgram, RSXFragmentProgram> unpack(pipeline_data &data)
{
RSXVertexProgram vp = load_vp_raw(data.vertex_program_hash);
RSXFragmentProgram fp = load_fp_raw(data.fragment_program_hash);
pipeline_storage_type pipeline = data.pipeline_properties;
std::tuple<pipeline_storage_type, RSXVertexProgram, RSXFragmentProgram> result;
kd-11 marked this conversation as resolved.
Show resolved Hide resolved
auto& [pipeline, vp, fp] = result;

vp = load_vp_raw(data.vertex_program_hash);
fp = load_fp_raw(data.fragment_program_hash);
pipeline = data.pipeline_properties;

vp.output_mask = data.vp_ctrl;
vp.texture_dimensions = data.vp_texture_dimensions;
Expand Down Expand Up @@ -741,7 +757,7 @@ namespace rsx
fp.shadow_textures = data.fp_shadow_textures;
fp.redirected_textures = data.fp_redirected_textures;

return std::make_tuple(pipeline, vp, fp);
return result;
}

pipeline_data pack(const pipeline_storage_type &pipeline, const RSXVertexProgram &vp, const RSXFragmentProgram &fp)
Expand Down