diff --git a/.gitmodules b/.gitmodules index cb00d4f21ed6..d6f27ad29f29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -23,10 +23,6 @@ path = 3rdparty/SPIRV/SPIRV-Headers url = ../../KhronosGroup/SPIRV-Headers.git ignore = dirty -[submodule "3rdparty/cereal"] - path = 3rdparty/cereal - url = ../../RPCS3/cereal.git - ignore = dirty [submodule "3rdparty/zlib"] path = 3rdparty/zlib url = ../../madler/zlib diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt index fdd847289374..047d09aaea50 100644 --- a/3rdparty/CMakeLists.txt +++ b/3rdparty/CMakeLists.txt @@ -117,12 +117,6 @@ else() target_include_directories(xxhash INTERFACE xxHash) endif() - -# cereal -add_library(3rdparty_cereal INTERFACE) -target_include_directories(3rdparty_cereal INTERFACE cereal/include) - - # OpenGL # Prefer GLVND for OpenGL rather than legacy, unless it's been defined elsewhere, in the case of AppImage builds @@ -365,7 +359,6 @@ add_library(3rdparty::yaml-cpp ALIAS yaml-cpp) add_library(3rdparty::xxhash ALIAS xxhash) add_library(3rdparty::hidapi ALIAS 3rdparty_hidapi) add_library(3rdparty::libpng ALIAS ${LIBPNG_TARGET}) -add_library(3rdparty::cereal ALIAS 3rdparty_cereal) add_library(3rdparty::opengl ALIAS 3rdparty_opengl) add_library(3rdparty::stblib ALIAS 3rdparty_stblib) add_library(3rdparty::discordRPC ALIAS 3rdparty_discordRPC) diff --git a/3rdparty/cereal b/3rdparty/cereal deleted file mode 160000 index 60c69df968d1..000000000000 --- a/3rdparty/cereal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 60c69df968d1c72c998cd5f23ba34e2e3718a84b diff --git a/buildfiles/msvc/rpcs3_default.props b/buildfiles/msvc/rpcs3_default.props index 90a88c1b5583..2ad27957d524 100644 --- a/buildfiles/msvc/rpcs3_default.props +++ b/buildfiles/msvc/rpcs3_default.props @@ -3,7 +3,7 @@ - .\;..\;..\3rdparty\asmjit\asmjit\src;..\3rdparty\yaml-cpp\include;..\3rdparty\ffmpeg\include;..\3rdparty\cereal\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);$(UniversalCRT_IncludePath);..\3rdparty\libpng\libpng;..\3rdparty\GL;..\3rdparty\stblib\include;..\3rdparty\OpenAL\include;..\3rdparty\pugixml\src;..\3rdparty\hidapi\hidapi;..\3rdparty\Optional;..\3rdparty\xxhash + .\;..\;..\3rdparty\asmjit\asmjit\src;..\3rdparty\yaml-cpp\include;..\3rdparty\ffmpeg\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);$(UniversalCRT_IncludePath);..\3rdparty\libpng\libpng;..\3rdparty\GL;..\3rdparty\stblib\include;..\3rdparty\OpenAL\include;..\3rdparty\pugixml\src;..\3rdparty\hidapi\hidapi;..\3rdparty\Optional;..\3rdparty\xxhash $(SolutionDir)lib\$(Configuration)-$(Platform)\ $(SolutionDir)lib\$(Configuration)-$(Platform)\;$(UniversalCRT_LibraryPath_x64);$(LibraryPath) $(SolutionDir)tmp\$(ProjectName)-$(Configuration)-$(Platform)\ diff --git a/rpcs3/Emu/CMakeLists.txt b/rpcs3/Emu/CMakeLists.txt index 5ff512f455f1..139f403c4898 100644 --- a/rpcs3/Emu/CMakeLists.txt +++ b/rpcs3/Emu/CMakeLists.txt @@ -38,7 +38,6 @@ target_sources(rpcs3_emu PRIVATE ../util/atomic.cpp ../util/logs.cpp ../util/yaml.cpp - ../util/cereal.cpp ../util/vm_native.cpp ../util/dyn_lib.cpp ../util/sysinfo.cpp @@ -76,12 +75,6 @@ else() set_source_files_properties("../util/yaml.cpp" PROPERTIES COMPILE_FLAGS -fexceptions) endif() -if(MSVC) - set_source_files_properties("../util/cereal.cpp" PROPERTIES COMPILE_FLAGS /GR) -else() - set_source_files_properties("../util/cereal.cpp" PROPERTIES COMPILE_FLAGS -frtti) -endif() - # Crypto target_sources(rpcs3_emu PRIVATE ../Crypto/aes.cpp @@ -499,7 +492,7 @@ endif() target_link_libraries(rpcs3_emu PUBLIC - 3rdparty::ffmpeg 3rdparty::cereal + 3rdparty::ffmpeg 3rdparty::opengl 3rdparty::stblib 3rdparty::vulkan 3rdparty::glew 3rdparty::libusb 3rdparty::wolfssl diff --git a/rpcs3/Emu/RSX/Capture/rsx_replay.h b/rpcs3/Emu/RSX/Capture/rsx_replay.h index 6e0a9d14aafe..6a53d500bd2f 100644 --- a/rpcs3/Emu/RSX/Capture/rsx_replay.h +++ b/rpcs3/Emu/RSX/Capture/rsx_replay.h @@ -8,8 +8,12 @@ namespace rsx { - constexpr u32 FRAME_CAPTURE_MAGIC = 0x52524300; // ascii 'RRC/0' - constexpr u32 FRAME_CAPTURE_VERSION = 0x4; + enum : u32 + { + FRAME_CAPTURE_MAGIC = 0x52524300, // ascii 'RRC/0' + FRAME_CAPTURE_VERSION = 0x5, + }; + struct frame_capture_data { struct memory_block_data @@ -138,8 +142,9 @@ namespace rsx } }; - u32 magic; - u32 version; + u32 magic = FRAME_CAPTURE_MAGIC; + u32 version = FRAME_CAPTURE_VERSION; + // hashmap of holding various states for tile std::unordered_map tile_map; // hashmap of various memory 'changes' that can be applied to ps3 memory @@ -157,7 +162,27 @@ namespace rsx void serialize(Archive & ar) { ar(magic); + + // Check if deserializing + if constexpr (std::is_same_v, bool>) + { + if (magic != FRAME_CAPTURE_MAGIC) + { + // Failure + return; + } + } + ar(version); + + if constexpr (std::is_same_v, bool>) + { + if (version != FRAME_CAPTURE_VERSION) + { + return; + } + } + ar(tile_map); ar(memory_map); ar(memory_data_map); diff --git a/rpcs3/Emu/RSX/RSXThread.cpp b/rpcs3/Emu/RSX/RSXThread.cpp index c0ad5a04c535..e31400348548 100644 --- a/rpcs3/Emu/RSX/RSXThread.cpp +++ b/rpcs3/Emu/RSX/RSXThread.cpp @@ -19,7 +19,7 @@ #include "Utilities/date_time.h" #include "Utilities/StrUtil.h" -#include "util/cereal.hpp" +#include "util/serialization.hpp" #include "util/asm.hpp" #include @@ -2861,11 +2861,14 @@ namespace rsx const std::string file_path = fs::get_config_dir() + "captures/" + Emu.GetTitleID() + "_" + date_time::current_time_narrow() + "_capture.rrc"; // todo: may want to compress this data? - const std::string file_data = cereal_serialize(frame_capture); + utils::serial save_manager; + save_manager.reserve(0x800'0000); // 128MB + save_manager(frame_capture); + fs::pending_file temp(file_path); - if (temp.file && (temp.file.write(file_data), temp.commit(false))) + if (temp.file && (temp.file.write(save_manager.data), temp.commit(false))) { rsx_log.success("Capture successful: %s", file_path); } diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index 963ff0508eca..cd1ed448a6a7 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -30,7 +30,7 @@ #include "../Crypto/unself.h" #include "util/yaml.hpp" #include "util/logs.hpp" -#include "util/cereal.hpp" +#include "util/serialization.hpp" #include #include @@ -380,7 +380,8 @@ bool Emulator::BootRsxCapture(const std::string& path) } std::unique_ptr frame = std::make_unique(); - cereal_deserialize(*frame, in_file.to_string()); + utils::deserial load_manager{ in_file.to_vector() }; + load_manager(*frame); in_file.close(); if (frame->magic != rsx::FRAME_CAPTURE_MAGIC) @@ -391,7 +392,7 @@ bool Emulator::BootRsxCapture(const std::string& path) if (frame->version != rsx::FRAME_CAPTURE_VERSION) { - sys_log.error("Rsx capture file version not supported! Expected %d, found %d", rsx::FRAME_CAPTURE_VERSION, frame->version); + sys_log.error("Rsx capture file version not supported! Expected %d, found %d", +rsx::FRAME_CAPTURE_VERSION, frame->version); return false; } diff --git a/rpcs3/emucore.vcxproj b/rpcs3/emucore.vcxproj index b24d4b0c4405..f5f4f1e96e0c 100644 --- a/rpcs3/emucore.vcxproj +++ b/rpcs3/emucore.vcxproj @@ -100,10 +100,6 @@ NotUsing Sync - - NotUsing - true - NotUsing @@ -490,6 +486,7 @@ + diff --git a/rpcs3/emucore.vcxproj.filters b/rpcs3/emucore.vcxproj.filters index 8c76ee1cecd7..2c6076d531e6 100644 --- a/rpcs3/emucore.vcxproj.filters +++ b/rpcs3/emucore.vcxproj.filters @@ -843,9 +843,6 @@ Utilities - - Utilities - Utilities @@ -1962,6 +1959,9 @@ Emu\GPU\RSX\Overlays + + Utilities + diff --git a/rpcs3/rpcs3.vcxproj b/rpcs3/rpcs3.vcxproj index 8da431c93152..0f43c9cab250 100644 --- a/rpcs3/rpcs3.vcxproj +++ b/rpcs3/rpcs3.vcxproj @@ -62,12 +62,12 @@ true - ..\3rdparty\7z\src;..\3rdparty\hidapi\hidapi\hidapi;.\;..\;..\3rdparty\asmjit\asmjit\src;..\3rdparty\yaml-cpp\include;..\3rdparty\ffmpeg\include;..\3rdparty\cereal\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);$(UniversalCRT_IncludePath);..\3rdparty\XAudio2Redist\include;..\3rdparty\libpng\libpng;..\3rdparty\GL;..\3rdparty\stblib\include;..\3rdparty\OpenAL\include;..\3rdparty\pugixml\src;..\3rdparty\Optional;..\3rdparty\discord-rpc\include;..\3rdparty\zlib + ..\3rdparty\7z\src;..\3rdparty\hidapi\hidapi\hidapi;.\;..\;..\3rdparty\asmjit\asmjit\src;..\3rdparty\yaml-cpp\include;..\3rdparty\ffmpeg\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);$(UniversalCRT_IncludePath);..\3rdparty\XAudio2Redist\include;..\3rdparty\libpng\libpng;..\3rdparty\GL;..\3rdparty\stblib\include;..\3rdparty\OpenAL\include;..\3rdparty\pugixml\src;..\3rdparty\Optional;..\3rdparty\discord-rpc\include;..\3rdparty\zlib $(SolutionDir)lib\$(Configuration)-$(Platform)\;$(UniversalCRT_LibraryPath_x64);$(LibraryPath) $(SolutionDir)lib\$(Configuration)-$(Platform)\;$(UniversalCRT_LibraryPath_x64);$(LibraryPath) - ..\3rdparty\7z\src;..\3rdparty\hidapi\hidapi\hidapi;.\;..\;..\3rdparty\asmjit\asmjit\src;..\3rdparty\yaml-cpp\include;..\3rdparty\ffmpeg\include;..\3rdparty\cereal\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);$(UniversalCRT_IncludePath);..\3rdparty\XAudio2Redist\include;..\3rdparty\libpng\libpng;..\3rdparty\GL;..\3rdparty\stblib\include;..\3rdparty\OpenAL\include;..\3rdparty\pugixml\src;..\3rdparty\Optional;..\3rdparty\discord-rpc\include;..\3rdparty\zlib + ..\3rdparty\7z\src;..\3rdparty\hidapi\hidapi\hidapi;.\;..\;..\3rdparty\asmjit\asmjit\src;..\3rdparty\yaml-cpp\include;..\3rdparty\ffmpeg\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);$(UniversalCRT_IncludePath);..\3rdparty\XAudio2Redist\include;..\3rdparty\libpng\libpng;..\3rdparty\GL;..\3rdparty\stblib\include;..\3rdparty\OpenAL\include;..\3rdparty\pugixml\src;..\3rdparty\Optional;..\3rdparty\discord-rpc\include;..\3rdparty\zlib diff --git a/rpcs3/util/cereal.cpp b/rpcs3/util/cereal.cpp deleted file mode 100644 index 221f8db4f9d1..000000000000 --- a/rpcs3/util/cereal.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "util/cereal.hpp" -#include -#include "Utilities/StrFmt.h" -#include "Emu/RSX/RSXThread.h" -#include "Emu/RSX/Capture/rsx_capture.h" - -#ifndef _MSC_VER -#pragma GCC diagnostic ignored "-Weffc++" -#endif - -#include "cereal/archives/binary.hpp" -#include -#include -#include -#include -#include -#include - -#include - -namespace cereal -{ - [[noreturn]] void throw_exception(const std::string& err) - { - fmt::throw_exception("%s", err); - } - - [[noreturn]] void throw_exception(const char* err) - { - fmt::throw_exception("%s", err); - } -} - -template <> -std::string cereal_serialize(const rsx::frame_capture_data& data) -{ - std::ostringstream os; - cereal::BinaryOutputArchive archive(os); - archive(data); - return os.str(); -} - -template <> -void cereal_deserialize(rsx::frame_capture_data& data, const std::string& src) -{ - std::istringstream is(src); - cereal::BinaryInputArchive archive(is); - archive(data); -} diff --git a/rpcs3/util/cereal.hpp b/rpcs3/util/cereal.hpp deleted file mode 100644 index 378a119150d5..000000000000 --- a/rpcs3/util/cereal.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include - -template -std::string cereal_serialize(const T&); - -template -void cereal_deserialize(T& data, const std::string& src); diff --git a/rpcs3/util/serialization.hpp b/rpcs3/util/serialization.hpp new file mode 100644 index 000000000000..b7da6f203e37 --- /dev/null +++ b/rpcs3/util/serialization.hpp @@ -0,0 +1,300 @@ +#pragma once + +#include "util/types.hpp" +#include + +namespace utils +{ + template + concept serializable = requires (Archive& ar, T& obj) { obj.serialize(ar); }; + + template + concept vector_alike = requires (T& obj) { obj.data()[0]; obj.reserve(0); }; + + template + concept array_alike = (!(requires (T& obj) { obj.clear(); })) && requires (T& obj) { std::data(obj)[0]; }; + + template + concept iterators_valid_most = (!(requires (T& obj) { obj.data()[0]; })) && requires (T& obj) { obj.insert(obj.end(), std::declval()); }; + + struct serial + { + std::vector data; + + void reserve(usz size) + { + data.reserve(data.size() + size); + } + + void write(const void* ptr, usz size) + { + data.insert(data.end(), static_cast(ptr), static_cast(ptr) + size); + } + + template + void write(T&& obj) + { + write(std::addressof(obj), sizeof(obj)); + } + + template + void write(T&& obj) requires serializable> + { + obj.serialize(*this); + } + + // std::vector, std::basic_string + template + void write(T&& obj) requires vector_alike> + { + for (usz i = obj.size();;) + { + const usz i_old = std::exchange(i, i >> 7); + write(static_cast(i_old % 0x80) | (u8{!!i} << 7)); + if (!i) break; + } + + if constexpr (std::is_trivially_copyable_v::value_type>) + { + write(obj.data(), sizeof(obj[0]) * obj.size()); + } + else + { + for (auto&& value : obj) + { + write(value); + } + } + } + + // C-array, std::array, std::span + template + void write(T&& obj) requires array_alike> + { + if constexpr (std::is_trivially_copyable_v()[0])>>) + { + write(obj.data(), sizeof(obj[0]) * std::size(obj)); + } + else + { + for (auto&& value : obj) + { + write(value); + } + } + } + + // std::deque, std::list, std::(unordered_)set, std::(unordered_)map, std::(unordered_)multiset, std::(unordered_)multimap + template + void write(T&& obj) requires iterators_valid_most> + { + for (usz i = obj.size();;) + { + const usz i_old = std::exchange(i, i >> 7); + write(static_cast(i_old % 0x80) | (u8{!!i} << 7)); + if (!i) break; + } + + for (auto&& value : obj) + { + write(value); + } + } + + template + void operator()(T&&... args) + { + (write(std::forward(args)), ...); + } + }; + + struct deserial + { + const std::vector data; + usz pos = 0; + + bool read(void* ptr, usz size) + { + if (pos == umax || data.size() - pos < size) + { + ensure(false); + //std::memset(ptr, 0, size); + //pos = -1; + //return false; + } + + std::memcpy(ptr, &data[pos], size); + pos += size; + return true; + } + + template + bool read(T& obj) + { + return read(std::addressof(obj), sizeof(obj)); + } + + template + bool read(T& obj) requires serializable + { + obj.serialize(*this); + return operator bool(); + } + + // std::vector, std::basic_string + template + bool read(T& obj) requires vector_alike + { + obj.clear(); + + usz size = 0; + + for (u32 i = 0;; i += 7) + { + u8 byte_data = 0; + + if (!read(byte_data)) + { + return false; + } + + size |= static_cast(byte_data % 0x80) << i; + + if (!(byte_data >> 7)) + { + break; + } + } + + obj.resize(size); + + if constexpr (std::is_trivially_copyable_v) + { + if (!read(obj.data(), sizeof(obj[0]) * size)) + { + obj.clear(); + return false; + } + } + else + { + for (auto&& value : obj) + { + if (!read(value)) + { + obj.clear(); + return false; + } + } + } + + return true; + } + + // C-array, std::array, std::span + template + bool read(T& obj) requires array_alike + { + if constexpr (std::is_trivially_copyable_v()[0])>>) + { + return read(std::data(obj), sizeof(obj[0]) * std::size(obj)); + } + + for (auto&& value : obj) + { + if (!read(value)) + { + return false; + } + } + + return true; + } + + // std::deque, std::list, std::(unordered_)set, std::(unordered_)map, std::(unordered_)multiset, std::(unordered_)multimap + template + bool read(T& obj) requires iterators_valid_most + { + obj.clear(); + + usz size = 0; + + for (u32 i = 0;; i += 7) + { + u8 byte_data = 0; + + if (!read(byte_data)) + { + return false; + } + + size |= static_cast(byte_data % 0x80) << i; + + if (!(byte_data >> 7)) + { + break; + } + } + + for (usz i = 0; i < size; i++) + { + using type = typename T::value_type; + + if constexpr (std::is_trivially_copyable_v) + { + char buf[sizeof(type)]{}; + + if (!read(buf)) + { + obj.clear(); + return false; + } + + obj.insert(obj.end(), std::bit_cast(buf)); + } + else + { + type value{}; + + if (!read(value)) + { + obj.clear(); + return false; + } + + obj.insert(obj.end(), std::move(value)); + } + } + + return true; + } + + template requires (std::is_copy_constructible_v && (std::is_constructible_v || std::is_trivially_copyable_v)) + operator T() + { + if constexpr (std::is_constructible_v) + { + T value{}; + read(value); + return value; + } + else + { + char buf[sizeof(T)]{}; + read(buf); + return std::bit_cast(buf); + } + } + + explicit operator bool() const + { + return pos <= data.size(); + } + + template + bool operator()(T&... args) + { + return (read(args), ...); + } + }; +}