Skip to content

Commit

Permalink
Savestates: Use LZ4 algorithm for faster decompression
Browse files Browse the repository at this point in the history
  • Loading branch information
@malleoz
malleoz committed Oct 2, 2023
1 parent 1772f16 commit 17290b4
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 81 deletions.
200 changes: 121 additions & 79 deletions Source/Core/Core/State.cpp
View file
Expand Up @@ -16,7 +16,7 @@

#include <fmt/format.h>

#include <lzo/lzo1x.h>
#include <lz4.h>

#include "Common/ChunkFile.h"
#include "Common/CommonTypes.h"
Expand Down Expand Up @@ -48,23 +48,6 @@

namespace State
{
#if defined(__LZO_STRICT_16BIT)
static const u32 IN_LEN = 8 * 1024u;
#elif defined(LZO_ARCH_I086) && !defined(LZO_HAVE_MM_HUGE_ARRAY)
static const u32 IN_LEN = 60 * 1024u;
#else
static const u32 IN_LEN = 128 * 1024u;
#endif

static const u32 OUT_LEN = IN_LEN + (IN_LEN / 16) + 64 + 3;

static unsigned char __LZO_MMODEL out[OUT_LEN];

#define HEAP_ALLOC(var, size) \
lzo_align_t __LZO_MMODEL var[((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t)]

static HEAP_ALLOC(wrkmem, LZO1X_1_MEM_COMPRESS);

static AfterLoadCallbackFunc s_on_after_load_callback;

// Temporary undo state buffer
Expand Down Expand Up @@ -96,7 +79,7 @@ static size_t s_state_writes_in_queue;
static std::condition_variable s_state_write_queue_is_empty;

// Don't forget to increase this after doing changes on the savestate system
constexpr u32 STATE_VERSION = 162; // Last changed in PR 11767
constexpr u32 STATE_VERSION = 163; // Last changed in PR 12217

// Maps savestate versions to Dolphin versions.
// Versions after 42 don't need to be added to this list,
Expand Down Expand Up @@ -354,6 +337,81 @@ static std::map<double, int> GetSavedStates()
return m;
}

static void CompressBufferToFile(const u8* raw_buffer, u32 size, File::IOFile& f)
{
u32 total_bytes_compressed = 0;

while (true)
{
u32 bytes_left_to_compress = size - total_bytes_compressed;

int bytes_to_compress =
static_cast<int>(std::min((u32)LZ4_MAX_INPUT_SIZE, bytes_left_to_compress));
int compressed_buffer_size = LZ4_compressBound(bytes_to_compress);
auto compressed_buffer = std::make_unique<char[]>(compressed_buffer_size);
int compressed_len =
LZ4_compress_default((char*)raw_buffer + total_bytes_compressed, compressed_buffer.get(),
bytes_to_compress, compressed_buffer_size);

if (compressed_len == 0)
PanicAlertFmtT("Internal LZ4 Error - compression failed");

// The size of the data to write is 'compressed_len'
f.WriteArray(&compressed_len, 1);
f.WriteBytes(compressed_buffer.get(), compressed_len);

total_bytes_compressed += bytes_to_compress;
if (total_bytes_compressed == size)
break;
}
}

static u32 WriteVersionInfo(const u8* buffer_data, File::IOFile& f)
{
u32 offset = 0;

// Write version cookie
f.WriteBytes(buffer_data, sizeof(u32));
offset += sizeof(u32);

// Write version string length
u32 version_string_len;
memcpy(&version_string_len, buffer_data + offset, sizeof(u32));
f.WriteArray(&version_string_len, 1);
offset += sizeof(u32);

// Write version string
f.WriteBytes(buffer_data + offset, version_string_len);
offset += version_string_len;

return offset;
}

static u32 ReadVersionInfo(std::vector<u8>& buffer, File::IOFile& f)
{
// Read version cookie
u32 cookie;
f.ReadBytes(&cookie, sizeof(u32));

// Read version string length
u32 version_string_len;
f.ReadArray(&version_string_len, 1);

// Resize the vector to fit the cookie, length, and string
buffer.resize(2 * sizeof(u32) + version_string_len);

// Write to the buffer
u8* ptr = buffer.data();
memcpy(ptr, &cookie, sizeof(u32));
u32 offset = sizeof(u32);
memcpy(ptr + offset, &version_string_len, sizeof(u32));
offset += sizeof(u32);
f.ReadBytes(ptr + offset, version_string_len);
offset += version_string_len;

return offset;
}

static void CompressAndDumpState(CompressAndDumpState_args& save_args)
{
const u8* const buffer_data = save_args.buffer_vector.data();
Expand Down Expand Up @@ -386,40 +444,13 @@ static void CompressAndDumpState(CompressAndDumpState_args& save_args)

f.WriteArray(&header, 1);

if (header.size != 0) // non-zero header size means the state is compressed
{
lzo_uint i = 0;
while (true)
{
lzo_uint32 cur_len = 0;
lzo_uint out_len = 0;

if ((i + IN_LEN) >= buffer_size)
{
cur_len = (lzo_uint32)(buffer_size - i);
}
else
{
cur_len = IN_LEN;
}

if (lzo1x_1_compress(buffer_data + i, cur_len, out, &out_len, wrkmem) != LZO_E_OK)
PanicAlertFmtT("Internal LZO Error - compression failed");

// The size of the data to write is 'out_len'
f.WriteArray((lzo_uint32*)&out_len, 1);
f.WriteBytes(out, out_len);
// Keep the version uncompressed to limit issues changing compression algo in future
u32 offset = WriteVersionInfo(buffer_data, f);

if (cur_len != IN_LEN)
break;

i += cur_len;
}
}
else // uncompressed
{
f.WriteBytes(buffer_data, buffer_size);
}
if (s_use_compression)
CompressBufferToFile(buffer_data + offset, (u32)buffer_size - offset, f);
else
f.WriteBytes(buffer_data + offset, buffer_size - offset);

const std::string last_state_filename = File::GetUserPath(D_STATESAVES_IDX) + "lastState.sav";
const std::string last_state_dtmname = last_state_filename + ".dtm";
Expand Down Expand Up @@ -557,6 +588,39 @@ u64 GetUnixTimeOfSlot(int slot)
return static_cast<u64>(header.time * MS_PER_SEC) + (DOUBLE_TIME_OFFSET * MS_PER_SEC);
}

static void DecompressBufferFromFile(const u8* raw_buffer, u32 size, File::IOFile& f)
{
u32 total_bytes_read = 0;
while (true)
{
u32 compressed_data_len;
f.ReadArray(&compressed_data_len, 1);

auto compressed_data = std::make_unique<char[]>(compressed_data_len);
f.ReadBytes(compressed_data.get(), compressed_data_len);

// We need to specify the output buffer's size for safety. This may exceed the positive bound
// of int, causing the buffer size to be interpreted as a negative value.
u32 max_decompress_size = std::min((u32)LZ4_MAX_INPUT_SIZE, size - total_bytes_read);

int bytes_read = LZ4_decompress_safe(compressed_data.get(), (char*)raw_buffer,
compressed_data_len, max_decompress_size);

if (bytes_read < 0)
{
PanicAlertFmtT("Internal LZ4 Error - decompression failed ({0}, {1}, {2}) \n"
"Try loading the state again",
bytes_read, compressed_data_len, max_decompress_size);
break;
}

total_bytes_read += bytes_read;

if (total_bytes_read == size)
break;
}
}

static void LoadFileStateData(const std::string& filename, std::vector<u8>& ret_data)
{
File::IOFile f;
Expand Down Expand Up @@ -594,43 +658,24 @@ static void LoadFileStateData(const std::string& filename, std::vector<u8>& ret_

std::vector<u8> buffer;

u32 offset = ReadVersionInfo(buffer, f);

if (header.size != 0) // non-zero size means the state is compressed
{
Core::DisplayMessage("Decompressing State...", 500);

buffer.resize(header.size);

lzo_uint i = 0;
while (true)
{
lzo_uint32 cur_len = 0; // number of bytes to read
lzo_uint new_len = 0; // number of bytes to write

if (!f.ReadArray(&cur_len, 1))
break;

f.ReadBytes(out, cur_len);
const int res = lzo1x_decompress(out, cur_len, &buffer[i], &new_len, nullptr);
if (res != LZO_E_OK)
{
// This doesn't seem to happen anymore.
PanicAlertFmtT("Internal LZO Error - decompression failed ({0}) ({1}, {2}) \n"
"Try loading the state again",
res, i, new_len);
return;
}

i += new_len;
}
DecompressBufferFromFile(buffer.data() + offset, header.size - offset, f);
}
else // uncompressed
{
const auto size = static_cast<size_t>(f.GetSize() - sizeof(StateHeader));
buffer.resize(size);

if (!f.ReadBytes(&buffer[0], size))
if (!f.ReadBytes(buffer.data() + offset, size - offset))
{
PanicAlertFmt("Error reading bytes: {0}", size);
PanicAlertFmt("Error reading bytes: {0}", size - offset);
return;
}
}
Expand Down Expand Up @@ -721,9 +766,6 @@ void SetOnAfterLoadCallback(AfterLoadCallbackFunc callback)

void Init()
{
if (lzo_init() != LZO_E_OK)
PanicAlertFmtT("Internal LZO Error - lzo_init() failed");

s_save_thread.Reset("Savestate Worker", [](CompressAndDumpState_args args) {
CompressAndDumpState(args);

Expand Down
4 changes: 2 additions & 2 deletions Source/Core/Core/State.h
View file
Expand Up @@ -21,13 +21,13 @@ struct StateHeader
{
char gameID[6];
u16 reserved1;
u32 must_be_zero; // PR#12217, prevent decompression so older Dolphin can read state version info
u32 size;
u32 reserved2;
double time;
};
constexpr size_t STATE_HEADER_SIZE = sizeof(StateHeader);
static_assert(STATE_HEADER_SIZE == 24);
static_assert(offsetof(StateHeader, size) == 8);
static_assert(offsetof(StateHeader, size) == 12);
static_assert(offsetof(StateHeader, time) == 16);

void Init();
Expand Down

0 comments on commit 17290b4

Please sign in to comment.