Add: Support Zstandard(zstd) savegame compression

.github/workflows/ci-build.yml

@@ -97,6 +97,7 @@ jobs:
           libfontconfig-dev \
           libicu-dev \
           liblzma-dev \
+          libzstd-dev \
           liblzo2-dev \
           ${{ matrix.libsdl }} \
           zlib1g-dev \
@@ -175,6 +176,7 @@ jobs:
       run: |
         vcpkg install --triplet=${{ matrix.arch }}-osx \
           liblzma \
+          zstd \
           libpng \
           lzo \
           zlib \
@@ -256,6 +258,7 @@ jobs:
       run: |
         vcpkg install --triplet=${{ matrix.arch }}-windows-static \
           liblzma \
+          zstd \
           libpng \
           lzo \
           zlib \

.github/workflows/release.yml

@@ -297,6 +297,7 @@ jobs:
           SDL2-devel \
           wget \
           xz-devel \
+          libzstd-devel \
           zlib-devel \
           # EOF
         echo "::endgroup::"
@@ -412,6 +413,7 @@ jobs:
           libfluidsynth-dev \
           libicu-dev \
           liblzma-dev \
+          libzstd-dev \
           liblzo2-dev \
           libsdl2-dev \
           lsb-release \
@@ -503,6 +505,8 @@ jobs:
         vcpkg install \
           liblzma:x64-osx \
           liblzma:arm64-osx \
+          zstd:x64-osx \
+          zstd:arm64-osx \
           libpng:x64-osx \
           libpng:arm64-osx \
           lzo:x64-osx \
@@ -703,6 +707,7 @@ jobs:
       run: |
         vcpkg install --triplet=${{ matrix.arch }}-windows-static \
           liblzma \
+          zstd \
           libpng \
           lzo \
           zlib \

CMakeLists.txt

@@ -117,6 +117,7 @@ find_package(Threads REQUIRED)
 find_package(ZLIB)
 find_package(LibLZMA)
 find_package(LZO)
+find_package(ZSTD)
 find_package(PNG)
 
 if(NOT WIN32)
@@ -244,6 +245,7 @@ link_package(PNG TARGET PNG::PNG ENCOURAGED)
 link_package(ZLIB TARGET ZLIB::ZLIB ENCOURAGED)
 link_package(LIBLZMA TARGET LibLZMA::LibLZMA ENCOURAGED)
 link_package(LZO)
+link_package(ZSTD TARGET ZSTD::ZSTD ENCOURAGED)
 
 if(NOT OPTION_DEDICATED)
     link_package(Fluidsynth)

COMPILING.md

@@ -8,6 +8,7 @@ The following libraries are used by OpenTTD for:
    heightmaps
 - liblzo2: (de)compressing of old (pre 0.3.0) savegames
 - liblzma: (de)compressing of savegames (1.1.0 and later)
+- libzstd: (de)compressing of savegames (1.11.0 and later)
 - libpng: making screenshots and loading heightmaps
 - libfreetype: loading generic fonts and rendering them
 - libfontconfig: searching for fonts, resolving font names to actual fonts
@@ -45,15 +46,16 @@ After this, you can install the dependencies OpenTTD needs. We advise to use
 the `static` versions, and OpenTTD currently needs the following dependencies:
 
 - liblzma
+- libzstd
 - libpng
 - lzo
 - zlib
 
 To install both the x64 (64bit) and x86 (32bit) variants (though only one is necessary), you can use:
 
 ```ps
-.\vcpkg install liblzma:x64-windows-static libpng:x64-windows-static lzo:x64-windows-static zlib:x64-windows-static
-.\vcpkg install liblzma:x86-windows-static libpng:x86-windows-static lzo:x86-windows-static zlib:x86-windows-static
+.\vcpkg install liblzma:x64-windows-static zstd:x64-windows-static libpng:x64-windows-static lzo:x64-windows-static zlib:x64-windows-static
+.\vcpkg install liblzma:x86-windows-static zstd:x86-windows-static libpng:x86-windows-static lzo:x86-windows-static zlib:x86-windows-static
 ```
 
 You can open the folder (as a CMake project). CMake will be detected, and you can compile from there.

Doxyfile.in

@@ -290,6 +290,7 @@ INCLUDE_FILE_PATTERNS  =
 PREDEFINED             = WITH_ZLIB \
                          WITH_LZO \
                          WITH_LIBLZMA \
+                         WITH_ZSTD \
                          WITH_SDL \
                          WITH_PNG \
                          WITH_FONTCONFIG \

cmake/FindZSTD.cmake

@@ -0,0 +1,89 @@
+#[=======================================================================[.rst:
+FindZSTD
+-------
+
+Finds the ZSTD library.
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+This will define the following variables:
+
+``ZSTD_FOUND``
+  True if the system has the ZSTD library.
+``ZSTD_INCLUDE_DIRS``
+  Include directories needed to use ZSTD.
+``ZSTD_LIBRARIES``
+  Libraries needed to link to ZSTD.
+``ZSTD_VERSION``
+  The version of the ZSTD library which was found.
+
+Cache Variables
+^^^^^^^^^^^^^^^
+
+The following cache variables may also be set:
+
+``ZSTD_INCLUDE_DIR``
+  The directory containing ``zstd.h``.
+``ZSTD_LIBRARY``
+  The path to the ZSTD library.
+
+#]=======================================================================]
+
+find_package(PkgConfig QUIET)
+pkg_check_modules(PC_ZSTD QUIET libzstd)
+
+find_path(ZSTD_INCLUDE_DIR
+    NAMES zstd.h
+    PATHS ${PC_ZSTD_INCLUDE_DIRS}
+)
+
+find_library(ZSTD_LIBRARY
+    NAMES zstd
+    PATHS ${PC_ZSTD_LIBRARY_DIRS}
+)
+
+# With vcpkg, the library path should contain both 'debug' and 'optimized'
+# entries (see target_link_libraries() documentation for more information)
+#
+# NOTE: we only patch up when using vcpkg; the same issue might happen
+# when not using vcpkg, but this is non-trivial to fix, as we have no idea
+# what the paths are. With vcpkg we do. And we only official support vcpkg
+# with Windows.
+#
+# NOTE: this is based on the assumption that the debug file has the same
+# name as the optimized file. This is not always the case, but so far
+# experiences has shown that in those case vcpkg CMake files do the right
+# thing.
+if(VCPKG_TOOLCHAIN AND ZSTD_LIBRARY)
+    if(ZSTD_LIBRARY MATCHES "/debug/")
+        set(ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY})
+        string(REPLACE "/debug/lib/" "/lib/" ZSTD_LIBRARY_RELEASE ${ZSTD_LIBRARY})
+    else()
+        set(ZSTD_LIBRARY_RELEASE ${ZSTD_LIBRARY})
+        string(REPLACE "/lib/" "/debug/lib/" ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY})
+    endif()
+    include(SelectLibraryConfigurations)
+    select_library_configurations(ZSTD)
+endif()
+
+set(ZSTD_VERSION ${PC_ZSTD_VERSION})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ZSTD
+    FOUND_VAR ZSTD_FOUND
+    REQUIRED_VARS
+        ZSTD_LIBRARY
+        ZSTD_INCLUDE_DIR
+    VERSION_VAR ZSTD_VERSION
+)
+
+if(ZSTD_FOUND)
+    set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+    set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+    ZSTD_INCLUDE_DIR
+    ZSTD_LIBRARY
+)

src/crashlog.cpp

@@ -56,6 +56,9 @@
 #ifdef WITH_LIBLZMA
 #	include <lzma.h>
 #endif
+#ifdef WITH_ZSTD
+#include <zstd.h>
+#endif
 #ifdef WITH_LZO
 #include <lzo/lzo1x.h>
 #endif
@@ -255,6 +258,10 @@ char *CrashLog::LogLibraries(char *buffer, const char *last) const
 	buffer += seprintf(buffer, last, " LZMA:       %s\n", lzma_version_string());
 #endif
 
+#ifdef WITH_ZSTD
+	buffer += seprintf(buffer, last, " ZSTD:       %s\n", ZSTD_versionString());
+#endif
+
 #ifdef WITH_LZO
 	buffer += seprintf(buffer, last, " LZO:        %s\n", lzo_version_string());
 #endif

src/saveload/saveload.cpp

@@ -2301,6 +2301,119 @@ struct LZMASaveFilter : SaveFilter {
 
 #endif /* WITH_LIBLZMA */
 
+/********************************************
+ ********** START OF ZSTD CODE **************
+ ********************************************/
+
+#if defined(WITH_ZSTD)
+#include <zstd.h>
+
+
+/** Filter using ZSTD compression. */
+struct ZSTDLoadFilter : LoadFilter {
+	ZSTD_DCtx *zstd;  ///< ZSTD decompression context
+	byte fread_buf[MEMORY_CHUNK_SIZE];  ///< Buffer for reading from the file
+	ZSTD_inBuffer input;  ///< ZSTD input buffer for fread_buf
+
+	/**
+	 * Initialise this filter.
+	 * @param chain The next filter in this chain.
+	 */
+	ZSTDLoadFilter(LoadFilter *chain) : LoadFilter(chain)
+	{
+		this->zstd = ZSTD_createDCtx();
+	    if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
+	    this->input = {this->fread_buf, 0, 0};
+	}
+
+	/** Clean everything up. */
+	~ZSTDLoadFilter()
+	{
+		ZSTD_freeDCtx(this->zstd);
+	}
+
+	size_t Read(byte *buf, size_t size) override
+	{
+		ZSTD_outBuffer output{buf, size, 0};
+
+		do {
+			/* read more bytes from the file? */
+			if (this->input.pos == this->input.size) {
+				this->input.size = this->chain->Read(this->fread_buf, sizeof(this->fread_buf));
+				this->input.pos = 0;
+			}
+
+			size_t ret = ZSTD_decompressStream(this->zstd, &output, &this->input);
+			if (ZSTD_isError(ret)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code");
+			if (ret == 0) break;
+		} while (output.pos < output.size);
+
+		return output.pos;
+	}
+};
+
+/** Filter using ZSTD compression. */
+struct ZSTDSaveFilter : SaveFilter {
+	ZSTD_CCtx *zstd;  ///< ZSTD compression context
+
+	/**
+	 * Initialise this filter.
+	 * @param chain             The next filter in this chain.
+	 * @param compression_level The requested level of compression.
+	 */
+	ZSTDSaveFilter(SaveFilter *chain, byte compression_level) : SaveFilter(chain)
+	{
+	    this->zstd = ZSTD_createCCtx();
+	    if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
+	    if (ZSTD_isError(ZSTD_CCtx_setParameter(this->zstd, ZSTD_c_compressionLevel, (int)compression_level - 100))) {
+			ZSTD_freeCCtx(this->zstd);
+			SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "invalid compresison level");
+	    }
+	}
+
+	/** Clean up what we allocated. */
+	~ZSTDSaveFilter()
+	{
+		ZSTD_freeCCtx(this->zstd);
+	}
+
+	/**
+	 * Helper loop for writing the data.
+	 * @param p      The bytes to write.
+	 * @param len    Amount of bytes to write.
+	 * @param mode   Mode for ZSTD_compressStream2.
+	 */
+	void WriteLoop(byte *p, size_t len, ZSTD_EndDirective mode)
+	{
+		byte buf[MEMORY_CHUNK_SIZE]; // output buffer
+		ZSTD_inBuffer input{p, len, 0};
+
+		bool finished;
+		do {
+			ZSTD_outBuffer output{buf, sizeof(buf), 0};
+			size_t remaining = ZSTD_compressStream2(this->zstd, &output, &input, mode);
+			if (ZSTD_isError(remaining)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code");
+
+			if (output.pos != 0) this->chain->Write(buf, output.pos);
+
+			finished = (mode == ZSTD_e_end ? (remaining == 0) : (input.pos == input.size));
+		} while (!finished);
+	}
+
+	void Write(byte *buf, size_t size) override
+	{
+		this->WriteLoop(buf, size, ZSTD_e_continue);
+	}
+
+	void Finish() override
+	{
+		this->WriteLoop(nullptr, 0, ZSTD_e_end);
+		this->chain->Finish();
+	}
+};
+
+#endif /* WITH_LIBZSTD */
+
 /*******************************************
  ************* END OF CODE *****************
  *******************************************/
@@ -2336,6 +2449,17 @@ static const SaveLoadFormat _saveload_formats[] = {
 #else
 	{"zlib",   TO_BE32X('OTTZ'), nullptr,                            nullptr,                            0, 0, 0},
 #endif
+#if defined(WITH_ZSTD)
+	/* Zstd provides a decent compression rate at a very high compression/decompression speed. Compared to lzma level 2
+	 * zstd saves are about 40% larger (on level 1) but it has about 30x faster compression and 5x decompression making it
+	 * a good choice for multiplayer servers. And zstd level 1 seems to be the optimal one for client connection speed
+	 * (compress + 10 MB/s download + decompress time), about 3x faster than lzma:2 and 1.5x than zlib:2 and lzo.
+	 * As zstd has negative compression levels the values were increased by 100 moving zstd level range -100..22 into
+	 * openttd 0..122. Also note that value 100 mathes zstd level 0 which is a special value for default level 3 (openttd 103) */
+	{"zstd",   TO_BE32X('OTTS'), CreateLoadFilter<ZSTDLoadFilter>,   CreateSaveFilter<ZSTDSaveFilter>,   0, 101, 122},
+#else
+	{"zstd",   TO_BE32X('OTTS'), nullptr,                            nullptr,                            0, 0, 0},
+#endif
 #if defined(WITH_LIBLZMA)
 	/* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves.
 	 * Higher compression levels are possible, and might improve savegame size by up to 25%, but are also up to 10 times slower.