[RFC] TexturePacker: add zstd compression support

CMakeLists.txt

@@ -36,6 +36,8 @@ if(NOT VERBOSE)
   set(CMAKE_REQUIRED_QUIET ON)
 endif()
 
+set(TEXTUREPACKER_COMPRESSION_METHOD "zstd" CACHE STRING "TexturePacker compression method [none|lzo|zstd]")
+
 # Includes
 include(cmake/modules/extra/ECMEnableSanitizers.cmake)
 include(cmake/scripts/common/GeneratorSetup.cmake)
@@ -187,6 +189,7 @@ set(required_deps ASS
                   TagLib
                   TinyXML
                   ZLIB
+                  ZSTD
                   ${PLATFORM_REQUIRED_DEPS})
 
 # Optional dependencies. Keep in alphabetical order please

cmake/modules/FindZSTD.cmake

@@ -0,0 +1,35 @@
+#.rst:
+# FindZSTD
+# --------
+# Finds the ZSTD library
+#
+# This will define the following variables::
+#
+# ZSTD_FOUND - system has ZSTD
+# ZSTD_INCLUDE_DIRS - the ZSTD include directory
+# ZSTD_LIBRARIES - the ZSTD libraries
+#
+
+if(PKG_CONFIG_FOUND)
+  pkg_check_modules(PC_LIBZSTD libzstd QUIET)
+endif()
+
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h
+                           PATHS ${PC_LIBZSTD_INCLUDEDIR})
+
+find_library(ZSTD_LIBRARY NAMES zstd libzstd
+                          PATHS ${PC_LIBZSTD_LIBDIR})
+
+set(ZSTD_VERSION ${PC_LIBZSTD_VERSION})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ZSTD
+                                  REQUIRED_VARS ZSTD_LIBRARY ZSTD_INCLUDE_DIR
+                                  VERSION_VAR ZSTD_VERSION)
+
+if(ZSTD_FOUND)
+  set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+  set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY)

cmake/scripts/common/ProjectMacros.cmake

@@ -15,6 +15,7 @@ function(pack_xbt input output)
                      ARGS    -input ${input}
                              -output ${output}
                              -dupecheck
+                             -compression-method ${TEXTUREPACKER_COMPRESSION_METHOD}
                      DEPENDS ${MEDIA_FILES})
   list(APPEND XBT_FILES ${output})
   set(XBT_FILES ${XBT_FILES} PARENT_SCOPE)

tools/depends/native/Makefile

@@ -30,7 +30,8 @@ NATIVE= \
   python3 \
   swig \
   TexturePacker \
-  zlib
+  zlib \
+  zstd
 
 ifneq ($(NATIVE_OS),osx)
   NATIVE += libffi
@@ -80,7 +81,7 @@ pugixml: cmake
 python3: $(EXPAT) $(LIBFFI) pkg-config zlib openssl autoconf-archive
 swig: pcre
 tar: xz automake
-TexturePacker: automake pkg-config libpng liblzo2 giflib libjpeg-turbo
+TexturePacker: automake pkg-config libpng liblzo2 giflib libjpeg-turbo zstd
 wayland-scanner: expat pkg-config
 waylandpp-scanner: cmake pugixml
 

tools/depends/native/TexturePacker/CMakeLists.txt

@@ -8,6 +8,7 @@ find_package(Lzo2 REQUIRED)
 find_package(PNG REQUIRED)
 find_package(GIF REQUIRED)
 find_package(JPEG REQUIRED)
+find_package(ZSTD REQUIRED)
 
 if(GIF_VERSION LESS 4)
   message(FATAL_ERROR "giflib < 4 not supported")
@@ -42,6 +43,7 @@ target_include_directories(TexturePacker
                            PRIVATE ${PNG_INCLUDE_DIRS}
                                     ${JPEG_INCLUDE_DIR}
                                     ${GIF_INCLUDE_DIR}
+                                    ${ZSTD_INCLUDE_DIRS}
                                     ${CMAKE_SOURCE_DIR}/xbmc
                                     ${CMAKE_CURRENT_SOURCE_DIR}/src
                                     ${CMAKE_CURRENT_SOURCE_DIR}/src/decoder)
@@ -50,5 +52,6 @@ target_link_libraries(TexturePacker
                               ${GIF_LIBRARIES}
                               ${PNG_LIBRARIES}
                               ${JPEG_LIBRARIES}
+                              ${ZSTD_LIBRARIES}
                               ${LZO2_LIBRARIES})
 target_compile_options(TexturePacker PRIVATE ${ARCH_DEFINES} ${SYSTEM_DEFINES})

tools/depends/native/TexturePacker/src/TexturePacker.cpp

@@ -27,30 +27,29 @@
 #include <inttypes.h>
 #define platform_stricmp strcasecmp
 #endif
-#include <cerrno>
-#include <dirent.h>
-#include <map>
-
+#include "DecoderManager.h"
+#include "XBTFWriter.h"
+#include "cmdlineargs.h"
 #include "guilib/XBTF.h"
 #include "guilib/XBTFReader.h"
+#include "md5.h"
 
-#include "DecoderManager.h"
+#include <cerrno>
+#include <map>
 
-#include "XBTFWriter.h"
-#include "md5.h"
-#include "cmdlineargs.h"
+#include <dirent.h>
+#include <zstd.h>
 
 #ifdef TARGET_WINDOWS
 #define strncasecmp _strnicmp
 #endif
 
+#include <chrono>
 #include <vector>
 
 #include <lzo/lzo1x.h>
 #include <sys/stat.h>
 
-#define FLAGS_USE_LZO     1
-
 #define DIR_SEPARATOR '/'
 
 namespace
@@ -91,10 +90,11 @@ bool HasAlpha(unsigned char* argb, unsigned int width, unsigned int height)
 void Usage()
 {
   puts("Usage:");
-  puts("  -help            Show this screen.");
-  puts("  -input <dir>     Input directory. Default: current dir");
-  puts("  -output <dir>    Output directory/filename. Default: Textures.xbt");
-  puts("  -dupecheck       Enable duplicate file detection. Reduces output file size. Default: off");
+  puts("  -help               Show this screen.");
+  puts("  -input <dir>        Input directory. Default: current dir");
+  puts("  -output <dir>       Output directory/filename. Default: Textures.xbt");
+  puts("  -dupecheck          Enable duplicate file detection. Reduces output file size. Default: off");
+  puts("  -compression-method Compression method to use. [lzo|none] Default: lzo");
 }
 
 } // namespace
@@ -111,7 +111,10 @@ class TexturePacker
 
   int createBundle(const std::string& InputDir, const std::string& OutputFile);
 
-  void SetFlags(unsigned int flags) { m_flags = flags; }
+  void SetCompressionMethod(XBTFCompressionMethod compressionMethod)
+  {
+    m_compressionMethod = compressionMethod;
+  }
 
 private:
   void CreateSkeletonHeader(CXBTFWriter& xbtfWriter,
@@ -128,7 +131,7 @@ class TexturePacker
   std::vector<unsigned int> m_dupes;
 
   bool m_dupecheck{false};
-  unsigned int m_flags{0};
+  XBTFCompressionMethod m_compressionMethod = XBTFCompressionMethod::NONE;
 };
 
 void TexturePacker::CreateSkeletonHeader(CXBTFWriter& xbtfWriter,
@@ -203,9 +206,9 @@ CXBTFFrame TexturePacker::CreateXBTFFrame(DecodedFrame& decodedFrame, CXBTFWrite
   const bool hasAlpha = HasAlpha(data, width, height);
 
   CXBTFFrame frame;
-  lzo_uint packedSize = size;
+  uint64_t packedSize = size;
 
-  if ((m_flags & FLAGS_USE_LZO) == FLAGS_USE_LZO)
+  if (m_compressionMethod == XBTFCompressionMethod::LZO)
   {
     // grab a temporary buffer for unpacking into
     packedSize = size + size / 16 + 64 + 3; // see simple.c in lzo
@@ -222,6 +225,8 @@ CXBTFFrame TexturePacker::CreateXBTFFrame(DecodedFrame& decodedFrame, CXBTFWrite
       // compression failed, or compressed size is bigger than uncompressed, so store as uncompressed
       packedSize = size;
       writer.AppendContent(data, size);
+
+      frame.SetCompressionMethod(XBTFCompressionMethod::NONE);
     }
     else
     { // success
@@ -231,16 +236,47 @@ CXBTFFrame TexturePacker::CreateXBTFFrame(DecodedFrame& decodedFrame, CXBTFWrite
       { //optimisation failed
         packedSize = size;
         writer.AppendContent(data, size);
+
+        frame.SetCompressionMethod(XBTFCompressionMethod::NONE);
       }
       else
       { // success
         writer.AppendContent(packed.data(), packedSize);
+
+        frame.SetCompressionMethod(XBTFCompressionMethod::LZO);
       }
     }
   }
+  else if (m_compressionMethod == XBTFCompressionMethod::ZSTD)
+  {
+    packedSize = ZSTD_compressBound(size);
+
+    std::vector<uint8_t> packed(packedSize);
+
+    packedSize = ZSTD_compress(packed.data(), packed.size(), data, size, ZSTD_CLEVEL_DEFAULT);
+
+    if (ZSTD_isError(packedSize) == 1)
+    {
+      fprintf(stderr, "ztd error: %s\n", ZSTD_getErrorName(packedSize));
+
+      packedSize = size;
+      writer.AppendContent(data, size);
+
+      frame.SetCompressionMethod(XBTFCompressionMethod::NONE);
+    }
+    else
+    {
+      packed.resize(packedSize);
+      writer.AppendContent(packed.data(), packed.size());
+
+      frame.SetCompressionMethod(XBTFCompressionMethod::ZSTD);
+    }
+  }
   else
   {
     writer.AppendContent(data, size);
+
+    frame.SetCompressionMethod(XBTFCompressionMethod::NONE);
   }
   frame.SetPackedSize(packedSize);
   frame.SetUnpackedSize(size);
@@ -291,6 +327,8 @@ int TexturePacker::createBundle(const std::string& InputDir, const std::string&
   std::vector<CXBTFFile> files = writer.GetFiles();
   m_dupes.resize(files.size());
 
+  const auto start = std::chrono::steady_clock::now();
+
   for (size_t i = 0; i < files.size(); i++)
   {
     struct MD5Context ctx;
@@ -340,17 +378,27 @@ int TexturePacker::createBundle(const std::string& InputDir, const std::string&
         CXBTFFrame frame = CreateXBTFFrame(frames.frameList[j], writer);
         file.GetFrames().push_back(frame);
         printf("    frame %4i (delay:%4i)                         %s%c (%d,%d @ %" PRIu64
-               " bytes)\n",
+               " -> %" PRIu64 " bytes, compression: %s)\n",
                j, frame.GetDuration(), GetFormatString(frame.GetFormat()),
                frame.HasAlpha() ? ' ' : '*', frame.GetWidth(), frame.GetHeight(),
-               frame.GetUnpackedSize());
+               frame.GetUnpackedSize(), frame.GetPackedSize(),
+               XBTFCompressionMethodMap.at(frame.GetCompressionMethod()).data());
       }
     }
     file.SetLoop(0);
 
     writer.UpdateFile(file);
   }
 
+  const auto end = std::chrono::steady_clock::now();
+
+  const auto diff =
+      std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(end - start);
+
+  printf(
+      "TexturePacker: processed %lu files in %.3f ms using %s\n", files.size(), diff.count(),
+      XBTFCompressionMethodMap.at(files.front().GetFrames().front().GetCompressionMethod()).data());
+
   if (!writer.UpdateHeader(m_dupes))
   {
     fprintf(stderr, "Error writing header to file\n");
@@ -385,7 +433,7 @@ int main(int argc, char* argv[])
 
   TexturePacker texturePacker;
 
-  texturePacker.SetFlags(FLAGS_USE_LZO);
+  texturePacker.SetCompressionMethod(XBTFCompressionMethod::LZO);
 
   for (unsigned int i = 1; i < args.size(); ++i)
   {
@@ -407,6 +455,25 @@ int main(int argc, char* argv[])
     {
       texturePacker.EnableVerboseOutput();
     }
+    else if (!strcmp(args[i], "-compression-method"))
+    {
+      std::string compressionMethod = args[++i];
+
+      if (compressionMethod == "lzo")
+      {
+        texturePacker.SetCompressionMethod(XBTFCompressionMethod::LZO);
+      }
+
+      if (compressionMethod == "zstd")
+      {
+        texturePacker.SetCompressionMethod(XBTFCompressionMethod::ZSTD);
+      }
+
+      if (compressionMethod == "none")
+      {
+        texturePacker.SetCompressionMethod(XBTFCompressionMethod::NONE);
+      }
+    }
     else if (!platform_stricmp(args[i], "-output") || !platform_stricmp(args[i], "-o"))
     {
       OutputFilename = args[++i];

tools/depends/native/TexturePacker/src/XBTFWriter.cpp

@@ -131,6 +131,7 @@ bool CXBTFWriter::UpdateHeader(const std::vector<unsigned int>& dupes)
       WRITE_U64(frame.GetUnpackedSize(), m_file);
       WRITE_U32(frame.GetDuration(), m_file);
       WRITE_U64(frame.GetOffset(), m_file);
+      WRITE_U32(static_cast<uint32_t>(frame.GetCompressionMethod()), m_file);
     }
   }
 

tools/depends/native/TexturePacker/src/cmdlineargs.h

@@ -26,8 +26,9 @@ char* GetCommandLine();
 #else
 #include <windows.h>
 #endif
-#include <vector>
+#include <cstring>
 #include <string>
+#include <vector>
 
 class CmdLineArgs : public std::vector<char*>
 {

tools/depends/native/TexturePacker/src/configure.ac

@@ -33,6 +33,9 @@ AC_CHECK_HEADER([jpeglib.h],, AC_MSG_ERROR("jpeglib.h not found"))
 AC_CHECK_LIB([jpeg],[main],, AC_MSG_ERROR("libjpeg not found"))
 AC_CHECK_HEADER([lzo/lzo1x.h],, AC_MSG_ERROR("lzo/lzo1x.h not found"))
 AC_CHECK_LIB([lzo2],[main],, AC_MSG_ERROR("liblzo2 not found"))
+AC_CHECK_HEADER([zstd.h],, AC_MSG_ERROR("zstd.h not found"))
+AC_CHECK_LIB([zstd],[main],, AC_MSG_ERROR("libzstd not found"))
+
 
 AC_SUBST(KODI_SRC_DIR)
 AC_SUBST(STATIC_FLAG)

tools/depends/native/zstd/Makefile

@@ -0,0 +1,30 @@
+include ../../Makefile.include
+PREFIX=$(NATIVEPREFIX)
+PLATFORM=$(NATIVEPLATFORM)
+DEPS = ../../Makefile.include Makefile ../../download-files.include
+
+LIBNAME=zstd
+VERSION=1.5.5
+SOURCE=$(LIBNAME)-$(VERSION)
+ARCHIVE=$(SOURCE).tar.gz
+SHA512=99109ec0e07fa65c2101c9cb36be56b672bbd0ee69d265f924718e61f9192ae8385c8d9e4d0c318be9edfa6d849fd3d60e5f164fa120961449429ea3c5dab6b6
+include ../../download-files.include
+
+all: .installed-$(PLATFORM)
+
+$(PLATFORM): $(DEPS) | $(TARBALLS_LOCATION)/$(ARCHIVE).$(HASH_TYPE)
+	rm -rf $(PLATFORM)/*; mkdir -p $(PLATFORM)
+	cd $(PLATFORM); $(ARCHIVE_TOOL) $(ARCHIVE_TOOL_FLAGS) $(TARBALLS_LOCATION)/$(ARCHIVE)
+
+.installed-$(PLATFORM): $(PLATFORM)
+	$(MAKE) -C $(PLATFORM)/lib libzstd.a PREFIX=$(PREFIX)
+	$(MAKE) -C $(PLATFORM)/lib install-pc install-static install-includes PREFIX=$(PREFIX)
+	touch $@
+
+clean:
+	$(MAKE) -C $(PLATFORM) clean
+	rm -f .installed-$(PLATFORM)
+
+distclean::
+	rm -rf $(PLATFORM) .installed-$(PLATFORM)
+

tools/depends/target/Makefile

@@ -48,7 +48,8 @@ DEPENDS = \
   sqlite3 \
   tinyxml \
   udfread \
-  xz
+  xz \
+  zstd
 
 ifeq ($(ENABLE_GPLV3),yes)
   DEPENDS+=samba-gplv3 libcdio-gplv3