Merge pull request #1257 from NiLuJe/cbb-c99

* CBB:
  * Reformat in a C99 fashion (tighter scope, restricted pointers).
     (This makes the diff a horrible mess to read, so, don't, and hope there aren't any more typos than those I already caught ;D).
  * Made some helper macros available in the header (e.g., GET_BB_*).
  * Very few logic changes, but a few which'll probably get back to me later :D.
  * Update the BlitBuffer structs to use unsigned types, and made the fact that stride is a byte count explicit by using a size_t.
  * Add support for straight-alpha blitFrom (useful for NanoSVG, which spits out straight alpha, unlike MµPDF).
  * Don't clobber the alpha byte in paintRect & fill (c.f., the tail end of koreader/koreader#6937)

* LBB:
  * Minor updates for feature-parity w/ CBB.
  * Fair warning about the stride is now size_t thing: a Lua number isn't precise enough to fit the full range of an uint64_t, so, on platforms where size_t is 64bit, it's *not* automatically coerced to a Lua number. The frontend companion PR takes care of the very few cases w<here this matters.
  * Fix a few instances where the invert flag was mishandled.

* Add ZSTD and an FFI binding.
  * ZSTD is now used by CRe to compress its cache
  * And it's also used by BookInfo to compress the thumbnails
  * BookInfo's now unused xutil has been moved to ffi/zlib

* Rejig our understanding and usage of the `__gc` metamethod: it doesn't run on plain tables in Lua 5.1, only on userdata. In LuaJIT, it's also usable on cdata metatypes, and cdata finalizers can also be setup explicitly.
Convert stuff to the sanest solution in each case, and, in cases where it's not possible, use the hacky workaround of introducing an empty `newproxy` userdata in the mix, via ffi/__gc.
This finally solves a number of puzzling code comments that were wondering why the hell our __gc methods were never called (hint: because it's not supported, or, in a few cases it's not supported and they were misnamed anyway ^^).

* Tweak lj-sqlite3 to stop coercing BLOBs as Lua strings. Instead return a raw pointer and a byte count, like the SQLite C API. This allows avoiding a few useless copies in BookInfo, our sole user of BLOB. It also just plain makes sense, because if I wanted strings, I'd be using a TEXT column ;).

* Use proper symbol visibility discipline for our Lua/C modules, much like was done in nnsvg. This avoids leaking a ton of useless symbols, and makes the dynamic loader's job that much easier.

* Update the stbiw version shipped by NanoSVG to the latest one (only used by a CRe helper function).

* Fun Valgrind experiments in cre.cpp if DEBUG_CRENGINE is set ;p.

.ci/emulator_install.sh

@@ -8,7 +8,7 @@ test -d "${HOME}/.luarocks" || {
     mkdir "${HOME}/.luarocks"
     cp /etc/luarocks/config-5.1.lua "${HOME}/.luarocks/config.lua"
     echo "wrap_bin_scripts = false" >>"${HOME}/.luarocks/config.lua"
-    travis_retry luarocks --local install busted 2.0.rc13-0
+    travis_retry luarocks --local install busted 2.0.0-1
     # for verbose_print module
     travis_retry luarocks --local install ansicolors
 }
@@ -18,11 +18,11 @@ eval "$(luarocks path --bin)"
 if [ "$CC" = "clang" ]; then
     wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
 
-    sudo tee -a /etc/apt/sources.list.d/clang7.list >/dev/null <<EOF
-deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main
-deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main
+    sudo tee -a /etc/apt/sources.list.d/clang11.list >/dev/null <<EOF
+deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main
+deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main
 EOF
 
     sudo apt-get update
-    sudo apt-get install -y clang-7
+    sudo apt-get install -y clang-11
 fi

.github/workflows/build.yml

@@ -32,8 +32,8 @@ jobs:
             ${{ runner.os }}-
 
       - name: Homebrew install dependencies
-        # Compared to the README, adds ccache for faster compilation times and removes sh5sum to prevent some conflict with coreutils
+        # Compared to the README, adds ccache for faster compilation times and removes sh5sum to prevent some conflict with coreutils, and gnu-getopt because we're not using kodev
         run: brew install ccache nasm ragel binutils coreutils libtool autoconf automake cmake makedepend sdl2 lua@5.1 luarocks gettext pkg-config wget
 
       - name: Building in progress…
-        run: export MACOSX_DEPLOYMENT_TARGET=10.14 PATH="/usr/local/opt/gettext/bin:$PATH" && make fetchthirdparty && make
+        run: export MACOSX_DEPLOYMENT_TARGET=10.14 PATH="$(brew --prefix)/opt/gettext/bin:${PATH}" && make fetchthirdparty && make

Makefile

@@ -22,6 +22,7 @@ all: $(OUTPUT_DIR)/libs $(if $(ANDROID),,$(LUAJIT)) \
 		$(TURBOJPEG_LIB) \
 		$(LODEPNG_LIB) \
 		$(GIF_LIB) \
+		$(ZSTD_LIB) \
 		$(if $(USE_LJ_WPACLIENT),$(LJ_WPACLIENT),) \
 		$(TURBO_FFI_WRAP_LIB) \
 		$(LUA_HTMLPARSER_ROCK) \
@@ -110,13 +111,14 @@ $(OUTPUT_DIR)/libs/libinkview-compat.so: input/inkview-compat.c
 
 $(OUTPUT_DIR)/libs/libkoreader-input.so: input/*.c input/*.h $(if $(KINDLE),$(POPEN_NOSHELL_LIB),)
 	@echo "Building koreader input module..."
-	$(CC) $(DYNLIB_CFLAGS) -I$(POPEN_NOSHELL_DIR) -I./input \
+	$(CC) $(DYNLIB_CFLAGS) $(SYMVIS_FLAGS) -I$(POPEN_NOSHELL_DIR) -I./input \
 		$(if $(CERVANTES),-DCERVANTES,) $(if $(KOBO),-DKOBO,) $(if $(KINDLE),-DKINDLE,) $(if $(POCKETBOOK),-DPOCKETBOOK,) $(if $(REMARKABLE),-DREMARKABLE,) $(if $(SONY_PRSTUX),-DSONY_PRSTUX,)\
 		-o $@ \
 		input/input.c \
 		$(if $(KINDLE),$(POPEN_NOSHELL_LIB),) \
 		$(if $(POCKETBOOK),-linkview,)
 
+# Would need a bit of patching to be able to use -fvisibility=hidden...
 $(OUTPUT_DIR)/libs/libkoreader-lfs.so: \
 			$(if $(USE_LUAJIT_LIB),$(LUAJIT_LIB),) \
 			luafilesystem/src/lfs.c
@@ -134,7 +136,7 @@ $(OUTPUT_DIR)/libs/libkoreader-djvu.so: djvu.c \
 			$(if $(USE_LUAJIT_LIB),$(LUAJIT_LIB),) \
 			$(DJVULIBRE_LIB) $(K2PDFOPT_LIB)
 	$(CC) -I$(DJVULIBRE_DIR) -I$(MUPDF_DIR)/include $(K2PDFOPT_CFLAGS) \
-		$(DYNLIB_CFLAGS) -o $@ $^ $(if $(ANDROID),,-lpthread)
+		$(DYNLIB_CFLAGS) $(SYMVIS_FLAGS) -o $@ $^ $(if $(ANDROID),,-lpthread)
 ifdef DARWIN
 	install_name_tool -change \
 		`otool -L "$@" | grep "libluajit" | awk '{print $$1}'` \
@@ -155,7 +157,7 @@ $(OUTPUT_DIR)/libs/libkoreader-cre.so: cre.cpp \
 			$(CRENGINE_LIB)
 	$(CXX) -I$(CRENGINE_SRC_DIR)/crengine/include/ $(DYNLIB_CXXFLAGS) \
 		-DLDOM_USE_OWN_MEM_MAN=$(if $(WIN32),0,1) \
-		$(if $(WIN32),-DQT_GL=1) -static-libstdc++ -o $@ $^
+		$(if $(WIN32),-DQT_GL=1) $(SYMVIS_FLAGS) -static-libstdc++ -o $@ $^
 ifdef DARWIN
 	install_name_tool -change \
 		`otool -L "$@" | grep "libluajit" | awk '{print $$1}'` \
@@ -174,7 +176,7 @@ $(OUTPUT_DIR)/libs/libkoreader-xtext.so: xtext.cpp \
 	-I$(HARFBUZZ_DIR)/include/harfbuzz \
 	-I$(FRIBIDI_DIR)/include \
 	-I$(LIBUNIBREAK_DIR)/include \
-	$(DYNLIB_CXXFLAGS) -static-libstdc++ -Wall -o $@ $^
+	$(DYNLIB_CXXFLAGS) $(SYMVIS_FLAGS) -static-libstdc++ -Wall -o $@ $^
 ifdef DARWIN
 	install_name_tool -change \
 		`otool -L "$@" | grep "libluajit" | awk '{print $$1}'` \
@@ -194,7 +196,7 @@ $(OUTPUT_DIR)/libs/libkoreader-nnsvg.so: nnsvg.c \
 			$(if $(USE_LUAJIT_LIB),$(LUAJIT_LIB),) \
 			$(NANOSVG_HEADERS)
 	$(CC) -I$(NANOSVG_INCLUDE_DIR) \
-	$(DYNLIB_CFLAGS) -lm -fvisibility=hidden -Wall -o $@ nnsvg.c $(if $(USE_LUAJIT_LIB),$(LUAJIT_LIB),)
+	$(DYNLIB_CFLAGS) -Wall $(SYMVIS_FLAGS) -lm -o $@ nnsvg.c $(if $(USE_LUAJIT_LIB),$(LUAJIT_LIB),)
 ifdef DARWIN
 	install_name_tool -change \
 		`otool -L "$@" | grep "libluajit" | awk '{print $$1}'` \
@@ -203,11 +205,11 @@ ifdef DARWIN
 endif
 
 $(OUTPUT_DIR)/libs/libblitbuffer.so: blitbuffer.c
-	$(CC) $(DYNLIB_CFLAGS) $(VECTO_CFLAGS) -o $@ $^
+	$(CC) $(DYNLIB_CFLAGS) $(VECTO_CFLAGS) $(SYMVIS_FLAGS) -o $@ $^
 
 $(OUTPUT_DIR)/libs/libwrap-mupdf.so: wrap-mupdf.c \
 			$(MUPDF_LIB)
-	$(CC) -I$(MUPDF_DIR)/include $(DYNLIB_CFLAGS) -o $@ $^
+	$(CC) -I$(MUPDF_DIR)/include $(DYNLIB_CFLAGS) $(SYMVIS_FLAGS) -o $@ $^
 ifdef DARWIN
 	install_name_tool -id \
 		libs/libwrap-mupdf.so \
@@ -217,15 +219,6 @@ endif
 $(OUTPUT_DIR)/libs/libXss.so.1: libxss-dummy.c
 	$(CC) $(DYNLIB_CFLAGS) -o $@ $^
 
-ffi/mupdf_h.lua: ffi-cdecl/mupdf_decl.c $(MUPDF_DIR)/include
-	CPPFLAGS="$(CFLAGS) -I. -I$(MUPDF_DIR)/include" $(FFI_CDECL) gcc ffi-cdecl/mupdf_decl.c $@
-
-ffi/SDL2_0_h.lua: ffi-cdecl/SDL2_0_decl.c
-	CPPFLAGS="$(CFLAGS) -I. -LSDL2" $(FFI_CDECL) gcc ffi-cdecl/SDL2_0_decl.c $@
-
-ffi/lodepng_h.lua: ffi-cdecl/lodepng_decl.c $(LODEPNG_DIR)
-	CPPFLAGS="$(CFLAGS) -I. -I$(LODEPNG_DIR)" $(FFI_CDECL) gcc ffi-cdecl/lodepng_decl.c $@
-
 # include all third party libs
 include Makefile.third
 
@@ -274,6 +267,8 @@ distclean:
 	-rm -rf build
 	-rm -rf $(THIRDPARTY_DIR)/{$(CMAKE_THIRDPARTY_LIBS)}/build
 
+dist-clean: distclean
+
 # ===========================================================================
 # start of unit tests section
 
@@ -292,4 +287,4 @@ test: $(OUTPUT_DIR)/spec $(OUTPUT_DIR)/.busted
 		--exclude-tags=notest \
 		-o gtest ./spec/base/unit
 
-.PHONY: all android-toolchain clean distclean test
+.PHONY: all android-toolchain clean distclean dist-clean test

Makefile.defs

@@ -672,6 +672,12 @@ ZLIB_BUILD_DIR=$(THIRDPARTY_DIR)/zlib/build/$(MACHINE)
 ZLIB_DIR=$(CURDIR)/$(ZLIB_BUILD_DIR)/zlib-prefix/src/zlib
 ZLIB_STATIC=$(ZLIB_DIR)/libz.a
 
+ZSTD_BUILD_DIR=$(THIRDPARTY_DIR)/zstd/build/$(MACHINE)
+ZSTD_DIR=$(CURDIR)/$(ZSTD_BUILD_DIR)/zstd-prefix/src/zstd
+ZSTD_DESTDIR=$(CURDIR)/$(ZSTD_BUILD_DIR)/zstd-prefix/src/zstd-build
+ZSTD_LIB_EXT=$(if $(WIN32),.dll,$(if $(DARWIN),.1.dylib,.so.1))
+ZSTD_LIB=$(OUTPUT_DIR)/libs/libzstd$(ZSTD_LIB_EXT)
+
 MINIZIP_BUILD_DIR=$(THIRDPARTY_DIR)/minizip/build/$(MACHINE)
 MINIZIP_DIR=$(CURDIR)/$(MINIZIP_BUILD_DIR)/minizip-prefix/src/minizip
 AES_LIB_DIR=$(MINIZIP_DIR)/aes
@@ -941,13 +947,16 @@ else
 	DYNLIB_CMAKE_CXXFLAGS+=-Wl,-E -Wl,-rpath,'$(ORIGIN_CMAKE_TO_AUTOCFG)'
 endif
 
+# Used by our custom Lua modules, where we want to hide private symbols by default
+SYMVIS_FLAGS+=-fvisibility=hidden
+
 LPEG_BUILD_DIR=$(THIRDPARTY_DIR)/lpeg/build/$(MACHINE)
 LPEG_DIR=$(CURDIR)/$(LPEG_BUILD_DIR)/lpeg-prefix/src/lpeg
 LPEG_RE=$(OUTPUT_DIR)/rocks/share/lua/5.1/re.lua
 LPEG_DYNLIB=$(OUTPUT_DIR)/rocks/lib/lua/5.1/lpeg.so
 
 
-CMAKE_THIRDPARTY_LIBS := kpvcrlib,nanosvg
+CMAKE_THIRDPARTY_LIBS := kpvcrlib,nanosvg,zstd
 CMAKE_THIRDPARTY_LIBS := $(CMAKE_THIRDPARTY_LIBS),evernote-sdk-lua,luajit,lpeg,turbo
 CMAKE_THIRDPARTY_LIBS := $(CMAKE_THIRDPARTY_LIBS),zyre,czmq,filemq,libzmq
 CMAKE_THIRDPARTY_LIBS := $(CMAKE_THIRDPARTY_LIBS),libk2pdfopt,tesseract,leptonica

Makefile.third

@@ -176,7 +176,7 @@ ifdef DARWIN
 endif
 
 # crengine, fetched via GIT as a submodule
-$(CRENGINE_LIB): $(ZLIB) $(PNG_LIB) $(FREETYPE_LIB) $(HARFBUZZ_LIB) $(FRIBIDI_LIB) \
+$(CRENGINE_LIB): $(ZLIB) $(ZSTD_LIB) $(PNG_LIB) $(FREETYPE_LIB) $(HARFBUZZ_LIB) $(FRIBIDI_LIB) \
 		$(LIBUNIBREAK_LIB) $(UTF8PROC_LIB) $(JPEG_LIB) $(NANOSVG_HEADERS) \
 		$(CRENGINE_SRC_FILES) $(THIRDPARTY_DIR)/kpvcrlib/*.*
 	install -d $(CRENGINE_BUILD_DIR)
@@ -189,6 +189,7 @@ $(CRENGINE_LIB): $(ZLIB) $(PNG_LIB) $(FREETYPE_LIB) $(HARFBUZZ_LIB) $(FRIBIDI_LI
 		LIBUNIBREAK_LIB="$(CURDIR)/$(LIBUNIBREAK_LIB)" \
 		UTF8PROC_LIB="$(CURDIR)/$(UTF8PROC_LIB)" \
 		ZLIB="$(CURDIR)/$(ZLIB)" \
+		ZSTD_LIB="$(CURDIR)/$(ZSTD_LIB)" \
 		LIBS_DIR="$(CURDIR)/$(OUTPUT_DIR)/libs" \
 		$(CMAKE) $(CMAKE_FLAGS) \
 		-DJPEGLIB_INCLUDE_DIR=$(JPEG_DIR)/include \
@@ -201,6 +202,7 @@ $(CRENGINE_LIB): $(ZLIB) $(PNG_LIB) $(FREETYPE_LIB) $(HARFBUZZ_LIB) $(FRIBIDI_LI
 		-DFRIBIDI_INCLUDE_DIR="$(FRIBIDI_DIR)/include" \
 		-DLIBUNIBREAK_INCLUDE_DIR="$(LIBUNIBREAK_DIR)/include" \
 		-DUTF8PROC_INCLUDE_DIR="$(UTF8PROC_DIR)" \
+		-DZSTD_INCLUDE_DIR="$(ZSTD_DESTDIR)/include" \
 		-DCMAKE_BUILD_TYPE=$(CMAKE_BUILD_TYPE) \
 		$(if $(WIN32),-DCMAKE_SYSTEM_NAME=Windows,) \
 		$(if $(ANDROID),-DCMAKE_SYSTEM_NAME=Android,) \
@@ -394,6 +396,16 @@ else
 	cp -fL $(ZLIB_DIR)/lib/$(notdir $(ZLIB)) $(ZLIB)
 endif
 
+$(ZSTD_LIB): $(THIRDPARTY_DIR)/zstd/*.*
+	install -d $(ZSTD_BUILD_DIR)
+	cd $(ZSTD_BUILD_DIR) && \
+		$(CMAKE) $(CMAKE_FLAGS) -DCC="$(CC)" -DCXX="$(CXX)" -DAR="$(AR)" \
+		-DCFLAGS="$(CFLAGS)" \
+		-DLDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(ORIGIN_CMAKE_TO_AUTOCFG)/./libs'" \
+		$(CURDIR)/$(THIRDPARTY_DIR)/zstd && \
+		$(CMAKE_MAKE_PROGRAM) $(CMAKE_MAKE_PROGRAM_FLAGS)
+	cp -fL $(ZSTD_DIR)/lib/$(notdir $(ZSTD_LIB)) $@
+
 # ===========================================================================
 # console version of StarDict(sdcv)
 $(OUTPUT_DIR)/sdcv: $(if $(ANDROID),$(GLIB_STATIC),$(GLIB)) $(ZLIB_STATIC) $(THIRDPARTY_DIR)/sdcv/*.*
@@ -797,7 +809,7 @@ $(NANOSVG_HEADERS): $(THIRDPARTY_DIR)/nanosvg/*.*
 	install -d $(NANOSVG_INCLUDE_DIR)
 	cp -fL $(NANOSVG_DIR)/nanosvg/src/nanosvg.h $(NANOSVG_INCLUDE_DIR)/
 	cp -fL $(NANOSVG_DIR)/nanosvg/src/nanosvgrast.h $(NANOSVG_INCLUDE_DIR)/
-	cp -fL $(NANOSVG_DIR)/nanosvg/example/stb_image_write.h $(NANOSVG_INCLUDE_DIR)/
+	cp -fL $(THIRDPARTY_DIR)/nanosvg/stb_image_write.h $(NANOSVG_INCLUDE_DIR)/
 
 # cURL
 $(CURL_LIB): $(THIRDPARTY_DIR)/curl/*.*