Initial smallz4 support, related to radareorg#19849 ##build

* Patch smallz4 to return -1 on error (as opposed to just exit)
* Add new function 'unlz4Block_userPtr' that can decompress an lz4 block
* New '--with-smallz4' flag to compile and use smallz4 instead of lz4

config-user.mk.acr

@@ -101,6 +101,7 @@ LIBZIP=@LIBZIP@
 LIBXXHASH=@LIBXXHASH@
 
 USE_SYSLZ4=@USE_SYSLZ4@
+USE_SMALLZ4=@USE_SMALLZ4@
 
 ifeq ($(HAVE_LIB_SSL),1)
 SSL_CFLAGS=@SSL_CFLAGS@

configure

@@ -34,6 +34,7 @@ USE_CS5=0
 USE_CS4=0
 WITH_CAPSTONE=0
 WITH_SYSLZ4=0
+WITH_SMALLZ4=0
 USE_ZIP=0
 USE_XXHASH=0
 WITH_GPL=1
@@ -184,8 +185,7 @@ System types:
   --target=TARGET        configure for building compilers for TARGET [HOST]
 EOF2
 
-printf "
-Optional Features:
+printf "\nOptional Features:
   --disable-debugger     disable native debugger features
   --with-sysmagic        force to use system's magic
   --disable-threads      disable use of thread apis
@@ -202,6 +202,7 @@ Optional Features:
   --with-capstone4       build v4 branch of capstone
   --with-syscapstone     force to use system-wide capstone
   --with-syslz4          force to use system's liblz4
+  --with-smallz4         build with smallz4 use it as the lz4 decompressor
   --with-syszip          force to use system's libzip and zlib
   --with-sysxxhash       force to use system's xxhash
   --without-gpl          do not build GPL code (grub, cxx, ... )
@@ -213,20 +214,16 @@ Optional Features:
   --with-ostype          Choose OS ( android windows wsl mingw32 bsd solaris gnulinux darwin haiku ) (USEROSTYPE=auto)
   --with-libversion      specify different libversion (LIBVERSION=xxx)
   --without-jemalloc     build without jemalloc
-  --with-checks-level    value between 0 and 3 to enable different level of assert (see R_CHECKS_LEVEL) (R_CHECKS_LEVEL=2)
-"
-printf "
-Some influential environment variables:
+  --with-checks-level    value between 0 and 3 to enable different level of assert (see R_CHECKS_LEVEL) (R_CHECKS_LEVEL=2)\n"
+printf "\nSome influential environment variables:
   CC          C compiler command
   CFLAGS      C compiler flags
   LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
               nonstandard directory <lib dir>
   CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
               headers in a nonstandard directory <include dir>
-  CPP         C preprocessor
-"
-printf "
-Report bugs to: pancake <pancake@nopcode.org>"
+  CPP         C preprocessor\n"
+printf "\nReport bugs to: pancake <pancake@nopcode.org>"
 echo ""
 exit 0
 }
@@ -287,7 +284,7 @@ echo "LANGS:     c"
 echo "REQUIRED:  libdl"
 echo "OPTIONAL:  libmagic libz libzip libxxhash libssl liblibuv>=1.0.0"
 echo "PKG-CONFIG:  capstone liblz4 openssl libuv"
-echo "FLAGS:     --disable-debugger --with-sysmagic --disable-threads --disable-loadlibs --enable-threadsafety --without-dylink --without-fork --without-ptrace-wrap --without-gperf --without-capstone --with-libr --with-static-themes --with-capstone5 --with-capstone4 --with-syscapstone --with-syslz4 --with-syszip --with-sysxxhash --without-gpl --with-ssl --with-ssl-crypto --with-libuv --with-rpath --with-compiler=gcc --with-ostype=auto --with-libversion=xxx --without-jemalloc --with-checks-level=2"
+echo "FLAGS:     --disable-debugger --with-sysmagic --disable-threads --disable-loadlibs --enable-threadsafety --without-dylink --without-fork --without-ptrace-wrap --without-gperf --without-capstone --with-libr --with-static-themes --with-capstone5 --with-capstone4 --with-syscapstone --with-syslz4 --with-smallz4 --with-syszip --with-sysxxhash --without-gpl --with-ssl --with-ssl-crypto --with-libuv --with-rpath --with-compiler=gcc --with-ostype=auto --with-libversion=xxx --without-jemalloc --with-checks-level=2"
 	exit 0
 	;;
 --cache-file)
@@ -353,6 +350,7 @@ echo "FLAGS:     --disable-debugger --with-sysmagic --disable-threads --disable-
 "--with-capstone4") USE_CS4="1"; ;;
 "--with-syscapstone") WITH_CAPSTONE="1"; ;;
 "--with-syslz4") WITH_SYSLZ4="1"; ;;
+"--with-smallz4") WITH_SMALLZ4="1"; ;;
 "--with-syszip") USE_ZIP="1"; ;;
 "--with-sysxxhash") USE_XXHASH="1"; ;;
 "--without-gpl") WITH_GPL="0"; ;;
@@ -382,7 +380,7 @@ parse_options "$1"
 shift
 done
 
-ENVWORDS="MANDIR INFODIR LIBDIR INCLUDEDIR LOCALSTATEDIR ETCDIR SYSCONFDIR DATADIR DOCDIR LIBEXECDIR SBINDIR BINDIR EPREFIX PREFIX SPREFIX TARGET HOST BUILD INSTALL INSTALL_LIB INSTALL_MAN INSTALL_PROGRAM INSTALL_PROGRAM_STRIP INSTALL_DIR INSTALL_SCRIPT INSTALL_DATA HOST_OS HOST_CPU BUILD_OS BUILD_CPU TARGET_OS TARGET_CPU VERSION VERSION_MAJOR VERSION_MINOR VERSION_PATCH VERSION_NUMBER PKGNAME VPATH CONTACT CONTACT_NAME CONTACT_MAIL CC CFLAGS CPPFLAGS LDFLAGS HAVE_LANG_C DEBUGGER HAVE_LIB_DL DL_LIBS PKGCONFIG HAVE_PATCH PATCH HAVE_AR AR HAVE_GIT GIT HAVE_GPERF GPERF HAVE_LIB_MAGIC HAVE_LINUX_CAN_H USE_MAGIC USE_LIB_MAGIC LIBMAGIC WANT_THREADS LOADLIBS R_CRITICAL_ENABLED WANT_DYLINK HAVE_FORK WANT_PTRACE_WRAP WANT_GPERF WANT_CAPSTONE WITH_LIBR WITH_STATIC_THEMES USE_CS5 USE_CS4 WITH_CAPSTONE CAPSTONE_CFLAGS CAPSTONE_LDFLAGS HAVE_PKGCFG_CAPSTONE USE_CAPSTONE LZ4_CFLAGS LZ4_LDFLAGS HAVE_PKGCFG_LIBLZ4 WITH_SYSLZ4 USE_SYSLZ4 HAVE_LIB_Z HAVE_LIB_ZIP USE_ZIP USE_LIB_ZIP LIBZIP HAVE_LIB_XXHASH USE_XXHASH USE_LIB_XXHASH LIBXXHASH WITH_GPL HAVE_DECL_ADDR_NO_RANDOMIZE HAVE_DECL___GLIBC__ HAVE_ARC4RANDOM_UNIFORM HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET HAVE_CLOCK_NANOSLEEP HAVE_SIGACTION HAVE_CLOCK_GETTIME CLOCK_LDFLAGS SUPPORT_GNU99 HAVE_LIB_GMP HAVE_LIB_SSL SSL_CFLAGS SSL_LDFLAGS HAVE_PKGCFG_OPENSSL HAVE_OPENSSL WANT_SSL WANT_SSL_CRYPTO WANT_LIBUV HAVE_LIBUV_VERSION_1_0_0 LIBUV_CFLAGS LIBUV_LDFLAGS HAVE_PKGCFG_LIBUV HAVE_LIBUV USE_RPATH USERCC USEROSTYPE LIBVERSION HAVE_JEMALLOC HAVE_PTRACE USE_PTRACE_WRAP R_CHECKS_LEVEL"
+ENVWORDS="MANDIR INFODIR LIBDIR INCLUDEDIR LOCALSTATEDIR ETCDIR SYSCONFDIR DATADIR DOCDIR LIBEXECDIR SBINDIR BINDIR EPREFIX PREFIX SPREFIX TARGET HOST BUILD INSTALL INSTALL_LIB INSTALL_MAN INSTALL_PROGRAM INSTALL_PROGRAM_STRIP INSTALL_DIR INSTALL_SCRIPT INSTALL_DATA HOST_OS HOST_CPU BUILD_OS BUILD_CPU TARGET_OS TARGET_CPU VERSION VERSION_MAJOR VERSION_MINOR VERSION_PATCH VERSION_NUMBER PKGNAME VPATH CONTACT CONTACT_NAME CONTACT_MAIL CC CFLAGS CPPFLAGS LDFLAGS HAVE_LANG_C DEBUGGER HAVE_LIB_DL DL_LIBS PKGCONFIG HAVE_PATCH PATCH HAVE_AR AR HAVE_GIT GIT HAVE_GPERF GPERF HAVE_LIB_MAGIC HAVE_LINUX_CAN_H USE_MAGIC USE_LIB_MAGIC LIBMAGIC WANT_THREADS LOADLIBS R_CRITICAL_ENABLED WANT_DYLINK HAVE_FORK WANT_PTRACE_WRAP WANT_GPERF WANT_CAPSTONE WITH_LIBR WITH_STATIC_THEMES USE_CS5 USE_CS4 WITH_CAPSTONE CAPSTONE_CFLAGS CAPSTONE_LDFLAGS HAVE_PKGCFG_CAPSTONE USE_CAPSTONE LZ4_CFLAGS LZ4_LDFLAGS HAVE_PKGCFG_LIBLZ4 WITH_SYSLZ4 USE_SYSLZ4 WITH_SMALLZ4 USE_SMALLZ4 HAVE_LIB_Z HAVE_LIB_ZIP USE_ZIP USE_LIB_ZIP LIBZIP HAVE_LIB_XXHASH USE_XXHASH USE_LIB_XXHASH LIBXXHASH WITH_GPL HAVE_DECL_ADDR_NO_RANDOMIZE HAVE_DECL___GLIBC__ HAVE_ARC4RANDOM_UNIFORM HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET HAVE_CLOCK_NANOSLEEP HAVE_SIGACTION HAVE_CLOCK_GETTIME CLOCK_LDFLAGS SUPPORT_GNU99 HAVE_LIB_GMP HAVE_LIB_SSL SSL_CFLAGS SSL_LDFLAGS HAVE_PKGCFG_OPENSSL HAVE_OPENSSL WANT_SSL WANT_SSL_CRYPTO WANT_LIBUV HAVE_LIBUV_VERSION_1_0_0 LIBUV_CFLAGS LIBUV_LDFLAGS HAVE_PKGCFG_LIBUV HAVE_LIBUV USE_RPATH USERCC USEROSTYPE LIBVERSION HAVE_JEMALLOC HAVE_PTRACE USE_PTRACE_WRAP R_CHECKS_LEVEL"
 
 create_environ
 
@@ -657,6 +655,10 @@ if [ 11 = "$WITH_SYSLZ4$LZ4_CFLAGS" ]; then
 USE_SYSLZ4="1"
 else
 USE_SYSLZ4="0"; fi
+if [ "$WITH_SMALLZ4" = "1" ]; then
+USE_SMALLZ4="1"
+else
+USE_SMALLZ4="0"; fi
 check_library HAVE_LIB_Z z 0
 check_library HAVE_LIB_ZIP zip 0
 if [ 11 = "$HAVE_LIB_Z$USE_ZIP" ]; then
@@ -929,7 +931,7 @@ do_remove
 if [ "$QUIET" = 0 ]; then
 echo
 echo "Final report:"
-for A in  BUILD CC CFLAGS DEBUGGER HAVE_ARC4RANDOM_UNIFORM HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET HAVE_FORK HAVE_GPERF HAVE_LIBUV HAVE_LIB_GMP HAVE_OPENSSL WANT_SSL_CRYPTO HAVE_PTRACE HOST LDFLAGS LIBVERSION PKGCONFIG PREFIX R_CHECKS_LEVEL TARGET USERCC USEROSTYPE USE_CAPSTONE USE_LIB_MAGIC USE_LIB_XXHASH USE_LIB_ZIP USE_PTRACE_WRAP USE_SYSLZ4 VERSION WANT_DYLINK ; do
+for A in  BUILD CC CFLAGS DEBUGGER HAVE_ARC4RANDOM_UNIFORM HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET HAVE_FORK HAVE_GPERF HAVE_LIBUV HAVE_LIB_GMP HAVE_OPENSSL WANT_SSL_CRYPTO HAVE_PTRACE HOST LDFLAGS LIBVERSION PKGCONFIG PREFIX R_CHECKS_LEVEL TARGET USERCC USEROSTYPE USE_CAPSTONE USE_LIB_MAGIC USE_LIB_XXHASH USE_LIB_ZIP USE_PTRACE_WRAP USE_SYSLZ4 VERSION WANT_DYLINK USE_SMALLZ4 ; do
 eval VAL="\$${A}"
 [ -z "${VAL}" ] && VAL="\"\""
 echo " - ${A} = ${VAL}"

configure.acr

@@ -69,6 +69,14 @@ IFAND WITH_SYSLZ4 LZ4_CFLAGS {
 
 }
 
+(( SMALLZ4 ))
+ARG_WITH WITH_SMALLZ4 smallz4 build with smallz4 use it as the lz4 decompressor ;
+IF WITH_SMALLZ4 {
+	USE_SMALLZ4 = 1 ;
+}{
+	USE_SMALLZ4 = 0 ;
+}
+
 (( ZIP ))
 CHKLIB z
 CHKLIB zip
@@ -279,7 +287,7 @@ REPORT
 	HAVE_EXPLICIT_MEMSET HAVE_FORK HAVE_GPERF HAVE_LIBUV HAVE_LIB_GMP
 	HAVE_OPENSSL WANT_SSL_CRYPTO HAVE_PTRACE HOST LDFLAGS LIBVERSION PKGCONFIG PREFIX
 	R_CHECKS_LEVEL TARGET USERCC USEROSTYPE USE_CAPSTONE USE_LIB_MAGIC
-	USE_LIB_XXHASH USE_LIB_ZIP USE_PTRACE_WRAP USE_SYSLZ4 VERSION WANT_DYLINK
+	USE_LIB_XXHASH USE_LIB_ZIP USE_PTRACE_WRAP USE_SYSLZ4 VERSION WANT_DYLINK USE_SMALLZ4
 ;
 
 (( TODO: Add the rest of .pc files here.. add a rule for acr? ))

libr/util/Makefile

@@ -37,6 +37,11 @@ OBJS+=big_gmp.o
 OBJS+=big_ssl.o
 OBJS+=big.o
 
+ifeq ($(USE_SMALLZ4),1)
+CFLAGS+=-DUSE_SMALLZ4
+OBJS+=$(SHLR)/smallz4/smallz4cat.o
+endif
+
 CWD=$(shell pwd)
 
 LDFLAGS+=${BN_LIBS}

libr/util/zip.c

@@ -7,6 +7,36 @@
 // set a maximum output buffer of 50MB
 #define MAXOUT 50000000
 
+#if USE_SMALLZ4
+#include "../../../shlr/smallz4/smallz4cat.h"
+
+struct UserPtr {
+	const ut8 * input;
+	ut64 inputPos;
+	ut8 * output;
+	ut64 outputPos;
+	ut32 * outputSize;
+	int error;
+};
+
+void smallz4Write(const unsigned char* data, unsigned int numBytes, void *userPtr) {
+  struct UserPtr* user = (struct UserPtr*)userPtr;
+  if (data != NULL && numBytes > 0) {
+	if (*(user->outputSize) - user->outputPos < numBytes) {
+		user->error = -1;
+		return;
+	}
+	memcpy(user->output + user->outputPos, data, numBytes);
+	user->outputPos += numBytes;
+  }
+}
+
+unsigned char smallz4GetByte(void *userPtr) {
+  struct UserPtr* user = (struct UserPtr*)userPtr;
+  return *(user->input + (user->inputPos++));
+}
+#endif
+
 static const char *gzerr(int n) {
 	const char * const errors[] = {
 		"",
@@ -83,12 +113,63 @@ static ut8 *r_inflatew(const ut8 *src, int srcLen, int *consumed, int *dstLen, i
 	return NULL;
 }
 
+#if USE_SMALLZ4
 R_API ut8 *r_inflate_lz4(const ut8 *src, int srcLen, int *consumed, int *dstLen) {
 	ut32 osz = srcLen * 5;
 	ut8 *obuf = calloc (srcLen, 5);
 	if (!obuf) {
 		return NULL;
 	}
+
+	struct UserPtr user =
+	{
+		.input = src,
+		.inputPos = 0,
+		.output = obuf,
+		.outputPos = 0,
+		.outputSize = &osz,
+		.error = 0
+	};
+	int res = unlz4Block_userPtr(smallz4GetByte, smallz4Write, &user, srcLen);
+	if (res < 1 || user.error != 0) {
+		int mul = srcLen / -res;
+		int nosz = osz * (5 * (mul + 1));
+		if (nosz < osz) {
+			free (obuf);
+			return NULL;
+		}
+		ut8 *nbuf = realloc (obuf, nosz);
+		if (!nbuf) {
+			free (obuf);
+			return NULL;
+		}
+		user.output = obuf = nbuf;
+		osz = nosz;
+		user.inputPos = 0;
+		user.outputPos = 0;
+		user.error = 0;
+		res = unlz4Block_userPtr (smallz4GetByte, smallz4Write, &user, srcLen);
+	}
+	user.output = NULL;
+	user.input = NULL;
+	if (res > 0) {
+		*dstLen = res;
+		*consumed = srcLen;
+		return obuf;
+	}
+	*dstLen = 0;
+	*consumed = 0;
+	free (obuf);
+	return NULL;
+}
+#else
+R_API ut8 *r_inflate_lz4(const ut8 *src, int srcLen, int *consumed, int *dstLen) {
+	ut32 osz = srcLen * 5;
+	ut8 *obuf = calloc (srcLen, 5);
+	if (!obuf) {
+		return NULL;
+	}
+
 	int res = LZ4_decompress_safe ((const char*)src, (char*)obuf, (uint32_t) srcLen, (uint32_t) osz);
 	if (res < 1) {
 		int mul = srcLen / -res;
@@ -116,6 +197,7 @@ R_API ut8 *r_inflate_lz4(const ut8 *src, int srcLen, int *consumed, int *dstLen)
 	free (obuf);
 	return NULL;
 }
+#endif
 
 R_API ut8 *r_inflate(const ut8 *src, int srcLen, int *consumed, int *dstLen) {
 	return r_inflatew (src, srcLen, consumed, dstLen, MAX_WBITS + 32);

shlr/smallz4/Makefile

@@ -0,0 +1,17 @@
+CFLAGS = -O2 -Wall -pedantic -s -std=c99
+
+smallz4cat.o:
+	$(CC) $(CCFLAGS) $(EXTRAFLAGS) -c smallz4cat.c
+
+all32: CFLAGS+=-m32
+
+sync:
+	rm -f smallz4cat.c
+	# Github repo is more up to date despite being a mirror
+	wget https://raw.githubusercontent.com/stbrumme/smallz4/master/smallz4cat.c
+	patch -p1 <unlz4block.patch
+
+.PHONY: clean
+clean:
+	@$(RM) *.o
+