Build with system liblz4 if available, else fall back on built-in lz4

(cherry picked from commit 78b19fc)
confluentinc · Apr 12, 2017 · 3f68ba0 · 3f68ba0
1 parent 11888fc
commit 3f68ba0
Show file tree

Hide file tree

Showing 13 changed files with 32 additions and 49 deletions.
diff --git a/.doozer.json b/.doozer.json
@@ -8,8 +8,7 @@
         "python",
         "zlib1g-dev",
         "libssl-dev",
-        "libsasl2-dev",
-        "liblz4-dev"
+        "libsasl2-dev"
       ],
       "buildcmd": [
         "./configure",
@@ -22,7 +21,7 @@
     },
 
     "xenial-i386": {
-
+      "_comment": "including liblz4-dev here to verify that WITH_LZ4_EXT works",
       "buildenv": "xenial-i386",
       "builddeps": [
         "build-essential",
@@ -50,8 +49,7 @@
         "python",
         "zlib1g-dev",
         "libssl-dev",
-        "libsasl2-dev",
-        "liblz4-dev"
+        "libsasl2-dev"
       ],
       "buildcmd": [
         "./configure",
@@ -74,7 +72,6 @@
         "zlib1g-dev",
         "libssl-dev",
         "libsasl2-dev",
-        "liblz4-dev",
         "cmake"
       ],
       "buildcmd": [

diff --git a/.travis.yml b/.travis.yml
@@ -10,18 +10,13 @@ os:
 - osx
 dist: trusty
 sudo: false
-addons:
-  apt:
-    packages:
-    - liblz4-dev
 before_script:
  - ccache -s || echo "CCache is not available."
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; brew install lz4 ; fi
 script:
 - rm -rf artifacts dest
 - mkdir dest artifacts
 - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then CPPFLAGS="-I/usr/local/opt/openssl/include
-  -L/usr/local/opt/openssl/lib" STATIC_LIB_lz4=/usr/local/lib/liblz4.a ./configure --enable-static --prefix="$PWD/dest" ; else ./configure --enable-static --prefix="$PWD/dest" ; fi
+  -L/usr/local/opt/openssl/lib" ./configure --enable-static --prefix="$PWD/dest" ; else ./configure --enable-static --prefix="$PWD/dest" ; fi
 - make -j2 all examples check && make -C tests run_local
 - make install
 - (cd dest && tar cvzf ../artifacts/librdkafka-X.tar.gz .)

diff --git a/README.win32 b/README.win32
@@ -9,10 +9,7 @@ Requirements:
    https://slproweb.com/products/Win32OpenSSL.html
    (This would be using NuGet too but the current
     OpenSSL packages are outdated and with broken
-	dependencies, so no luck)
- * lz4 is installed automatically from NuGet.
-   xxhash.[ch] from lz4 are used instead of the ones provided by librdkafka.
-
+    dependencies, so no luck)
 
 The Visual Studio solution file for librdkafka resides in win32/librdkafka.sln
 

diff --git a/configure.librdkafka b/configure.librdkafka
@@ -30,7 +30,7 @@ mkl_toggle_option "Development" ENABLE_REFCNT_DEBUG "--enable-refcnt-debug" "Ena
 
 mkl_toggle_option "Development" ENABLE_SHAREDPTR_DEBUG "--enable-sharedptr-debug" "Enable sharedptr debugging" "n"
 
-mkl_toggle_option "Feature" ENABLE_LZ4 "--enable-lz4" "Enable LZ4 support" "y"
+mkl_toggle_option "Feature" ENABLE_LZ4_EXT "--enable-lz4" "Enable external LZ4 library support" "y"
 
 mkl_toggle_option "Feature" ENABLE_SSL "--enable-ssl" "Enable SSL support" "y"
 mkl_toggle_option "Feature" ENABLE_SASL "--enable-sasl" "Enable SASL support" "y"
@@ -47,8 +47,8 @@ function checks {
                   "#include <zlib.h>"
     mkl_lib_check "libcrypto" "" disable CC "-lcrypto"
 
-    if [[ "$ENABLE_LZ4" == "y" ]]; then
-        mkl_lib_check --static=-llz4 "liblz4" "WITH_LZ4" disable CC "-llz4" \
+    if [[ "$ENABLE_LZ4_EXT" == "y" ]]; then
+        mkl_lib_check --static=-llz4 "liblz4" "WITH_LZ4_EXT" disable CC "-llz4" \
                       "#include <lz4frame.h>"
     fi
 

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -34,6 +34,10 @@ set(
     rdstring.c
     snappy.c
     tinycthread.c
+    xxhash.c
+    lz4.c
+    lz4frame.c
+    lz4hc.c
 )
 
 if(WITH_SASL)

diff --git a/src/Makefile b/src/Makefile
@@ -12,7 +12,13 @@ endif
 SRCS_$(WITH_SASL) += rdkafka_sasl.c rdkafka_sasl_cyrus.c
 SRCS_$(WITH_SNAPPY) += snappy.c
 SRCS_$(WITH_ZLIB) += rdgz.c
-SRCS_$(WITH_LZ4) += xxhash.c
+
+SRCS_LZ4 = xxhash.c
+ifneq ($(WITH_LZ4_EXT), y)
+# Use built-in liblz4
+SRCS_LZ4 += lz4.c lz4frame.c lz4hc.c
+endif
+SRCS_y += $(SRCS_LZ4)
 
 ifeq ($(HAVE_REGEX), n)
 SRCS_y += regexp.c
@@ -53,6 +59,9 @@ install: lib-install
 
 clean: lib-clean
 
+# Compile LZ4 with -O3
+$(SRCS_LZ4:.c=.o): CFLAGS:=$(CFLAGS) -O3
+
 ifeq ($(WITH_LDS),y)
 # Enable linker script if supported by platform
 LIB_LDFLAGS+= $(LDFLAG_LINKERSCRIPT)$(LIBNAME).lds

diff --git a/src/rdkafka_broker.c b/src/rdkafka_broker.c
@@ -62,9 +62,14 @@
 #if WITH_SNAPPY
 #include "snappy.h"
 #endif
-#if WITH_LZ4
+#if WITH_LZ4_EXT
 #include <lz4frame.h>
+#else
+#include "lz4frame.h"
+#endif
 #include "xxhash.h"
+#if WITH_SSL
+#include <openssl/err.h>
 #endif
 #include "rdendian.h"
 
@@ -2072,7 +2077,6 @@ static void rd_kafka_produce_msgset_reply (rd_kafka_t *rk,
 
 
 
-#if WITH_LZ4
 /**
  * Fix-up bad LZ4 framing caused by buggy Kafka client / broker.
  * The LZ4F framing format is described in detail here:
@@ -2457,7 +2461,6 @@ rd_kafka_lz4_compress (rd_kafka_broker_t *rkb, int proper_hc,
         return err;
 
 }
-#endif
 
 
 /**
@@ -2476,9 +2479,7 @@ static int rd_kafka_compress_MessageSet_buf (rd_kafka_broker_t *rkb,
 	size_t coutlen = 0;
 	int    outlen;
 	int r;
-#if WITH_LZ4
 	rd_kafka_resp_err_t err;
-#endif
 #if WITH_SNAPPY
 	int    siovlen = 1;
 	struct snappy_env senv;
@@ -2621,7 +2622,6 @@ static int rd_kafka_compress_MessageSet_buf (rd_kafka_broker_t *rkb,
 		break;
 #endif
 
-#if WITH_LZ4
         case RD_KAFKA_COMPRESSION_LZ4:
 		/* Skip LZ4 compression if broker doesn't support it. */
 		if (!(rkb->rkb_features & RD_KAFKA_FEATURE_LZ4))
@@ -2638,7 +2638,6 @@ static int rd_kafka_compress_MessageSet_buf (rd_kafka_broker_t *rkb,
                 if (err)
                         return -1;
                 break;
-#endif
 
 
 	default:
@@ -3851,7 +3850,6 @@ rd_kafka_messageset_handle (rd_kafka_broker_t *rkb,
 		break;
 #endif
 
-#if WITH_LZ4
                 case RD_KAFKA_COMPRESSION_LZ4:
 		{
                         err = rd_kafka_lz4_decompress(rkb,
@@ -3865,7 +3863,6 @@ rd_kafka_messageset_handle (rd_kafka_broker_t *rkb,
                                 goto per_msg_err;
                 }
                 break;
-#endif
 
 
 		default:

diff --git a/src/rdkafka_conf.c b/src/rdkafka_conf.c
@@ -120,9 +120,7 @@ static const struct rd_kafka_property rd_kafka_properties[] = {
 		{ 0x8, "sasl" },
 #endif
 		{ 0x10, "regex" },
-#if WITH_LZ4
 		{ 0x20, "lz4" },
-#endif
 		{ 0, NULL }
 		}
 	},
@@ -633,9 +631,7 @@ static const struct rd_kafka_property rd_kafka_properties[] = {
 #if WITH_SNAPPY
 			{ RD_KAFKA_COMPRESSION_SNAPPY, "snappy" },
 #endif
-#if WITH_LZ4
                         { RD_KAFKA_COMPRESSION_LZ4, "lz4" },
-#endif
 			{ 0 }
 		} },
 	{ _RK_GLOBAL|_RK_PRODUCER, "batch.num.messages", _RK_C_INT,
@@ -715,10 +711,7 @@ static const struct rd_kafka_property rd_kafka_properties[] = {
 #if WITH_SNAPPY
 		  { RD_KAFKA_COMPRESSION_SNAPPY, "snappy" },
 #endif
-#if WITH_LZ4
 		  { RD_KAFKA_COMPRESSION_LZ4, "lz4" },
-#endif
-
 		  { RD_KAFKA_COMPRESSION_INHERIT, "inherit" },
 		  { 0 }
 		} },

diff --git a/src/win32_config.h b/src/win32_config.h
@@ -34,6 +34,5 @@
 #define WITH_SSL 1
 #define WITH_ZLIB 1
 #define WITH_SNAPPY 1
-#define WITH_LZ4 1
 #define WITH_SASL 1
 #define ENABLE_DEVEL 0
diff --git a/tests/0017-compression.c b/tests/0017-compression.c
@@ -52,9 +52,7 @@ int main_0017_compression(int argc, char **argv) {
 #if WITH_SNAPPY
                 "snappy",
 #endif
-#if WITH_LZ4
                 "lz4",
-#endif
                 NULL
         };
         const char *topics[CODEC_CNT];

diff --git a/win32/librdkafka.autopkg.template b/win32/librdkafka.autopkg.template
@@ -29,13 +29,6 @@ nuget {
         tags: { native, kafka, librdkafka, C, C++ };
  };
 
-
- dependencies {
-	packages: {
-		lz4/1.3.1.2
-	};
- };
-
  files {
 	#defines {
 	  TOPDIR = ..\;

diff --git a/win32/librdkafka.vcxproj b/win32/librdkafka.vcxproj
@@ -182,6 +182,10 @@
     <ClCompile Include="..\src\regexp.c" />
     <ClCompile Include="..\src\rdports.c" />
     <ClCompile Include="..\src\rdavl.c" />
+    <ClCompile Include="..\src\xxhash.c" />
+    <ClCompile Include="..\src\lz4.c" />
+    <ClCompile Include="..\src\lz4frame.c" />
+    <ClCompile Include="..\src\lz4hc.c" />
   </ItemGroup>
   <ItemGroup>
     <Text Include="..\LICENSE..txt" />
@@ -193,13 +197,11 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
     <Import Project="packages\zlib.$(PlatformToolset).windesktop.msvcstl.dyn.rt-dyn.1.2.8.8\build\native\zlib.$(PlatformToolset).windesktop.msvcstl.dyn.rt-dyn.targets" Condition="Exists('packages\zlib.$(PlatformToolset).windesktop.msvcstl.dyn.rt-dyn.1.2.8.8\build\native\zlib.$(PlatformToolset).windesktop.msvcstl.dyn.rt-dyn.targets')" />
-    <Import Project="packages\lz4.1.3.1.2\build\native\lz4.targets" Condition="Exists('packages\lz4.1.3.1.2\build\native\lz4.targets')" />
   </ImportGroup>
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
     <Error Condition="!Exists('packages\zlib.$(PlatformToolset).windesktop.msvcstl.dyn.rt-dyn.1.2.8.8\build\native\zlib.$(PlatformToolset).windesktop.msvcstl.dyn.rt-dyn.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\zlib.$(PlatformToolset).windesktop.msvcstl.dyn.rt-dyn.1.2.8.8\build\native\zlib.$(PlatformToolset).windesktop.msvcstl.dyn.rt-dyn.targets'))" />
-    <Error Condition="!Exists('packages\lz4.1.3.1.2\build\native\lz4.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\lz4.1.3.1.2\build\native\lz4.targets'))" />
   </Target>
 </Project>
diff --git a/win32/packages.config b/win32/packages.config
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="lz4" version="1.3.1.2" targetFramework="native" />
   <package id="zlib" version="1.2.8.8" targetFramework="Native" />
   <package id="zlib.v120.windesktop.msvcstl.dyn.rt-dyn" version="1.2.8.8" targetFramework="Native" />
   <package id="zlib.v140.windesktop.msvcstl.dyn.rt-dyn" version="1.2.8.8" targetFramework="Native" />