Implementation of LZ4 and LZ4HC compression

Currently we support ZLIB based gzip compression and LZO2 compression.
LZ4 and LZ4HC (High Compression) are like LZO2 but are much faster on
decompression and are implemented as a single source file per compressor.

As we don't want to handle things like uncompressable data etc ourself
we settled for a wrapper library which mimics the zlib API and supports
the following compression algorithms:

- FASTLZ
- LZ4
- LZ4HC

The library is available at https://github.com/exalead/fastlzlib
and the few fixes needed to make it work with Bareos are currently
in a pullrequest to the original project.

For the period until the patches merge we have a clone under the bareos project
on github.

Fixes #33: Implementation of LZ4 and LZ4HC compression

Signed-off-by: Marco van Wieringen <marco.van.wieringen@bareos.com>

autoconf/configure.in

@@ -2651,14 +2651,17 @@ CPPFLAGS="${saved_CPPFLAGS} ${ZLIB_INC}"
 
 AC_CHECK_HEADERS(zlib.h)
 
+AC_MSG_CHECKING(for deflate in zlib library)
 AC_TRY_LINK(
    [
        #include <zlib.h>
    ], [
        deflate(Z_NULL, 0);
    ], [
+       AC_MSG_RESULT(yes)
        have_zlib="yes"
    ], [
+       AC_MSG_RESULT(no)
        have_zlib="no"
    ]
 )
@@ -2708,15 +2711,18 @@ CPPFLAGS="${saved_CPPFLAGS} ${LZO_INC}"
 AC_CHECK_HEADERS(lzo/lzoconf.h)
 AC_CHECK_HEADERS(lzo/lzo1x.h)
 
+AC_MSG_CHECKING(for lzo1x_1_compress in lzo library)
 AC_TRY_LINK(
    [
        #include <lzo/lzoconf.h>
        #include <lzo/lzo1x.h>
    ], [
        lzo1x_1_compress(NULL, 0, NULL, NULL, NULL);
    ], [
+       AC_MSG_RESULT(yes)
        have_lzo="yes"
    ], [
+       AC_MSG_RESULT(no)
        have_lzo="no"
    ]
 )
@@ -2735,6 +2741,65 @@ fi
 AC_SUBST(LZO_INC)
 AC_SUBST(LZO_LIBS)
 
+dnl
+dnl Check for fastlz
+dnl
+FASTLZ_LIBS="-lfastlz"
+FASTLZ_INC=""
+have_fastlz=no
+AC_ARG_WITH(fastlz,
+   AC_HELP_STRING([--with-fastlz@<:@=DIR@:>@], [Directory holding fastlz includes/libs]),
+   with_fastlz_directory=${withval}
+)
+
+if test "x${with_fastlz_directory}" != "xyes" && test x"${with_fastlz_directory}" != "x"; then
+   #
+   # Make sure the $with_fastlz_directory also makes sense
+   #
+   if test -d "${with_fastlz_directory}/lib" -a -d "${with_fastlz_directory}/include"; then
+      FASTLZ_LIBS="-L${with_fastlz_directory}/lib ${FASTLZ_LIBS}"
+      FASTLZ_INC="-I${with_fastlz_directory}/include ${FASTLZ_INC}"
+   fi
+fi
+
+saved_LIBS="${LIBS}"
+saved_CFLAGS="${CFLAGS}"
+saved_CPPFLAGS="${CPPFLAGS}"
+LIBS="${saved_LIBS} ${FASTLZ_LIBS}"
+CFLAGS="${saved_CFLAGS} ${FASTLZ_INC}"
+CPPFLAGS="${saved_CPPFLAGS} ${FASTLZ_INC}"
+
+AC_CHECK_HEADERS(fastlzlib.h)
+
+AC_MSG_CHECKING(for fastlzlibCompress in fastlz library)
+AC_TRY_LINK(
+   [
+       #include <fastlzlib.h>
+   ], [
+       fastlzlibCompress(Z_NULL, 0);
+   ], [
+       AC_MSG_RESULT(yes)
+       have_fastlz="yes"
+   ], [
+       AC_MSG_RESULT(no)
+       have_fastlz="no"
+   ]
+)
+
+LIBS="${saved_LIBS}"
+CFLAGS="${saved_CFLAGS}"
+CPPFLAGS="${saved_CPPFLAGS}"
+
+if test "x${have_fastlz}" = "xyes"; then
+   AC_DEFINE(HAVE_FASTLZ, 1, [Define to 1 if you have fastlz lib])
+else
+   FASTLZ_LIBS=""
+   FASTLZ_INC=""
+fi
+
+AC_SUBST(FASTLZ_INC)
+AC_SUBST(FASTLZ_LIBS)
+
 dnl
 dnl Check if we have AFS on this system
 dnl
@@ -3872,6 +3937,7 @@ Configuration on `date`:
    GNUTLS support:          ${have_gnutls}
    ZLIB support:            ${have_zlib}
    LZO support:             ${have_lzo}
+   FASTLZ support:          ${have_fastlz}
    NDMP support:            ${support_ndmp}
    enable-smartalloc:       ${support_smartalloc}
    enable-lockmgr:          ${support_lockmgr}

src/dird/inc_conf.c

@@ -233,6 +233,9 @@ static struct s_fs_opt FS_options[] = {
    { "gzip8", INC_KW_COMPRESSION, "Z8" },
    { "gzip9", INC_KW_COMPRESSION, "Z9" },
    { "lzo", INC_KW_COMPRESSION, "Zo" },
+   { "lzfast", INC_KW_COMPRESSION, "Zff" },
+   { "lz4", INC_KW_COMPRESSION, "Zf4" },
+   { "lz4hc", INC_KW_COMPRESSION, "Zfh" },
    { "blowfish", INC_KW_ENCRYPTION, "B"}, /* ***FIXME*** not implemented */
    { "3des", INC_KW_ENCRYPTION, "3"}, /* ***FIXME*** not implemented */
    { "yes", INC_KW_ONEFS, "0" },
@@ -289,12 +292,11 @@ static struct s_fs_opt FS_options[] = {
 static void scan_include_options(LEX *lc, int keyword, char *opts, int optlen)
 {
    int i;
-   char option[3];
+   char option[64];
    int lcopts = lc->options;
    struct s_sz_matching size_matching;
 
-   option[0] = 0;                     /* default option = none */
-   option[2] = 0;                     /* terminate options */
+   memset(option, 0, sizeof(option));
    lc->options |= LOPT_STRING;        /* force string */
    lex_get_token(lc, T_STRING);       /* expect at least one option */
    if (keyword == INC_KW_VERIFY) { /* special case */
@@ -335,11 +337,9 @@ static void scan_include_options(LEX *lc, int keyword, char *opts, int optlen)
       /*
        * Standard keyword options for Include/Exclude
        */
-      for (i=0; FS_options[i].name; i++) {
+      for (i = 0; FS_options[i].name; i++) {
          if (FS_options[i].keyword == keyword && bstrcasecmp(lc->str, FS_options[i].name)) {
-            /* NOTE! maximum 2 letters here or increase option[3] */
-            option[0] = FS_options[i].option[0];
-            option[1] = FS_options[i].option[1];
+            strncpy(option, FS_options[i].option, sizeof(option));
             i = 0;
             break;
          }

src/filed/Makefile.in

@@ -22,7 +22,7 @@ DEBUG=@DEBUG@
 
 GETTEXT_LIBS = @LIBINTL@
 
-CPPFLAGS += @ZLIB_INC@ @LZO_INC@
+COMPRESS_CPPFLAGS += @ZLIB_INC@ @LZO_INC@ @FASTLZ_INC@
 
 first_rule: all
 dummy:
@@ -37,6 +37,7 @@ SVROBJS = $(SVRSRCS:.c=.o)
 FDLIBS += @CAP_LIBS@
 FDLIBS += @ZLIB_LIBS@
 FDLIBS += @LZO_LIBS@
+FDLIBS += @FASTLZ_LIBS@
 FDLIBS += @AFS_LIBS_NONSHARED@
 FDLIBS += @ACL_LIBS_NONSHARED@
 FDLIBS += @XATTR_LIBS_NONSHARED@
@@ -52,6 +53,10 @@ INCLUDES += -I$(srcdir) -I$(basedir) -I$(basedir)/include
 	@echo "Compiling $<"
 	$(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(DINCLUDE) $(CXXFLAGS) $<
 #-------------------------------------------------------------------------
+compression.o: compression.c
+	@echo "Compiling $<"
+	$(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(COMPRESS_CPPFLAGS) $(INCLUDES) $(DINCLUDE) $(CXXFLAGS) $<
+
 all: Makefile bareos-fd @STATIC_FD@
 	@echo "==== Make of filed is good ===="
 	@echo " "

src/filed/backup.c

@@ -96,7 +96,9 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
 
    jcr->buf_size = sd->msglen;
 
-   adjust_compression_buffers(jcr);
+   if (!adjust_compression_buffers(jcr)) {
+      return false;
+   }
 
    if (!crypto_session_start(jcr)) {
       return false;
@@ -257,7 +259,7 @@ static inline bool save_rsrc_and_finder(b_save_ctx &bsctx)
  *
  * The signing digest is a single algorithm depending on
  * whether or not we have SHA2.
- *   ****FIXME****  the signing algoritm should really be
+ *   ****FIXME****  the signing algorithm should really be
  *   determined a different way!!!!!!  What happens if
  *   sha2 was available during backup but not restore?
  */