Update brotli, lz4 and zstd libraries

inikep · Jan 3, 2018 · 5b44393 · 5b44393
1 parent df29c60
commit 5b44393
Show file tree

Hide file tree

Showing 142 changed files with 9,270 additions and 6,682 deletions.
diff --git a/Makefile b/Makefile
@@ -57,7 +57,7 @@ else
 endif
 
 
-DEFINES     += $(addprefix -I$(SOURCE_PATH),. zstd/lib zstd/lib/common xpack/common libcsc)
+DEFINES     += $(addprefix -I$(SOURCE_PATH),. zstd/lib zstd/lib/common brotli/include xpack/common libcsc)
 DEFINES     += -DHAVE_CONFIG_H
 CODE_FLAGS  += -Wno-unknown-pragmas -Wno-sign-compare -Wno-conversion
 OPT_FLAGS   ?= -fomit-frame-pointer -fstrict-aliasing -ffast-math
@@ -119,7 +119,8 @@ BROTLI_FILES += brotli/enc/cluster.o brotli/enc/bit_cost.o brotli/enc/backward_r
 
 ZSTD_FILES = zstd/lib/decompress/zstd_decompress.o zstd/lib/decompress/huf_decompress.o zstd/lib/common/zstd_common.o zstd/lib/common/fse_decompress.o 
 ZSTD_FILES += zstd/lib/common/xxhash.o zstd/lib/common/error_private.o zstd/lib/common/entropy_common.o zstd/lib/common/pool.o 
-ZSTD_FILES += zstd/lib/compress/zstd_compress.o zstd/lib/compress/zstdmt_compress.o zstd/lib/compress/fse_compress.o zstd/lib/compress/huf_compress.o 
+ZSTD_FILES += zstd/lib/compress/zstd_compress.o zstd/lib/compress/zstdmt_compress.o zstd/lib/compress/zstd_double_fast.o zstd/lib/compress/zstd_fast.o
+ZSTD_FILES += zstd/lib/compress/zstd_lazy.o zstd/lib/compress/zstd_ldm.o zstd/lib/compress/zstd_opt.o zstd/lib/compress/fse_compress.o zstd/lib/compress/huf_compress.o 
 
 BRIEFLZ_FILES = brieflz/brieflz.o brieflz/depacks.o 
 

diff --git a/NEWS b/NEWS
@@ -1,3 +1,9 @@
+v1.7.3
+- brotli updated to 2017-12-12
+- lz4 updated to v1.8.0
+- zstd updated to v1.3.3
+- added zstd compressor with long distance matcher
+
 v1.7.2
 - zstd updated to v1.3.1
 

diff --git a/_lzbench/compressors.cpp b/_lzbench/compressors.cpp
@@ -1768,6 +1768,7 @@ int64_t lzbench_zstd_compress(char *inbuf, size_t insize, char *outbuf, size_t o
 #if 1
     zstd_params->zparams = ZSTD_getParams(level, insize, 0);
     zstd_params->zparams.fParams.contentSizeFlag = 1;
+
     if (windowLog && zstd_params->zparams.cParams.windowLog > windowLog) {
         zstd_params->zparams.cParams.windowLog = windowLog;
         zstd_params->zparams.cParams.chainLog = windowLog + ((zstd_params->zparams.cParams.strategy == ZSTD_btlazy2) || (zstd_params->zparams.cParams.strategy == ZSTD_btopt) || (zstd_params->zparams.cParams.strategy == ZSTD_btultra));
@@ -1791,11 +1792,24 @@ int64_t lzbench_zstd_decompress(char *inbuf, size_t insize, char *outbuf, size_t
     return ZSTD_decompressDCtx(zstd_params->dctx, outbuf, outsize, inbuf, insize);
 }
 
+char* lzbench_zstd_LDM_init(size_t insize, size_t level, size_t windowLog)
+{
+    zstd_params_s* zstd_params = (zstd_params_s*) lzbench_zstd_init(insize, level, windowLog);
+    if (!zstd_params) return NULL;
+    ZSTD_CCtx_setParameter(zstd_params->cctx, ZSTD_p_enableLongDistanceMatching, 1);
+    return (char*) zstd_params;
+}
 
+int64_t lzbench_zstd_LDM_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t windowLog, char* workmem)
+{
+    zstd_params_s* zstd_params = (zstd_params_s*) workmem;
+    if (!zstd_params || !zstd_params->cctx) return 0;
+    ZSTD_CCtx_setParameter(zstd_params->cctx, ZSTD_p_enableLongDistanceMatching, 1);
+    return lzbench_zstd_compress(inbuf, insize, outbuf, outsize, level, windowLog, (char*) zstd_params);
+}
 #endif
 
 
-
 #ifdef BENCH_HAS_NAKAMICHI
 #include "nakamichi/nakamichi.h"
 

diff --git a/_lzbench/compressors.h b/_lzbench/compressors.h
@@ -458,15 +458,19 @@ int64_t lzbench_return_0(char *inbuf, size_t insize, char *outbuf, size_t outsiz
 
 
 #ifndef BENCH_REMOVE_ZSTD
-    char* lzbench_zstd_init(size_t insize, size_t level, size_t);
-    void lzbench_zstd_deinit(char* workmem);
+	char* lzbench_zstd_init(size_t insize, size_t level, size_t);
+	void lzbench_zstd_deinit(char* workmem);
 	int64_t lzbench_zstd_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*);
 	int64_t lzbench_zstd_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*);
+	char* lzbench_zstd_LDM_init(size_t insize, size_t level, size_t);
+	int64_t lzbench_zstd_LDM_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*);
 #else
 	#define lzbench_zstd_init NULL
 	#define lzbench_zstd_deinit NULL
 	#define lzbench_zstd_compress NULL
 	#define lzbench_zstd_decompress NULL
+	#define lzbench_zstd_LDM_init NULL
+	#define lzbench_zstd_LDM_compress NULL
 #endif
 
 

diff --git a/_lzbench/lzbench.h b/_lzbench/lzbench.h
@@ -10,7 +10,7 @@
 #include "lizard/lizard_compress.h"    // LIZARD_MAX_CLEVEL
 
 #define PROGNAME "lzbench"
-#define PROGVERSION "1.7.2"
+#define PROGVERSION "1.7.3"
 #define PAD_SIZE (16*1024)
 #define MIN_PAGE_SIZE 4096  // smallest page size we expect, if it's wrong the first algorithm might be a bit slower
 #define DEFAULT_LOOP_TIME (100*1000000)  // 1/10 of a second
@@ -137,26 +137,26 @@ typedef struct
 
 
 
-#define LZBENCH_COMPRESSOR_COUNT 65
+#define LZBENCH_COMPRESSOR_COUNT 68
 
 static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] =
 {
     { "memcpy",     "",            0,   0,    0,       0, lzbench_return_0,            lzbench_memcpy,                NULL,                    NULL },
     { "blosclz",    "2015-11-10",  1,   9,    0, 64*1024, lzbench_blosclz_compress,    lzbench_blosclz_decompress,    NULL,                    NULL },
     { "brieflz",    "1.1.0",       0,   0,    0,       0, lzbench_brieflz_compress,    lzbench_brieflz_decompress,    lzbench_brieflz_init,    lzbench_brieflz_deinit },
-    { "brotli",     "2017-03-10",  0,  11,    0,       0, lzbench_brotli_compress,     lzbench_brotli_decompress,     NULL,                    NULL },
-    { "brotli22",   "2017-03-10",  0,  11,   22,       0, lzbench_brotli_compress,     lzbench_brotli_decompress,     NULL,                    NULL },
-    { "brotli24",   "2017-03-10",  0,  11,   24,       0, lzbench_brotli_compress,     lzbench_brotli_decompress,     NULL,                    NULL },
+    { "brotli",     "2017-12-12",  0,  11,    0,       0, lzbench_brotli_compress,     lzbench_brotli_decompress,     NULL,                    NULL },
+    { "brotli22",   "2017-12-12",  0,  11,   22,       0, lzbench_brotli_compress,     lzbench_brotli_decompress,     NULL,                    NULL },
+    { "brotli24",   "2017-12-12",  0,  11,   24,       0, lzbench_brotli_compress,     lzbench_brotli_decompress,     NULL,                    NULL },
     { "crush",      "1.0",         0,   2,    0,       0, lzbench_crush_compress,      lzbench_crush_decompress,      NULL,                    NULL },
     { "csc",        "2016-10-13",  1,   5,    0,       0, lzbench_csc_compress,        lzbench_csc_decompress,        NULL,                    NULL },
     { "density",    "0.12.5 beta", 1,   3,    0,       0, lzbench_density_compress,    lzbench_density_decompress,    NULL,                    NULL }, // decompression error (shortened output)
     { "fastlz",     "0.1",         1,   2,    0,       0, lzbench_fastlz_compress,     lzbench_fastlz_decompress,     NULL,                    NULL },
     { "gipfeli",    "2016-07-13",  0,   0,    0,       0, lzbench_gipfeli_compress,    lzbench_gipfeli_decompress,    NULL,                    NULL },
     { "glza",       "0.8",         0,   0,    0,       0, lzbench_glza_compress,       lzbench_glza_decompress,       NULL,                    NULL },
     { "libdeflate", "0.7",         1,  12,    0,       0, lzbench_libdeflate_compress, lzbench_libdeflate_decompress, NULL,                    NULL },
-    { "lz4",        "1.7.5",       0,   0,    0,       0, lzbench_lz4_compress,        lzbench_lz4_decompress,        NULL,                    NULL },
-    { "lz4fast",    "1.7.5",       1,  99,    0,       0, lzbench_lz4fast_compress,    lzbench_lz4_decompress,        NULL,                    NULL },
-    { "lz4hc",      "1.7.5",       1,  12,    0,       0, lzbench_lz4hc_compress,      lzbench_lz4_decompress,        NULL,                    NULL },
+    { "lz4",        "1.8.0",       0,   0,    0,       0, lzbench_lz4_compress,        lzbench_lz4_decompress,        NULL,                    NULL },
+    { "lz4fast",    "1.8.0",       1,  99,    0,       0, lzbench_lz4fast_compress,    lzbench_lz4_decompress,        NULL,                    NULL },
+    { "lz4hc",      "1.8.0",       1,  12,    0,       0, lzbench_lz4hc_compress,      lzbench_lz4_decompress,        NULL,                    NULL },
     { "lizard",     "1.0",  LIZARD_MIN_CLEVEL, LIZARD_MAX_CLEVEL, 0, 0, lzbench_lizard_compress,      lzbench_lizard_decompress,        NULL,                    NULL },
     { "lzf",        "3.6",         0,   1,    0,       0, lzbench_lzf_compress,        lzbench_lzf_decompress,        NULL,                    NULL },
     { "lzfse",      "2017-03-08",  0,   0,    0,       0, lzbench_lzfse_compress,      lzbench_lzfse_decompress,      lzbench_lzfse_init,      lzbench_lzfse_deinit },
@@ -202,9 +202,12 @@ static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] =
     { "yappy",      "2014-03-22",  0,  99,    0,       0, lzbench_yappy_compress,      lzbench_yappy_decompress,      lzbench_yappy_init,      NULL },
     { "zlib",       "1.2.11",      1,   9,    0,       0, lzbench_zlib_compress,       lzbench_zlib_decompress,       NULL,                    NULL },
     { "zling",      "2016-01-10",  0,   4,    0,       0, lzbench_zling_compress,      lzbench_zling_decompress,      NULL,                    NULL },
-    { "zstd",       "1.3.1",       1,  22,    0,       0, lzbench_zstd_compress,       lzbench_zstd_decompress,       lzbench_zstd_init,       lzbench_zstd_deinit },
-    { "zstd22",     "1.3.1",       1,  22,   22,       0, lzbench_zstd_compress,       lzbench_zstd_decompress,       lzbench_zstd_init,       lzbench_zstd_deinit },
-    { "zstd24",     "1.3.1",       1,  22,   24,       0, lzbench_zstd_compress,       lzbench_zstd_decompress,       lzbench_zstd_init,       lzbench_zstd_deinit },
+    { "zstd",       "1.3.3",       1,  22,    0,       0, lzbench_zstd_compress,       lzbench_zstd_decompress,       lzbench_zstd_init,       lzbench_zstd_deinit },
+    { "zstd22",     "1.3.3",       1,  22,   22,       0, lzbench_zstd_compress,       lzbench_zstd_decompress,       lzbench_zstd_init,       lzbench_zstd_deinit },
+    { "zstd24",     "1.3.3",       1,  22,   24,       0, lzbench_zstd_compress,       lzbench_zstd_decompress,       lzbench_zstd_init,       lzbench_zstd_deinit },
+    { "zstdLDM",    "1.3.3",       1,  22,    0,       0, lzbench_zstd_LDM_compress,   lzbench_zstd_decompress,       lzbench_zstd_LDM_init,   lzbench_zstd_deinit },
+    { "zstd22LDM",  "1.3.3",       1,  22,   22,       0, lzbench_zstd_LDM_compress,   lzbench_zstd_decompress,       lzbench_zstd_LDM_init,   lzbench_zstd_deinit },
+    { "zstd24LDM",  "1.3.3",       1,  22,   24,       0, lzbench_zstd_LDM_compress,   lzbench_zstd_decompress,       lzbench_zstd_LDM_init,   lzbench_zstd_deinit },
     { "nakamichi",  "okamigan",    0,   0,    0,       0, lzbench_nakamichi_compress,  lzbench_nakamichi_decompress,  NULL,                    NULL },
 };
 

diff --git a/brotli/common/constants.h b/brotli/common/constants.h
@@ -38,6 +38,8 @@
                                      BROTLI_MAX_NDIRECT +              \
                                      (BROTLI_MAX_DISTANCE_BITS <<      \
                                       (BROTLI_MAX_NPOSTFIX + 1)))
+/* Distance that is guaranteed to be representable in any stream. */
+#define BROTLI_MAX_DISTANCE 0x3FFFFFC
 
 /* 7.1. Context modes and context ID lookup for literals */
 /* "context IDs for literals are in the range of 0..63" */