zlib-ng 2023-01-09 (43fd1418)

Code extracted from:

    https://github.com/zlib-ng/zlib-ng.git

at commit 43fd141840821bd09b96a8ce2b4dec5b5ad563ff (develop).

.gitattributes

@@ -1,6 +1,7 @@
 * text=auto
 *.c text
 *.h text
+crc32_braid_tbl.h hooks-max-size=1000000
 Makefile text
 configure text eol=lf
 * -whitespace

CMakeLists.txt

@@ -77,6 +77,7 @@ option(WITH_GZFILEOP "Compile with support for gzFile related functions" ON)
 option(ZLIB_COMPAT "Compile with zlib compatible API" OFF)
 option(ZLIB_ENABLE_TESTS "Build test binaries" ON)
 option(ZLIBNG_ENABLE_TESTS "Test zlib-ng specific API" ON)
+option(WITH_GTEST "Build gtest_zlib" ON)
 option(WITH_FUZZERS "Build test/fuzz" OFF)
 option(WITH_BENCHMARKS "Build test/benchmarks" OFF)
 option(WITH_BENCHMARK_APPS "Build application benchmarks" OFF)
@@ -1187,6 +1188,7 @@ add_feature_info(ZLIB_COMPAT ZLIB_COMPAT "Compile with zlib compatible API")
 add_feature_info(ZLIB_ENABLE_TESTS ZLIB_ENABLE_TESTS "Build test binaries")
 add_feature_info(ZLIBNG_ENABLE_TESTS ZLIBNG_ENABLE_TESTS "Test zlib-ng specific API")
 add_feature_info(WITH_SANITIZER WITH_SANITIZER "Enable sanitizer support")
+add_feature_info(WITH_GTEST WITH_GTEST "Build gtest_zlib")
 add_feature_info(WITH_FUZZERS WITH_FUZZERS "Build test/fuzz")
 add_feature_info(WITH_BENCHMARKS WITH_BENCHMARKS "Build test/benchmarks")
 add_feature_info(WITH_BENCHMARK_APPS WITH_BENCHMARK_APPS "Build application benchmarks")

README.md

@@ -120,6 +120,7 @@ Build Options
 | WITH_NEW_STRATEGIES      | --without-new-strategies | Use new strategies                                                                    | ON      |
 | WITH_NATIVE_INSTRUCTIONS | --native                 | Compiles with full instruction set supported on this host (gcc/clang -march=native)   | OFF     |
 | WITH_SANITIZER           |                          | Build with sanitizer (memory, address, undefined)                                     | OFF     |
+| WITH_GTEST               |                          | Build gtest_zlib                                                                      | ON      |
 | WITH_FUZZERS             |                          | Build test/fuzz                                                                       | OFF     |
 | WITH_BENCHMARKS          |                          | Build test/benchmarks                                                                 | OFF     |
 | WITH_MAINTAINER_WARNINGS |                          | Build with project maintainer warnings                                                | OFF     |

arch/s390/README.md

@@ -63,15 +63,15 @@ integrated with the rest of zlib-ng using hook macros.
 DFLTCC takes as arguments a parameter block, an input buffer, an output
 buffer and a window. `ZALLOC_DEFLATE_STATE()`, `ZALLOC_INFLATE_STATE()`,
 `ZFREE_STATE()`, `ZCOPY_DEFLATE_STATE()`, `ZCOPY_INFLATE_STATE()`,
-`ZALLOC_WINDOW()` and `TRY_FREE_WINDOW()` macros encapsulate allocation
-details for the parameter block (which is allocated alongside zlib-ng
-state) and the window (which must be page-aligned).
-
-While inflate software and hardware window formats match, this is not
-the case for deflate. Therefore, `deflateSetDictionary()` and
-`deflateGetDictionary()` need special handling, which is triggered using
-`DEFLATE_SET_DICTIONARY_HOOK()` and `DEFLATE_GET_DICTIONARY_HOOK()`
-macros.
+`ZALLOC_WINDOW()`, `ZCOPY_WINDOW()` and `TRY_FREE_WINDOW()` macros encapsulate
+allocation  details for the parameter block (which is allocated alongside
+zlib-ng state) and the window (which must be page-aligned and large enough).
+
+Software and hardware window formats do not match, therefore,
+`deflateSetDictionary()`, `deflateGetDictionary()`, `inflateSetDictionary()`
+and `inflateGetDictionary()` need special handling, which is triggered using
+`DEFLATE_SET_DICTIONARY_HOOK()`, `DEFLATE_GET_DICTIONARY_HOOK()`,
+`INFLATE_SET_DICTIONARY_HOOK()` and `INFLATE_GET_DICTIONARY_HOOK()` macros.
 
 `deflateResetKeep()` and `inflateResetKeep()` update the DFLTCC
 parameter block using `DEFLATE_RESET_KEEP_HOOK()` and

arch/s390/crc32-vx.c

@@ -202,12 +202,12 @@ uint32_t Z_INTERNAL PREFIX(s390_crc32_vx)(uint32_t crc, const unsigned char *buf
     uint64_t prealign, aligned, remaining;
 
     if (len < VX_MIN_LEN + VX_ALIGN_MASK)
-        return crc32_braid(crc, buf, len);
+        return PREFIX(crc32_braid)(crc, buf, len);
 
     if ((uintptr_t)buf & VX_ALIGN_MASK) {
         prealign = VX_ALIGNMENT - ((uintptr_t)buf & VX_ALIGN_MASK);
         len -= prealign;
-        crc = crc32_braid(crc, buf, prealign);
+        crc = PREFIX(crc32_braid)(crc, buf, prealign);
         buf += prealign;
     }
     aligned = len & ~VX_ALIGN_MASK;
@@ -216,7 +216,7 @@ uint32_t Z_INTERNAL PREFIX(s390_crc32_vx)(uint32_t crc, const unsigned char *buf
     crc = crc32_le_vgfm_16(crc ^ 0xffffffff, buf, (size_t)aligned) ^ 0xffffffff;
 
     if (remaining)
-        crc = crc32_braid(crc, buf + aligned, remaining);
+        crc = PREFIX(crc32_braid)(crc, buf + aligned, remaining);
 
     return crc;
 }

arch/s390/dfltcc_common.c

@@ -20,16 +20,20 @@ void Z_INTERNAL *PREFIX(dfltcc_alloc_window)(PREFIX3(streamp) strm, uInt items,
     void *w;
 
     /* To simplify freeing, we store the pointer to the allocated buffer right
-     * before the window.
+     * before the window. Note that DFLTCC always uses HB_SIZE bytes.
      */
-    p = ZALLOC(strm, sizeof(void *) + items * size + PAGE_ALIGN, sizeof(unsigned char));
+    p = ZALLOC(strm, sizeof(void *) + MAX(items * size, HB_SIZE) + PAGE_ALIGN, sizeof(unsigned char));
     if (p == NULL)
         return NULL;
     w = ALIGN_UP((char *)p + sizeof(void *), PAGE_ALIGN);
     *(void **)((char *)w - sizeof(void *)) = p;
     return w;
 }
 
+void Z_INTERNAL PREFIX(dfltcc_copy_window)(void *dest, const void *src, size_t n) {
+    memcpy(dest, src, MAX(n, HB_SIZE));
+}
+
 void Z_INTERNAL PREFIX(dfltcc_free_window)(PREFIX3(streamp) strm, void *w) {
     if (w)
         ZFREE(strm, *(void **)((unsigned char *)w - sizeof(void *)));

arch/s390/dfltcc_common.h

@@ -4,12 +4,15 @@
 #include "zutil.h"
 
 void Z_INTERNAL *PREFIX(dfltcc_alloc_window)(PREFIX3(streamp) strm, uInt items, uInt size);
+void Z_INTERNAL PREFIX(dfltcc_copy_window)(void *dest, const void *src, size_t n);
 void Z_INTERNAL PREFIX(dfltcc_free_window)(PREFIX3(streamp) strm, void *w);
 
 #define ZFREE_STATE ZFREE
 
 #define ZALLOC_WINDOW PREFIX(dfltcc_alloc_window)
 
+#define ZCOPY_WINDOW PREFIX(dfltcc_copy_window)
+
 #define ZFREE_WINDOW PREFIX(dfltcc_free_window)
 
 #define TRY_FREE_WINDOW PREFIX(dfltcc_free_window)

arch/s390/dfltcc_deflate.c

@@ -240,7 +240,10 @@ int Z_INTERNAL PREFIX(dfltcc_deflate)(PREFIX3(streamp) strm, int flush, block_st
         *strm->next_out = (unsigned char)state->bi_buf;
     /* Honor history and check value */
     param->nt = 0;
-    param->cv = state->wrap == 2 ? ZSWAP32(state->crc_fold.value) : strm->adler;
+    if (state->wrap == 1)
+        param->cv = strm->adler;
+    else if (state->wrap == 2)
+        param->cv = ZSWAP32(state->crc_fold.value);
 
     /* When opening a block, choose a Huffman-Table Type */
     if (!param->bcf) {
@@ -271,10 +274,10 @@ int Z_INTERNAL PREFIX(dfltcc_deflate)(PREFIX3(streamp) strm, int flush, block_st
         state->bi_buf = 0; /* Avoid accessing next_out */
     else
         state->bi_buf = *strm->next_out & ((1 << state->bi_valid) - 1);
-    if (state->wrap == 2)
-        state->crc_fold.value = ZSWAP32(param->cv);
-    else
+    if (state->wrap == 1)
         strm->adler = param->cv;
+    else if (state->wrap == 2)
+        state->crc_fold.value = ZSWAP32(param->cv);
 
     /* Unmask the input data */
     strm->avail_in += masked_avail_in;
@@ -376,36 +379,6 @@ int Z_INTERNAL PREFIX(dfltcc_can_set_reproducible)(PREFIX3(streamp) strm, int re
 /*
    Preloading history.
 */
-static void append_history(struct dfltcc_param_v0 *param, unsigned char *history, const unsigned char *buf, uInt count) {
-    size_t offset;
-    size_t n;
-
-    /* Do not use more than 32K */
-    if (count > HB_SIZE) {
-        buf += count - HB_SIZE;
-        count = HB_SIZE;
-    }
-    offset = (param->ho + param->hl) % HB_SIZE;
-    if (offset + count <= HB_SIZE)
-        /* Circular history buffer does not wrap - copy one chunk */
-        memcpy(history + offset, buf, count);
-    else {
-        /* Circular history buffer wraps - copy two chunks */
-        n = HB_SIZE - offset;
-        memcpy(history + offset, buf, n);
-        memcpy(history, buf + n, count - n);
-    }
-    n = param->hl + count;
-    if (n <= HB_SIZE)
-        /* All history fits into buffer - no need to discard anything */
-        param->hl = n;
-    else {
-        /* History does not fit into buffer - discard extra bytes */
-        param->ho = (param->ho + (n - HB_SIZE)) % HB_SIZE;
-        param->hl = HB_SIZE;
-    }
-}
-
 int Z_INTERNAL PREFIX(dfltcc_deflate_set_dictionary)(PREFIX3(streamp) strm,
                                                 const unsigned char *dictionary, uInt dict_length) {
     deflate_state *state = (deflate_state *)strm->state;
@@ -423,16 +396,8 @@ int Z_INTERNAL PREFIX(dfltcc_deflate_get_dictionary)(PREFIX3(streamp) strm, unsi
     struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
     struct dfltcc_param_v0 *param = &dfltcc_state->param;
 
-    if (dictionary) {
-        if (param->ho + param->hl <= HB_SIZE)
-            /* Circular history buffer does not wrap - copy one chunk */
-            memcpy(dictionary, state->window + param->ho, param->hl);
-        else {
-            /* Circular history buffer wraps - copy two chunks */
-            memcpy(dictionary, state->window + param->ho, HB_SIZE - param->ho);
-            memcpy(dictionary + HB_SIZE - param->ho, state->window, param->ho + param->hl - HB_SIZE);
-        }
-    }
+    if (dictionary)
+        get_history(param, state->window, dictionary);
     if (dict_length)
         *dict_length = param->hl;
     return Z_OK;

arch/s390/dfltcc_detail.h

@@ -267,3 +267,46 @@ static inline void dfltcc_reset_state(struct dfltcc_state *dfltcc_state) {
 static inline void dfltcc_copy_state(void *dst, const void *src, uInt size, uInt extension_size) {
     memcpy(dst, src, ALIGN_UP(size, 8) + extension_size);
 }
+
+static inline void append_history(struct dfltcc_param_v0 *param, unsigned char *history,
+                                  const unsigned char *buf, uInt count) {
+    size_t offset;
+    size_t n;
+
+    /* Do not use more than 32K */
+    if (count > HB_SIZE) {
+        buf += count - HB_SIZE;
+        count = HB_SIZE;
+    }
+    offset = (param->ho + param->hl) % HB_SIZE;
+    if (offset + count <= HB_SIZE)
+        /* Circular history buffer does not wrap - copy one chunk */
+        memcpy(history + offset, buf, count);
+    else {
+        /* Circular history buffer wraps - copy two chunks */
+        n = HB_SIZE - offset;
+        memcpy(history + offset, buf, n);
+        memcpy(history, buf + n, count - n);
+    }
+    n = param->hl + count;
+    if (n <= HB_SIZE)
+        /* All history fits into buffer - no need to discard anything */
+        param->hl = n;
+    else {
+        /* History does not fit into buffer - discard extra bytes */
+        param->ho = (param->ho + (n - HB_SIZE)) % HB_SIZE;
+        param->hl = HB_SIZE;
+    }
+}
+
+static inline void get_history(struct dfltcc_param_v0 *param, const unsigned char *history,
+                               unsigned char *buf) {
+    if (param->ho + param->hl <= HB_SIZE)
+        /* Circular history buffer does not wrap - copy one chunk */
+        memcpy(buf, history + param->ho, param->hl);
+    else {
+        /* Circular history buffer wraps - copy two chunks */
+        memcpy(buf, history + param->ho, HB_SIZE - param->ho);
+        memcpy(buf + HB_SIZE - param->ho, history, param->ho + param->hl - HB_SIZE);
+    }
+}

arch/s390/dfltcc_inflate.c

@@ -39,10 +39,6 @@ int Z_INTERNAL PREFIX(dfltcc_can_inflate)(PREFIX3(streamp) strm) {
     struct inflate_state *state = (struct inflate_state *)strm->state;
     struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
 
-    /* Unsupported compression settings */
-    if (state->wbits != HB_BITS)
-        return 0;
-
     /* Unsupported hardware */
     return is_bit_set(dfltcc_state->af.fns, DFLTCC_XPND) && is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0);
 }
@@ -96,13 +92,12 @@ dfltcc_inflate_action Z_INTERNAL PREFIX(dfltcc_inflate)(PREFIX3(streamp) strm, i
     }
 
     /* Translate stream to parameter block */
-    param->cvt = state->flags ? CVT_CRC32 : CVT_ADLER32;
+    param->cvt = ((state->wrap & 4) && state->flags) ? CVT_CRC32 : CVT_ADLER32;
     param->sbb = state->bits;
-    param->hl = state->whave; /* Software and hardware history formats match */
-    param->ho = (state->wnext - state->whave) & ((1 << HB_BITS) - 1);
     if (param->hl)
         param->nt = 0; /* Honor history for the first block */
-    param->cv = state->flags ? ZSWAP32(state->check) : state->check;
+    if (state->wrap & 4)
+        param->cv = state->flags ? ZSWAP32(state->check) : state->check;
 
     /* Inflate */
     do {
@@ -113,9 +108,8 @@ dfltcc_inflate_action Z_INTERNAL PREFIX(dfltcc_inflate)(PREFIX3(streamp) strm, i
     strm->msg = oesc_msg(dfltcc_state->msg, param->oesc);
     state->last = cc == DFLTCC_CC_OK;
     state->bits = param->sbb;
-    state->whave = param->hl;
-    state->wnext = (param->ho + param->hl) & ((1 << HB_BITS) - 1);
-    strm->adler = state->check = state->flags ? ZSWAP32(param->cv) : param->cv;
+    if (state->wrap & 4)
+        strm->adler = state->check = state->flags ? ZSWAP32(param->cv) : param->cv;
     if (cc == DFLTCC_CC_OP2_CORRUPT && param->oesc != 0) {
         /* Report an error if stream is corrupted */
         state->mode = BAD;
@@ -134,9 +128,33 @@ int Z_INTERNAL PREFIX(dfltcc_was_inflate_used)(PREFIX3(streamp) strm) {
     return !param->nt;
 }
 
+/*
+   Rotates a circular buffer.
+   The implementation is based on https://cplusplus.com/reference/algorithm/rotate/
+ */
+static void rotate(unsigned char *start, unsigned char *pivot, unsigned char *end) {
+    unsigned char *p = pivot;
+    unsigned char tmp;
+
+    while (p != start) {
+        tmp = *start;
+        *start = *p;
+        *p = tmp;
+
+        start++;
+        p++;
+
+        if (p == end)
+            p = pivot;
+        else if (start == pivot)
+            pivot = p;
+    }
+}
+
 int Z_INTERNAL PREFIX(dfltcc_inflate_disable)(PREFIX3(streamp) strm) {
     struct inflate_state *state = (struct inflate_state *)strm->state;
     struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
+    struct dfltcc_param_v0 *param = &dfltcc_state->param;
 
     if (!PREFIX(dfltcc_can_inflate)(strm))
         return 0;
@@ -148,5 +166,40 @@ int Z_INTERNAL PREFIX(dfltcc_inflate_disable)(PREFIX3(streamp) strm) {
         return 1;
     /* DFLTCC was not used yet - decompress in software */
     memset(&dfltcc_state->af, 0, sizeof(dfltcc_state->af));
+    /* Convert the window from the hardware to the software format */
+    rotate(state->window, state->window + param->ho, state->window + HB_SIZE);
+    state->whave = state->wnext = MIN(param->hl, state->wsize);
     return 0;
 }
+
+/*
+   Preloading history.
+*/
+int Z_INTERNAL PREFIX(dfltcc_inflate_set_dictionary)(PREFIX3(streamp) strm,
+                                                     const unsigned char *dictionary, uInt dict_length) {
+    struct inflate_state *state = (struct inflate_state *)strm->state;
+    struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
+    struct dfltcc_param_v0 *param = &dfltcc_state->param;
+
+    if (PREFIX(inflate_ensure_window)(state)) {
+        state->mode = MEM;
+        return Z_MEM_ERROR;
+    }
+
+    append_history(param, state->window, dictionary, dict_length);
+    state->havedict = 1;
+    return Z_OK;
+}
+
+int Z_INTERNAL PREFIX(dfltcc_inflate_get_dictionary)(PREFIX3(streamp) strm,
+                                                     unsigned char *dictionary, uInt *dict_length) {
+    struct inflate_state *state = (struct inflate_state *)strm->state;
+    struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
+    struct dfltcc_param_v0 *param = &dfltcc_state->param;
+
+    if (dictionary && state->window)
+        get_history(param, state->window, dictionary);
+    if (dict_length)
+        *dict_length = param->hl;
+    return Z_OK;
+}

arch/s390/dfltcc_inflate.h

@@ -15,6 +15,10 @@ typedef enum {
 dfltcc_inflate_action Z_INTERNAL PREFIX(dfltcc_inflate)(PREFIX3(streamp) strm, int flush, int *ret);
 int Z_INTERNAL PREFIX(dfltcc_was_inflate_used)(PREFIX3(streamp) strm);
 int Z_INTERNAL PREFIX(dfltcc_inflate_disable)(PREFIX3(streamp) strm);
+int Z_INTERNAL PREFIX(dfltcc_inflate_set_dictionary)(PREFIX3(streamp) strm,
+                                                     const unsigned char *dictionary, uInt dict_length);
+int Z_INTERNAL PREFIX(dfltcc_inflate_get_dictionary)(PREFIX3(streamp) strm,
+                                                     unsigned char *dictionary, uInt* dict_length);
 
 #define ZALLOC_INFLATE_STATE PREFIX(dfltcc_alloc_inflate_state)
 #define ZCOPY_INFLATE_STATE PREFIX(dfltcc_copy_inflate_state)
@@ -51,4 +55,16 @@ int Z_INTERNAL PREFIX(dfltcc_inflate_disable)(PREFIX3(streamp) strm);
         if (PREFIX(dfltcc_was_inflate_used)((strm))) return Z_STREAM_ERROR; \
     } while (0)
 
+#define INFLATE_SET_DICTIONARY_HOOK(strm, dict, dict_len) \
+    do { \
+        if (PREFIX(dfltcc_can_inflate)((strm))) \
+            return PREFIX(dfltcc_inflate_set_dictionary)((strm), (dict), (dict_len)); \
+    } while (0)
+
+#define INFLATE_GET_DICTIONARY_HOOK(strm, dict, dict_len) \
+    do { \
+        if (PREFIX(dfltcc_can_inflate)((strm))) \
+            return PREFIX(dfltcc_inflate_get_dictionary)((strm), (dict), (dict_len)); \
+    } while (0)
+
 #endif