zlib-ng 2023-08-07 (73bbb54c)

Code extracted from:

    https://github.com/zlib-ng/zlib-ng.git

at commit 73bbb54cf6686a81710a1326c4cf1cfee9a49784 (develop).

Author: kwrobot

CMakeLists.txt

@@ -486,16 +486,6 @@ if(NOT HAVE_PTRDIFF_T)
     endif()
 endif()
 
-# Macro to check if source compiles
-# (and, when compiling very natively, also runs).
-macro(check_c_source_compile_or_run source flag)
-    if(CMAKE_CROSSCOMPILING OR NOT WITH_NATIVE_INSTRUCTIONS)
-        check_c_source_compiles("${source}" ${flag})
-    else()
-        check_c_source_runs("${source}" ${flag})
-    endif()
-endmacro()
-
 add_compile_options($<$<CONFIG:Debug>:-DZLIB_DEBUG>)
 
 if(MSVC)
@@ -736,7 +726,7 @@ if(WITH_OPTIM)
                 list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/riscv_features.c)
                 # FIXME: we will not set compile flags for riscv_features.c when 
                 # the kernels update hwcap or hwprobe for riscv
-                set(RVV_SRCS ${ARCHDIR}/riscv_features.c ${ARCHDIR}/compare256_rvv.c ${ARCHDIR}/slide_hash_rvv.c)
+                set(RVV_SRCS ${ARCHDIR}/riscv_features.c ${ARCHDIR}/adler32_rvv.c ${ARCHDIR}/compare256_rvv.c ${ARCHDIR}/slide_hash_rvv.c)
                 list(APPEND ZLIB_ARCH_SRCS ${RVV_SRCS})
                 set_property(SOURCE ${RVV_SRCS} PROPERTY COMPILE_FLAGS "${RISCVFLAG} ${NOLTOFLAG}")
             else()
@@ -827,15 +817,12 @@ if(WITH_OPTIM)
         endif()
         if(WITH_SSE42)
             check_sse42_intrinsics()
-            if(HAVE_SSE42CRC_INLINE_ASM OR HAVE_SSE42CRC_INTRIN)
+            if(HAVE_SSE42_INTRIN)
                 add_definitions(-DX86_SSE42)
                 set(SSE42_SRCS ${ARCHDIR}/adler32_sse42.c ${ARCHDIR}/insert_string_sse42.c)
                 add_feature_info(SSE42_CRC 1 "Support SSE4.2 optimized CRC hash generation, using \"${SSE42FLAG}\"")
                 list(APPEND ZLIB_ARCH_SRCS ${SSE42_SRCS})
                 set_property(SOURCE ${SSE42_SRCS} PROPERTY COMPILE_FLAGS "${SSE42FLAG} ${NOLTOFLAG}")
-                if(HAVE_SSE42CRC_INTRIN)
-                    add_definitions(-DX86_SSE42_CRC_INTRIN)
-                endif()
             else()
                 set(WITH_SSE42 OFF)
             endif()
@@ -951,35 +938,18 @@ if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
     endif()
 endif()
 
-# Refer to prefix symbolically to ease relocation by end user,
-# as Makefile-generated .pc file does.
-string(FIND "${CMAKE_INSTALL_INCLUDEDIR}" "${CMAKE_INSTALL_PREFIX}/" INCLUDEDIR_POS)
-string(FIND "${CMAKE_INSTALL_LIBDIR}" "${CMAKE_INSTALL_PREFIX}/" LIBDIR_POS)
-string(LENGTH "${CMAKE_INSTALL_PREFIX}/" INSTALL_PREFIX_LEN)
-
-if(NOT IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}")
-  set(PC_INC_INSTALL_DIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
-elseif(INCLUDEDIR_POS EQUAL 0)
-  string(SUBSTRING "${CMAKE_INSTALL_INCLUDEDIR}" "${INSTALL_PREFIX_LEN}" "-1" INCLUDEDIR_RELATIVE)
-  set(PC_INC_INSTALL_DIR "\${prefix}/${INCLUDEDIR_RELATIVE}")
+# The user is allowed (but discouraged) to set absolute CMAKE_INSTALL_*DIR paths.
+# If they do, we copy these non-relocatable paths into the pkg-config file.
+if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}")
+    set(PC_INC_INSTALL_DIR "${CMAKE_INSTALL_INCLUDEDIR}")
 else()
-  set(PC_INC_INSTALL_DIR "${CMAKE_INSTALL_INCLUDEDIR}")
+    set(PC_INC_INSTALL_DIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
 endif()
 
-if(APPLE)
-  option(WITH_RPATH "Enable RPATH for shared library" OFF)
-endif()
-if(NOT IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}")
-  if(APPLE AND WITH_RPATH)
+if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}")
     set(PC_LIB_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}")
-  else()
-    set(PC_LIB_INSTALL_DIR "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
-  endif()
-elseif(LIBDIR_POS EQUAL 0)
-  string(SUBSTRING "${CMAKE_INSTALL_LIBDIR}" "${INSTALL_PREFIX_LEN}" "-1" LIBDIR_RELATIVE)
-  set(PC_LIB_INSTALL_DIR "\${exec_prefix}/${LIBDIR_RELATIVE}")
 else()
-  set(PC_LIB_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}")
+    set(PC_LIB_INSTALL_DIR "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
 #============================================================================
@@ -1140,13 +1110,11 @@ if(NOT DEFINED BUILD_SHARED_LIBS OR BUILD_SHARED_LIBS)
             set_target_properties(zlib PROPERTIES COMPILE_FLAGS "-fno-semantic-interposition")
         endif()
         if(NOT APPLE)
+            if(NOT ZLIB_COMPAT)
+                add_definitions(-DHAVE_SYMVER)
+            endif()
             set_target_properties(zlib PROPERTIES LINK_FLAGS
                 "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib${SUFFIX}.map\"")
-        elseif(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}" OR NOT WITH_RPATH)
-            # Match configure/make's behavior (i.e. don't use @rpath on mac when using absolute path).
-            set_target_properties(zlib PROPERTIES INSTALL_NAME_DIR "@rpath/${CMAKE_INSTALL_FULL_LIBDIR}")
-        else()
-            set_target_properties(zlib PROPERTIES INSTALL_NAME_DIR "@rpath/${CMAKE_INSTALL_LIBDIR}")
         endif()
     endif()
     if(MSYS)
@@ -1183,11 +1151,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zlib${SUFFIX}.h.in
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gzread.c.in
     ${CMAKE_CURRENT_BINARY_DIR}/gzread.c @ONLY)
 
-# Fix install directory after generating zlib.pc/zlib-ng.pc
-if (NOT IS_ABSOLUTE CMAKE_INSTALL_LIBDIR AND WITH_RPATH)
-    set(CMAKE_INSTALL_LIBDIR "/${CMAKE_INSTALL_LIBDIR}")
-endif()
-
 if (NOT ZLIB_SYMBOL_PREFIX STREQUAL "")
     add_feature_info(ZLIB_SYMBOL_PREFIX ON "Publicly exported symbols have a custom prefix")
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zlib_name_mangling${SUFFIX}.h.in
@@ -1286,8 +1249,4 @@ endif()
 
 add_feature_info(INSTALL_UTILS INSTALL_UTILS "Copy minigzip and minideflate during install")
 
-if(APPLE)
-    add_feature_info(WITH_RPATH WITH_RPATH "Enable RPATH for shared library")
-endif()
-
 FEATURE_SUMMARY(WHAT ALL INCLUDE_QUIET_PACKAGES)

arch/arm/arm_features.c

@@ -45,7 +45,7 @@ static int arm_has_crc32() {
 }
 
 /* AArch64 has neon. */
-#if !defined(__aarch64__) && !defined(_M_ARM64)
+#if !defined(__aarch64__) && !defined(_M_ARM64) && !defined(_M_ARM64EC)
 static inline int arm_has_neon() {
 #if defined(__linux__) && defined(ARM_AUXV_HAS_NEON)
 #  ifdef HWCAP_ARM_NEON
@@ -73,7 +73,7 @@ static inline int arm_has_neon() {
 #endif
 
 void Z_INTERNAL arm_check_features(struct arm_cpu_features *features) {
-#if defined(__aarch64__) || defined(_M_ARM64)
+#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
     features->has_neon = 1; /* always available */
 #else
     features->has_neon = arm_has_neon();

arch/arm/chunkset_neon.c

@@ -68,13 +68,11 @@ static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t
     lut_rem_pair lut_rem = perm_idx_lut[dist - 3];
     *chunk_rem = lut_rem.remval;
 
-#ifdef Z_MEMORY_SANITIZER
     /* See note in chunkset_ssse3.c for why this is ok */
     __msan_unpoison(buf + dist, 16 - dist);
-#endif
 
     /* This version of table is only available on aarch64 */
-#if defined(_M_ARM64) || defined(__aarch64__)
+#if defined(_M_ARM64) || defined(_M_ARM64EC) || defined(__aarch64__)
     uint8x16_t ret_vec = vld1q_u8(buf);
 
     uint8x16_t perm_vec = vld1q_u8(permute_table + lut_rem.idx);

arch/arm/crc32_acle.c

@@ -33,7 +33,7 @@ Z_INTERNAL uint32_t crc32_acle(uint32_t crc, const uint8_t *buf, size_t len) {
         buf4 = (const uint32_t *) buf;
     }
 
-#if defined(__aarch64__) || defined(_M_ARM64)
+#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
     if ((len >= sizeof(uint32_t)) && ((ptrdiff_t)buf & sizeof(uint32_t))) {
         c = __crc32w(c, *buf4++);
         len -= sizeof(uint32_t);

arch/arm/neon_intrins.h

@@ -1,13 +1,13 @@
 #ifndef ARM_NEON_INTRINS_H
 #define ARM_NEON_INTRINS_H
 
-#ifdef _M_ARM64
+#if defined(_M_ARM64) || defined(_M_ARM64EC)
 #  include <arm64_neon.h>
 #else
 #  include <arm_neon.h>
 #endif
 
-#if defined(ARM_NEON) && !defined(__aarch64__) && !defined(_M_ARM64)
+#if defined(ARM_NEON) && !defined(__aarch64__) && !defined(_M_ARM64) && !defined(_M_ARM64EC)
 /* Compatibility shim for the _high family of functions */
 #define vmull_high_u8(a, b) vmull_u8(vget_high_u8(a), vget_high_u8(b))
 #define vmlal_high_u8(a, b, c) vmlal_u8(a, vget_high_u8(b), vget_high_u8(c))

arch/riscv/adler32_rvv.c

@@ -0,0 +1,117 @@
+/* adler32_rvv.c - RVV version of adler32
+ * Copyright (C) 2023 SiFive, Inc. All rights reserved.
+ * Contributed by Alex Chiang <alex.chiang@sifive.com>
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ */
+
+#ifdef RISCV_RVV
+
+#include <riscv_vector.h>
+#include <stdint.h>
+
+#include "../../zbuild.h"
+#include "../../adler32_p.h"
+
+Z_INTERNAL uint32_t adler32_rvv(uint32_t adler, const uint8_t *buf, size_t len) {
+    /* split Adler-32 into component sums */
+    uint32_t sum2 = (adler >> 16) & 0xffff;
+    adler &= 0xffff;
+
+    /* in case user likes doing a byte at a time, keep it fast */
+    if (len == 1) {
+        return adler32_len_1(adler, buf, sum2);
+    }
+
+    /* initial Adler-32 value (deferred check for len == 1 speed) */
+    if (buf == NULL)
+        return 1L;
+
+    /* in case short lengths are provided, keep it somewhat fast */
+    if (len < 16) {
+        return adler32_len_16(adler, buf, len, sum2);
+    }
+
+    size_t left = len;
+    size_t vl = __riscv_vsetvlmax_e8m1();
+    vl = vl > 256 ? 256 : vl;
+    vuint32m4_t v_buf32_accu = __riscv_vmv_v_x_u32m4(0, vl);
+    vuint32m4_t v_adler32_prev_accu = __riscv_vmv_v_x_u32m4(0, vl);
+    vuint16m2_t v_buf16_accu;
+
+    /*
+     * We accumulate 8-bit data, and to prevent overflow, we have to use a 32-bit accumulator.
+     * However, adding 8-bit data into a 32-bit accumulator isn't efficient. We use 16-bit & 32-bit
+     * accumulators to boost performance.
+     *
+     * The block_size is the largest multiple of vl that <= 256, because overflow would occur when
+     * vl > 256 (255 * 256 <= UINT16_MAX).
+     *
+     * We accumulate 8-bit data into a 16-bit accumulator and then
+     * move the data into the 32-bit accumulator at the last iteration.
+     */
+    size_t block_size = (256 / vl) * vl;
+    size_t nmax_limit = (NMAX / block_size);
+    size_t cnt = 0;
+    while (left >= block_size) {
+        v_buf16_accu = __riscv_vmv_v_x_u16m2(0, vl);
+        size_t subprob = block_size;
+        while (subprob > 0) {
+            vuint8m1_t v_buf8 = __riscv_vle8_v_u8m1(buf, vl);
+            v_adler32_prev_accu = __riscv_vwaddu_wv_u32m4(v_adler32_prev_accu, v_buf16_accu, vl);
+            v_buf16_accu = __riscv_vwaddu_wv_u16m2(v_buf16_accu, v_buf8, vl);
+            buf += vl;
+            subprob -= vl;
+        }
+        v_adler32_prev_accu = __riscv_vmacc_vx_u32m4(v_adler32_prev_accu, block_size / vl, v_buf32_accu, vl);
+        v_buf32_accu = __riscv_vwaddu_wv_u32m4(v_buf32_accu, v_buf16_accu, vl);
+        left -= block_size;
+        /* do modulo once each block of NMAX size */
+        if (++cnt >= nmax_limit) {
+            v_adler32_prev_accu = __riscv_vremu_vx_u32m4(v_adler32_prev_accu, BASE, vl);
+            cnt = 0;
+        }
+    }
+    /* the left len <= 256 now, we can use 16-bit accum safely */
+    v_buf16_accu = __riscv_vmv_v_x_u16m2(0, vl);
+    size_t res = left;
+    while (left >= vl) {
+        vuint8m1_t v_buf8 = __riscv_vle8_v_u8m1(buf, vl);
+        v_adler32_prev_accu = __riscv_vwaddu_wv_u32m4(v_adler32_prev_accu, v_buf16_accu, vl);
+        v_buf16_accu = __riscv_vwaddu_wv_u16m2(v_buf16_accu, v_buf8, vl);
+        buf += vl;
+        left -= vl;
+    }
+    v_adler32_prev_accu = __riscv_vmacc_vx_u32m4(v_adler32_prev_accu, res / vl, v_buf32_accu, vl);
+    v_adler32_prev_accu = __riscv_vremu_vx_u32m4(v_adler32_prev_accu, BASE, vl);
+    v_buf32_accu = __riscv_vwaddu_wv_u32m4(v_buf32_accu, v_buf16_accu, vl);
+
+    vuint32m4_t v_seq = __riscv_vid_v_u32m4(vl);
+    vuint32m4_t v_rev_seq = __riscv_vrsub_vx_u32m4(v_seq, vl, vl);
+    vuint32m4_t v_sum32_accu = __riscv_vmul_vv_u32m4(v_buf32_accu, v_rev_seq, vl);
+
+    v_sum32_accu = __riscv_vadd_vv_u32m4(v_sum32_accu, __riscv_vmul_vx_u32m4(v_adler32_prev_accu, vl, vl), vl);
+
+    vuint32m1_t v_sum2_sum = __riscv_vmv_s_x_u32m1(0, vl);
+    v_sum2_sum = __riscv_vredsum_vs_u32m4_u32m1(v_sum32_accu, v_sum2_sum, vl);
+    uint32_t sum2_sum = __riscv_vmv_x_s_u32m1_u32(v_sum2_sum);
+
+    sum2 += (sum2_sum + adler * (len - left));
+
+    vuint32m1_t v_adler_sum = __riscv_vmv_s_x_u32m1(0, vl);
+    v_adler_sum = __riscv_vredsum_vs_u32m4_u32m1(v_buf32_accu, v_adler_sum, vl);
+    uint32_t adler_sum = __riscv_vmv_x_s_u32m1_u32(v_adler_sum);
+
+    adler += adler_sum;
+
+    while (left--) {
+        adler += *buf++;
+        sum2 += adler;
+    }
+
+    sum2 %= BASE;
+    adler %= BASE;
+
+    return adler | (sum2 << 16);
+}
+
+#endif // RISCV_RVV

arch/riscv/compare256_rvv.c

@@ -21,7 +21,7 @@ static inline uint32_t compare256_rvv_static(const uint8_t *src0, const uint8_t
         vuint8m4_t v_src1 = __riscv_vle8_v_u8m4(src1, vl);
         vbool2_t v_mask = __riscv_vmsne_vv_u8m4_b2(v_src0, v_src1, vl);
         found_diff = __riscv_vfirst_m_b2(v_mask, vl);
-        if (found_diff >= 0) 
+        if (found_diff >= 0)
             return len + (uint32_t)found_diff;
         src0 += vl, src1 += vl, len += vl;
     } while (len < 256);

arch/riscv/riscv_features.h

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2023 SiFive, Inc. All rights reserved.
  * Contributed by Alex Chiang <alex.chiang@sifive.com>
- * 
+ *
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 

arch/s390/dfltcc_detail.h

@@ -166,9 +166,7 @@ static inline dfltcc_cc dfltcc(int fn, void *param,
                                unsigned char **op1, size_t *len1,
                                z_const unsigned char **op2, size_t *len2, void *hist) {
     unsigned char *t2 = op1 ? *op1 : NULL;
-#ifdef Z_MEMORY_SANITIZER
     unsigned char *orig_t2 = t2;
-#endif
     size_t t3 = len1 ? *len1 : 0;
     z_const unsigned char *t4 = op2 ? *op2 : NULL;
     size_t t5 = len2 ? *len2 : 0;
@@ -203,7 +201,6 @@ static inline dfltcc_cc dfltcc(int fn, void *param,
                      : "cc", "memory");
     t2 = r2; t3 = r3; t4 = r4; t5 = r5;
 
-#ifdef Z_MEMORY_SANITIZER
     switch (fn & DFLTCC_FN_MASK) {
     case DFLTCC_QAF:
         __msan_unpoison(param, DFLTCC_SIZEOF_QAF);
@@ -220,7 +217,6 @@ static inline dfltcc_cc dfltcc(int fn, void *param,
         __msan_unpoison(orig_t2, t2 - orig_t2);
         break;
     }
-#endif
 
     if (op1)
         *op1 = t2;

arch/x86/adler32_avx2_tpl.h

@@ -7,7 +7,7 @@
 #include <immintrin.h>
 #include "../../adler32_fold.h"
 #include "../../adler32_p.h"
-#include "../../fallback_builtins.h"
+#include "x86_intrins.h"
 #include "adler32_avx2_p.h"
 
 #ifdef X86_SSE42