MDEV-19935 Create unified CRC-32 interface

Add CRC32C code to mysys. The x86-64 implementation uses PCMULQDQ in addition to CRC32 instruction
after Intel whitepaper, and is ported from rocksdb code.

Optimized ARM and POWER CRC32 were already present in mysys.

config.h.cmake

@@ -103,15 +103,6 @@
 #cmakedefine HAVE_LIBWRAP 1
 #cmakedefine HAVE_SYSTEMD 1
 
-#cmakedefine HAVE_CPUID_INSTRUCTION 1
-#cmakedefine HAVE_CLMUL_INSTRUCTION 1
-#cmakedefine HAVE_CRC32_VPMSUM 1
-
-/* Support ARMv8 crc + crypto  */
-#cmakedefine HAVE_ARMV8_CRC 1
-#cmakedefine HAVE_ARMV8_CRYPTO 1
-#cmakedefine HAVE_ARMV8_CRC_CRYPTO_INTRINSICS 1
-
 /* Does "struct timespec" have a "sec" and "nsec" field? */
 #cmakedefine HAVE_TIMESPEC_TS_SEC 1
 

extra/CMakeLists.txt

@@ -73,20 +73,8 @@ IF(WITH_INNOBASE_STORAGE_ENGINE)
   # We use the InnoDB code directly in case the code changes.
   ADD_DEFINITIONS("-DUNIV_INNOCHECKSUM")
 
-  # Avoid generating Hardware Capabilities due to crc32 instructions
-  IF(CMAKE_SYSTEM_NAME MATCHES "SunOS" AND CMAKE_SYSTEM_PROCESSOR MATCHES "i386")
-    MY_CHECK_CXX_COMPILER_FLAG("-Wa,-nH")
-    IF(have_CXX__Wa__nH)
-      ADD_COMPILE_FLAGS(
-        ../storage/innobase/ut/ut0crc32.cc
-        COMPILE_FLAGS "-Wa,-nH"
-      )
-    ENDIF()
-  ENDIF()
-
   SET(INNOBASE_SOURCES
       ../storage/innobase/buf/buf0checksum.cc
-      ../storage/innobase/ut/ut0crc32.cc
       ../storage/innobase/ut/ut0ut.cc
       ../storage/innobase/buf/buf0buf.cc
       ../storage/innobase/page/page0zip.cc

extra/innochecksum.cc

@@ -1583,7 +1583,6 @@ int main(
 	/* enable when space_id of given file is zero. */
 	bool		is_system_tablespace = false;
 
-	ut_crc32_init();
 	MY_INIT(argv[0]);
 	DBUG_ENTER("main");
 	DBUG_PROCESS(argv[0]);

extra/mariabackup/backup_copy.cc

@@ -1833,7 +1833,6 @@ copy_back()
 
 	srv_max_n_threads = 1000;
 	sync_check_init();
-	ut_crc32_init();
 
 	/* copy undo tablespaces */
 

extra/mariabackup/xbstream.cc

@@ -97,8 +97,6 @@ main(int argc, char **argv)
 {
 	MY_INIT(argv[0]);
 
-	my_checksum_init();
-
 	if (get_options(&argc, &argv)) {
 		goto err;
 	}

extra/mariabackup/xtrabackup.cc

@@ -4011,9 +4011,6 @@ static bool xtrabackup_backup_func()
 	ut_d(sync_check_enable());
 	/* Reset the system variables in the recovery module. */
 	trx_pool_init();
-
-	ut_crc32_init();
-	my_checksum_init();
 	recv_sys.create();
 
 #ifdef WITH_INNODB_DISALLOW_WRITES
@@ -5386,7 +5383,6 @@ static bool xtrabackup_prepare_func(char** argv)
 
 		sync_check_init();
 		ut_d(sync_check_enable());
-		ut_crc32_init();
 		recv_sys.create();
 		log_sys.create();
 		recv_sys.recovery_on = true;

include/my_sys.h

@@ -901,18 +901,10 @@ extern int my_compress_buffer(uchar *dest, size_t *destLen,
 extern int packfrm(const uchar *, size_t, uchar **, size_t *);
 extern int unpackfrm(uchar **, size_t *, const uchar *);
 
-void my_checksum_init(void);
-#ifdef HAVE_CRC32_VPMSUM
-extern ha_checksum my_checksum(ha_checksum, const void *, size_t);
-#else
-typedef ha_checksum (*my_crc32_t)(ha_checksum, const void *, size_t);
-extern MYSQL_PLUGIN_IMPORT my_crc32_t my_checksum;
-#endif
+extern uint32 my_checksum(uint32, const void *, size_t);
+extern uint32 my_crc32c(uint32, const void *, size_t);
 
-#if defined(__GNUC__) && defined(HAVE_ARMV8_CRC)
-int crc32_aarch64_available(void);
-const char *crc32c_aarch64_available(void);
-#endif
+extern const char *my_crc32c_implementation();
 
 #ifdef DBUG_ASSERT_EXISTS
 extern void my_debug_put_break_here(void);

mysys/CMakeLists.txt

@@ -16,7 +16,7 @@
 
 INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/mysys)
 
-SET(MYSYS_SOURCES  array.c charset-def.c charset.c checksum.c my_default.c
+SET(MYSYS_SOURCES  array.c charset-def.c charset.c crc32ieee.cc my_default.c
                 get_password.c
 				errors.c hash.c list.c
                                 mf_cache.c mf_dirname.c mf_fn_ext.c
@@ -45,7 +45,7 @@ SET(MYSYS_SOURCES  array.c charset-def.c charset.c checksum.c my_default.c
                                 my_uuid.c wqueue.c waiting_threads.c ma_dyncol.c ../sql-common/my_time.c
 				my_rdtsc.c my_context.c psi_noop.c
                                 my_atomic_writes.c my_cpu.c my_likely.c my_largepage.c
-                                file_logger.c my_dlerror.c)
+                                file_logger.c my_dlerror.c   crc32/crc32c.cc)
 
 IF (WIN32)
   SET (MYSYS_SOURCES ${MYSYS_SOURCES}
@@ -59,25 +59,23 @@ IF (WIN32)
 ENDIF()
 
 IF(MSVC)
-  SET(HAVE_CPUID_INSTRUCTION 1 CACHE BOOL "")
-  SET(HAVE_CLMUL_INSTRUCTION 1 CACHE BOOL "")
   SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32_x86.c)
+  ADD_DEFINITIONS(-DHAVE_SSE42 -DHAVE_PCLMUL)
   IF(CLANG_CL)
-    SET_SOURCE_FILES_PROPERTIES(crc32/crc32_x86.c PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
+    SET_SOURCE_FILES_PROPERTIES(crc32/crc32_x86.cc crc32/crc32c.c PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
   ENDIF()
 ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|i386|i686")
-  SET(HAVE_CPUID_INSTRUCTION 1 CACHE BOOL "")
   MY_CHECK_C_COMPILER_FLAG(-msse4.2)
   MY_CHECK_C_COMPILER_FLAG(-mpclmul)
   CHECK_INCLUDE_FILE(cpuid.h HAVE_CPUID_H)
   CHECK_INCLUDE_FILE(x86intrin.h HAVE_X86INTRIN_H)
   IF(have_C__msse4.2 AND have_C__mpclmul AND HAVE_CPUID_H AND HAVE_X86INTRIN_H)
-    SET(HAVE_CLMUL_INSTRUCTION 1 CACHE BOOL "")
     SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32_x86.c)
-    SET_SOURCE_FILES_PROPERTIES(crc32/crc32_x86.c PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
+    SET_SOURCE_FILES_PROPERTIES(crc32/crc32_x86.c crc32/crc32c.cc PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
+    ADD_DEFINITIONS(-DHAVE_SSE42 -DHAVE_PCLMUL)
   ENDIF()
 ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
-  IF(CMAKE_COMPILER_IS_GNUCC AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.1)
+  IF(CMAKE_COMPILER_IS_GNUCC)
     include(CheckCXXSourceCompiles)
 
     CHECK_CXX_SOURCE_COMPILES("
@@ -99,23 +97,29 @@ ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
     #include <sys/auxv.h>
     int main() { foo(0); getauxval(AT_HWCAP); }" HAVE_ARMV8_CRYPTO)
 
-    CHECK_C_COMPILER_FLAG(-march=armv8-a+crc+crypto HAVE_ARMV8_CRC_CRYPTO_INTRINSICS)
-    IF(HAVE_ARMV8_CRC_CRYPTO_INTRINSICS)
+    CHECK_C_COMPILER_FLAG(-march=armv8-a+crc+crypto HAVE_ARMV8_CRC_CRYPTO_MARCH)
+
+    IF(HAVE_ARMV8_CRC_CRYPTO_MARCH)
+      CHECK_INCLUDE_FILE(arm_acle.h HAVE_ARM_ACLE_H -march=armv8-a+crc+crypto)
+      IF(HAVE_ARM_ACLE_H)
+       ADD_DEFINITIONS(-DHAVE_ARMV8_CRC_CRYPTO_INTRINSICS)
+      ENDIF()
+      IF(HAVE_ARMV8_CRC)
+        ADD_DEFINITIONS(-DHAVE_ARMV8_CRC)
+      ENDIF()
+      IF(HAVE_ARMV8_CRYPTO)
+        ADD_DEFINITIONS(-DHAVE_ARMV8_CRYPTO)
+      ENDIF()
       SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32_arm64.c)
       SET_SOURCE_FILES_PROPERTIES(crc32/crc32_arm64.c PROPERTIES
         COMPILE_FLAGS "-march=armv8-a+crc+crypto")
     ENDIF()
   ENDIF()
 ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64")
-  SET(HAVE_CRC32_VPMSUM 1 PARENT_SCOPE)
-  SET(MYSYS_SOURCES ${MYSYS_SOURCES} $<TARGET_OBJECTS:crc32c> $<TARGET_OBJECTS:crc32ieee>)
-
-  ADD_LIBRARY(crc32c OBJECT crc32/crc32_ppc64.c)
-  ADD_LIBRARY(crc32ieee OBJECT crc32/crc32_ppc64.c)
-
-  SET_TARGET_PROPERTIES(crc32c crc32ieee PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} -maltivec -mvsx -mpower8-vector -mcrypto -mpower8-vector")
-  SET_TARGET_PROPERTIES(crc32ieee PROPERTIES COMPILE_DEFINITIONS "CRC32_FUNCTION=my_checksum;CRC32_CONSTANTS_HEADER=\"pcc_crc32_constants.h\"")
-  SET_TARGET_PROPERTIES(crc32c PROPERTIES COMPILE_DEFINITIONS "CRC32_FUNCTION=crc32c_vpmsum;CRC32_CONSTANTS_HEADER=\"pcc_crc32c_constants.h\"")
+  SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32_ppc64.c crc32/crc32c_ppc.c)
+  SET_SOURCE_FILES_PROPERTIES(crc32/crc32_ppc64.c crc32/crc32c_ppc.c PROPERTIES
+        COMPILE_FLAGS "${COMPILE_FLAGS} -maltivec -mvsx -mpower8-vector -mcrypto -mpower8-vector")
+  ADD_DEFINITIONS(-DHAVE_POWER8 -DHAS_ALTIVEC)
 ENDIF()
 
 IF(UNIX)

mysys/crc32/crc32_arm64.c

@@ -57,6 +57,12 @@ asm(".arch_extension crypto");
 #define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
 #define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
 
+#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
+#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
+#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
+#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
+
+
 #define CRC32C3X8(buffer, ITR) \
   __asm__("crc32cx %w[c1], %w[c1], %x[v]":[c1]"+r"(crc1):[v]"r"(*((const uint64_t *)buffer + 42*1 + (ITR))));\
   __asm__("crc32cx %w[c2], %w[c2], %x[v]":[c2]"+r"(crc2):[v]"r"(*((const uint64_t *)buffer + 42*2 + (ITR))));\
@@ -73,6 +79,11 @@ asm(".arch_extension crypto");
 #define CRC32CH(crc, value) (crc) = __crc32ch((crc), (value))
 #define CRC32CB(crc, value) (crc) = __crc32cb((crc), (value))
 
+#define CRC32X(crc, value) (crc) = __crc32d((crc), (value))
+#define CRC32W(crc, value) (crc) = __crc32w((crc), (value))
+#define CRC32H(crc, value) (crc) = __crc32h((crc), (value))
+#define CRC32B(crc, value) (crc) = __crc32b((crc), (value))
+
 #define CRC32C3X8(buffer, ITR) \
   crc1 = __crc32cd(crc1, *((const uint64_t *)buffer + 42*1 + (ITR)));\
   crc2 = __crc32cd(crc2, *((const uint64_t *)buffer + 42*2 + (ITR)));\
@@ -119,7 +130,7 @@ uint32_t crc32c_aarch64(uint32_t crc, const unsigned char *buffer, uint64_t len)
   uint32_t crc0, crc1, crc2;
   int64_t length= (int64_t)len;
 
-  crc= 0xFFFFFFFFU;
+  crc^= 0xffffffff;
 
   /* Pmull runtime check here.
    * Raspberry Pi 4 supports crc32 but doesn't support pmull (MDEV-23030).
@@ -282,16 +293,16 @@ unsigned int crc32_aarch64(unsigned int crc, const void *buf, size_t len)
   /* if start pointer is not 8 bytes aligned */
   while ((buf1 != (const uint8_t *) buf8) && len)
   {
-    crc= __crc32b(crc, *buf1++);
+    CRC32B(crc, *buf1++);
     len--;
   }
 
   for (; len >= 8; len-= 8)
-    crc= __crc32d(crc, *buf8++);
+    CRC32X(crc, *buf8++);
 
   buf1= (const uint8_t *) buf8;
   while (len--)
-    crc= __crc32b(crc, *buf1++);
+    CRC32B(crc, *buf1++);
 
   return ~crc;
 }