sha256 x86_64 optimization v2 openzfs#2351

This is a revision of openzfs#2332

Currently, the optimization only applies to kernel space,
because I haven't figured out how to it properly in user space.

AVX2 is untested because I don't have such CPU.
So use it with you own discretion.

config/user-arch.m4

@@ -16,4 +16,21 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_ARCH], [
 
 	AC_SUBST([TARGET_ASM_DIR])
 	AC_MSG_RESULT([$TARGET_ASM_DIR])
+	ZFS_AC_CONFIG_USER_INCLUDE_ASM_DIR
+])
+
+dnl #
+dnl # Setup include/asm dir
+dnl # Will copy every *.h from include/asm-generic to include/asm.
+dnl # Then, copy every *.h from include/$TARGET_ASM_DIR
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_USER_INCLUDE_ASM_DIR], [
+	AC_MSG_CHECKING(Setup include/asm dir)
+
+	test -d include/asm && rm include/asm/*.h
+	mkdir -p include/asm
+	test -d include/asm-generic && cp include/asm-generic/*.h include/asm
+	test asm-generic != $TARGET_ASM_DIR && test -d include/$TARGET_ASM_DIR && cp include/$TARGET_ASM_DIR/*.h include/asm
+
+	AC_MSG_RESULT([done])
 ])

configure.ac

@@ -119,13 +119,16 @@ AC_CONFIG_FILES([
 	module/unicode/Makefile
 	module/zcommon/Makefile
 	module/zfs/Makefile
+	module/zfs/asm-x86_64/Makefile
 	module/zpios/Makefile
 	include/Makefile
 	include/linux/Makefile
 	include/sys/Makefile
 	include/sys/fs/Makefile
 	include/sys/fm/Makefile
 	include/sys/fm/fs/Makefile
+	include/asm-generic/Makefile
+	include/asm-x86_64/Makefile
 	scripts/Makefile
 	scripts/zpios-profile/Makefile
 	scripts/zpios-test/Makefile

include/.gitignore

@@ -0,0 +1 @@
+/asm

include/Makefile.am

@@ -1,4 +1,4 @@
-SUBDIRS = linux sys
+SUBDIRS = linux sys asm-generic asm-x86_64
 
 COMMON_H = \
 	$(top_srcdir)/include/zfeature_common.h \

include/asm-generic/Makefile.am

@@ -0,0 +1,18 @@
+COMMON_H = \
+	$(top_srcdir)/include/asm-generic/sha256.h
+
+KERNEL_H =
+
+USER_H =
+
+EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H)
+
+if CONFIG_USER
+libzfsdir = $(includedir)/libzfs/linux
+libzfs_HEADERS = $(COMMON_H) $(USER_H)
+endif
+
+if CONFIG_KERNEL
+kerneldir = /usr/src/zfs-$(VERSION)/include/linux
+kernel_HEADERS = $(COMMON_H) $(KERNEL_H)
+endif

include/asm-generic/sha256.h

@@ -0,0 +1,14 @@
+#ifndef	_ASM_SHA256_H
+#define	_ASM_SHA256_H
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+static inline void arch_sha256_init(void) { }
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _ASM_SHA256_H */

include/asm-x86_64/Makefile.am

@@ -0,0 +1,18 @@
+COMMON_H = \
+	$(top_srcdir)/include/asm-x86_64/sha256.h
+
+KERNEL_H =
+
+USER_H =
+
+EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H)
+
+if CONFIG_USER
+libzfsdir = $(includedir)/libzfs/linux
+libzfs_HEADERS = $(COMMON_H) $(USER_H)
+endif
+
+if CONFIG_KERNEL
+kerneldir = /usr/src/zfs-$(VERSION)/include/linux
+kernel_HEADERS = $(COMMON_H) $(KERNEL_H)
+endif

include/asm-x86_64/sha256.h

@@ -0,0 +1,18 @@
+#ifndef	_ASM_SHA256_H
+#define	_ASM_SHA256_H
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#ifdef _KERNEL
+extern void arch_sha256_init(void);
+#else
+static inline void arch_sha256_init(void) { }
+#endif
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _ASM_SHA256_H */

include/sys/Makefile.am

@@ -40,6 +40,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/rrwlock.h \
 	$(top_srcdir)/include/sys/sa.h \
 	$(top_srcdir)/include/sys/sa_impl.h \
+	$(top_srcdir)/include/sys/sha256.h \
 	$(top_srcdir)/include/sys/sdt.h \
 	$(top_srcdir)/include/sys/spa_boot.h \
 	$(top_srcdir)/include/sys/space_map.h \

include/sys/sha256.h

@@ -0,0 +1,23 @@
+#ifndef	_SYS_SHA256_H
+#define	_SYS_SHA256_H
+
+#include <sys/zfs_context.h>
+#include <asm/sha256.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#define	SHA256_SHIFT	(6)
+#define	SHA256_BLOCK	(1<<SHA256_SHIFT)
+
+extern void (*sha256_transform)(const void *, uint32_t *, uint64_t);
+extern void sha256_transform_generic(const void *, uint32_t *, uint64_t);
+
+extern void zio_checksum_SHA256_init(void);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _SYS_SHA256_H */

include/sys/zio_checksum.h

@@ -68,6 +68,8 @@ extern void zio_checksum_compute(zio_t *zio, enum zio_checksum checksum,
 extern int zio_checksum_error(zio_t *zio, zio_bad_cksum_t *out);
 extern enum zio_checksum spa_dedup_checksum(spa_t *spa);
 
+extern void zio_checksum_init(void);
+
 #ifdef	__cplusplus
 }
 #endif

lib/libzpool/Makefile.am

@@ -128,4 +128,8 @@ EXTRA_DIST = \
 	$(top_srcdir)/module/zfs/zpl_super.c \
 	$(top_srcdir)/module/zfs/zpl_xattr.c \
 	$(top_srcdir)/module/zfs/zvol.c \
-	$(top_srcdir)/module/zpios/pios.c
+	$(top_srcdir)/module/zpios/pios.c \
+	$(top_srcdir)/module/zfs/asm-x86_64/sha256-ssse3-asm.S \
+	$(top_srcdir)/module/zfs/asm-x86_64/sha256-avx-asm.S \
+	$(top_srcdir)/module/zfs/asm-x86_64/sha256-avx2-asm.S \
+	$(top_srcdir)/module/zfs/asm-x86_64/sha256_x86_64.c

module/.gitignore

@@ -7,3 +7,5 @@
 /.tmp_versions
 /Module.markers
 /Module.symvers
+
+!Makefile.in

module/Makefile.in

@@ -7,6 +7,9 @@ subdir-m += zpios
 
 INSTALL_MOD_DIR ?= extra
 
+TARGET_ASM_DIR = @TARGET_ASM_DIR@
+export TARGET_ASM_DIR
+
 ZFS_MODULE_CFLAGS += -include @SPL_OBJ@/spl_config.h
 ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h
 ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include -I@SPL@/include -I@SPL@

module/zfs/Makefile.in

@@ -102,3 +102,12 @@ $(MODULE)-objs += @top_srcdir@/module/zfs/zrlock.o
 $(MODULE)-objs += @top_srcdir@/module/zfs/zvol.o
 $(MODULE)-objs += @top_srcdir@/module/zfs/dsl_destroy.o
 $(MODULE)-objs += @top_srcdir@/module/zfs/dsl_userhold.o
+
+#
+# Include arch specific Makefile if applicable
+#
+ifneq (,$(TARGET_ASM_DIR))
+ifneq (,$(wildcard $(src)/$(TARGET_ASM_DIR)/Makefile))
+include $(src)/$(TARGET_ASM_DIR)/Makefile
+endif
+endif

module/zfs/asm-x86_64/Makefile.in

@@ -0,0 +1,4 @@
+$(MODULE)-objs += asm-x86_64/sha256-ssse3-asm.o
+$(MODULE)-objs += asm-x86_64/sha256-avx-asm.o
+$(MODULE)-objs += asm-x86_64/sha256-avx2-asm.o
+$(MODULE)-objs += asm-x86_64/sha256_x86_64.o