RFC Add BLAKE3 Checksum to ZFS

- this is just a big FirstCommit for review
- it will be separated to smaller pieces in the real PR

modified:   AUTHORS
modified:   include/sys/Makefile.am
new file:   include/sys/blake3.h
modified:   include/sys/zio.h
modified:   include/sys/zio_checksum.h
modified:   include/zfeature_common.h
modified:   lib/libicp/Makefile.am
modified:   lib/libzpool/Makefile.am
modified:   man/man5/zpool-features.5
modified:   man/man8/zfsprops.8
modified:   module/icp/Makefile.in
new file:   module/icp/algs/blake3/blake3.c
new file:   module/icp/algs/blake3/blake3_generic.c
new file:   module/icp/algs/blake3/blake3_impl.c
new file:   module/icp/algs/blake3/blake3_neon.c
new file:   module/icp/algs/blake3/blake3_x86-64.c
new file:   module/icp/asm-x86_64/blake3/blake3_avx2.S
new file:   module/icp/asm-x86_64/blake3/blake3_avx512.S
new file:   module/icp/asm-x86_64/blake3/blake3_sse2.S
new file:   module/icp/asm-x86_64/blake3/blake3_sse41.S
new file:   module/icp/include/blake3/blake3_impl.h
modified:   module/zcommon/zfeature_common.c
modified:   module/zcommon/zfs_prop.c
modified:   module/zfs/Makefile.in
new file:   module/zfs/blake3_zfs.c
modified:   module/zfs/zio_checksum.c

Fixes for the first RFC BLAKE3 Checksum PR
- add zfs_hashes.c - this will get the micro-benchmark code
- add FEATURE_FLAG to include/sys/zfs_ioctl.h
- use kmem_cache_alloc() for the blake3 ctx (stack size)

 Bitte geben Sie eine Commit-Beschreibung für Ihre Änderungen ein. Zeilen,

AUTHORS

@@ -285,6 +285,7 @@ CONTRIBUTORS:
     Tim Connors <tconnors@rather.puzzling.org>
     Tim Crawford <tcrawford@datto.com>
     Tim Haley <Tim.Haley@Sun.COM>
+    Tino Reichardt <milky-zfs@mcmilk.de>
     Tobin Harding <me@tobin.cc>
     Tom Caputi <tcaputi@datto.com>
     Tom Matthews <tom@axiom-partners.com>

include/sys/Makefile.am

@@ -9,6 +9,7 @@ COMMON_H = \
 	avl.h \
 	avl_impl.h \
 	bitops.h \
+	blake3.h \
 	blkptr.h \
 	bplist.h \
 	bpobj.h \

include/sys/blake3.h

@@ -0,0 +1,98 @@
+
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Based on BLAKE3 v0.3.7, https://github.com/BLAKE3-team/BLAKE3
+ * Copyright (c) 2019-2020 Samuel Neves and Jack O'Connor
+ * Copyright (c) 2021 Tino Reichardt <milky-zfs@mcmilk.de>
+ */
+
+#ifndef BLAKE3_H
+#define	BLAKE3_H
+
+#ifdef  _KERNEL
+#include <sys/types.h>
+#else
+#include <stdint.h>
+#include <stdlib.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	BLAKE3_KEY_LEN		32
+#define	BLAKE3_OUT_LEN		32
+#define	BLAKE3_MAX_DEPTH	54
+#define	BLAKE3_BLOCK_LEN	64
+#define	BLAKE3_CHUNK_LEN	1024
+
+/*
+ * This struct is a private implementation detail.
+ * It has to be here because it's part of BLAKE3_CTX below.
+ */
+typedef struct {
+	uint32_t cv[8];
+	uint64_t chunk_counter;
+	uint8_t buf[BLAKE3_BLOCK_LEN];
+	uint8_t buf_len;
+	uint8_t blocks_compressed;
+	uint8_t flags;
+} blake3_chunk_state_t;
+
+typedef struct {
+	uint32_t key[8];
+	blake3_chunk_state_t chunk;
+	uint8_t cv_stack_len;
+
+	/*
+	 * The stack size is MAX_DEPTH + 1 because we do lazy merging. For
+	 * example, with 7 chunks, we have 3 entries in the stack. Adding an
+	 * 8th chunk requires a 4th entry, rather than merging everything down
+	 * to 1, because we don't know whether more input is coming. This is
+	 * different from how the reference implementation does things.
+	 */
+	uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN];
+} BLAKE3_CTX;
+
+/* init the context for hash operation */
+void Blake3_Init(BLAKE3_CTX *ctx);
+
+/* init the context for a MAC and/or tree hash operation */
+void Blake3_InitKeyed(BLAKE3_CTX *ctx, const uint8_t key[BLAKE3_KEY_LEN]);
+
+/* process the input bytes */
+void Blake3_Update(BLAKE3_CTX *ctx, const void *input, size_t input_len);
+
+/* finalize the hash computation and output the result */
+void Blake3_Final(const BLAKE3_CTX *ctx, uint8_t *out);
+
+/* finalize the hash computation and output the result */
+void Blake3_FinalSeek(const BLAKE3_CTX *ctx, uint64_t seek, uint8_t *out,
+    size_t out_len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* BLAKE3_H */

include/sys/zfs_ioctl.h

@@ -124,6 +124,7 @@ typedef enum drr_headertype {
  * default use of "zfs send" won't encounter the bug mentioned above.
  */
 #define	DMU_BACKUP_FEATURE_SWITCH_TO_LARGE_BLOCKS (1 << 27)
+#define	DMU_BACKUP_FEATURE_BLAKE3		(1 << 28)
 
 /*
  * Mask of all supported backup features
@@ -134,7 +135,7 @@ typedef enum drr_headertype {
     DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_LARGE_DNODE | \
     DMU_BACKUP_FEATURE_RAW | DMU_BACKUP_FEATURE_HOLDS | \
     DMU_BACKUP_FEATURE_REDACTED | DMU_BACKUP_FEATURE_SWITCH_TO_LARGE_BLOCKS | \
-    DMU_BACKUP_FEATURE_ZSTD)
+    DMU_BACKUP_FEATURE_ZSTD | DMU_BACKUP_FEATURE_BLAKE3)
 
 /* Are all features in the given flag word currently supported? */
 #define	DMU_STREAM_SUPPORTED(x)	(!((x) & ~DMU_BACKUP_FEATURE_MASK))

include/sys/zio.h

@@ -91,6 +91,7 @@ enum zio_checksum {
 #if !defined(__FreeBSD__)
 	ZIO_CHECKSUM_EDONR,
 #endif
+	ZIO_CHECKSUM_BLAKE3,
 	ZIO_CHECKSUM_FUNCTIONS
 };
 

include/sys/zio_checksum.h

@@ -21,7 +21,8 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2016 by Delphix. All rights reserved.
- * Copyright Saso Kiselkov 2013, All rights reserved.
+ * Copyright (c) 2013 Saso Kiselkov, All rights reserved.
+ * Copyright (c) 2021 Tino Reichardt <milky-zfs@mcmilk.de>
  */
 
 #ifndef _SYS_ZIO_CHECKSUM_H
@@ -106,6 +107,13 @@ extern zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS];
 /*
  * Checksum routines.
  */
+
+/* Fletcher 4 */
+extern zio_abd_checksum_func_t fletcher_4_abd_ops;
+extern zio_checksum_t abd_fletcher_4_native;
+extern zio_checksum_t abd_fletcher_4_byteswap;
+
+/* SHA2 */
 extern zio_checksum_t abd_checksum_SHA256;
 extern zio_checksum_t abd_checksum_SHA512_native;
 extern zio_checksum_t abd_checksum_SHA512_byteswap;
@@ -122,9 +130,11 @@ extern zio_checksum_t abd_checksum_edonr_byteswap;
 extern zio_checksum_tmpl_init_t abd_checksum_edonr_tmpl_init;
 extern zio_checksum_tmpl_free_t abd_checksum_edonr_tmpl_free;
 
-extern zio_abd_checksum_func_t fletcher_4_abd_ops;
-extern zio_checksum_t abd_fletcher_4_native;
-extern zio_checksum_t abd_fletcher_4_byteswap;
+/* BLAKE3 */
+extern zio_checksum_t abd_checksum_blake3_native;
+extern zio_checksum_t abd_checksum_blake3_byteswap;
+extern zio_checksum_tmpl_init_t abd_checksum_blake3_tmpl_init;
+extern zio_checksum_tmpl_free_t abd_checksum_blake3_tmpl_free;
 
 extern int zio_checksum_equal(spa_t *, blkptr_t *, enum zio_checksum,
     void *, uint64_t, uint64_t, zio_bad_cksum_t *);

include/zfeature_common.h

@@ -75,6 +75,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_DEVICE_REBUILD,
 	SPA_FEATURE_ZSTD_COMPRESS,
 	SPA_FEATURE_DRAID,
+	SPA_FEATURE_BLAKE3,
 	SPA_FEATURES
 } spa_feature_t;
 

include/zfs_hashes.h

@@ -0,0 +1,43 @@
+
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2021 Tino Reichardt <milky-zfs@mcmilk.de>
+ */
+
+#ifndef	_ZFS_HASHES_H
+#define	_ZFS_HASHES_H
+
+#include <sys/types.h>
+#include <sys/spa_checksum.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+void hashes_init(zio_cksum_t *);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _ZFS_HASHES_H */

lib/libicp/Makefile.am

@@ -14,6 +14,10 @@ ASM_SOURCES_C = asm-x86_64/aes/aeskey.c
 ASM_SOURCES_AS = \
 	asm-x86_64/aes/aes_amd64.S \
 	asm-x86_64/aes/aes_aesni.S \
+	asm-x86_64/blake3/blake3_avx2.S \
+	asm-x86_64/blake3/blake3_avx512.S \
+	asm-x86_64/blake3/blake3_sse2.S \
+	asm-x86_64/blake3/blake3_sse41.S \
 	asm-x86_64/modes/gcm_pclmulqdq.S \
 	asm-x86_64/modes/aesni-gcm-x86_64.S \
 	asm-x86_64/modes/ghash-x86_64.S \
@@ -37,6 +41,11 @@ KERNEL_C = \
 	algs/aes/aes_impl_x86-64.c \
 	algs/aes/aes_impl.c \
 	algs/aes/aes_modes.c \
+	algs/blake3/blake3.c \
+	algs/blake3/blake3_generic.c \
+	algs/blake3/blake3_impl.c \
+	algs/blake3/blake3_neon.c \
+	algs/blake3/blake3_x86-64.c \
 	algs/edonr/edonr.c \
 	algs/modes/modes.c \
 	algs/modes/cbc.c \

lib/libzpool/Makefile.am

@@ -54,6 +54,7 @@ KERNEL_C = \
 	aggsum.c \
 	arc.c \
 	arc_os.c \
+	blake3_zfs.c \
 	blkptr.c \
 	bplist.c \
 	bpobj.c \

man/man5/zpool-features.5

@@ -17,7 +17,7 @@
 .\" Copyright (c) 2019, Klara Inc.
 .\" Copyright (c) 2019, Allan Jude
 .\" Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
-.TH ZPOOL-FEATURES 5 "Aug 24, 2020" OpenZFS
+.TH ZPOOL-FEATURES 5 "Apr 9, 2021" OpenZFS
 .SH NAME
 zpool\-features \- ZFS pool feature descriptions
 .SH DESCRIPTION
@@ -268,6 +268,30 @@ through the \fBfreeing\fR property.
 This feature is only \fBactive\fR while \fBfreeing\fR is non\-zero.
 .RE
 
+.sp
+.ne 2
+.na
+\fBblake3\fR
+.ad
+.RS 4n
+.TS
+l l .
+GUID	org.openzfs:blake3
+READ\-ONLY COMPATIBLE	no
+DEPENDENCIES	extensible_dataset
+.TE
+
+This feature enables the use of the Blake3 hash algorithm for checksum
+and dedup. Blake3 is a high-performance secure hash algorithm... XXX / TODO
+
+When the \fBblake3\fR feature is set to \fBenabled\fR, the administrator
+can turn on the \fBblake3\fR checksum on any dataset using
+\fBzfs set checksum=blake3\fR. See zfs(8). This feature becomes
+\fBactive\fR once a \fBchecksum\fR property has been set to \fBblake3\fR,
+and will return to being \fBenabled\fR once all filesystems that have
+ever had their checksum set to \fBblake3\fR are destroyed.
+.RE
+
 .sp
 .ne 2
 .na

man/man8/zfsprops.8

@@ -749,7 +749,7 @@ This property is not inherited.
 .It Xo
 .Sy checksum Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy fletcher2 Ns | Ns
 .Sy fletcher4 Ns | Ns Sy sha256 Ns | Ns Sy noparity Ns | Ns
-.Sy sha512 Ns | Ns Sy skein Ns | Ns Sy edonr
+.Sy sha512 Ns | Ns Sy skein Ns | Ns Sy edonr Ns | Ns Sy blake3
 .Xc
 Controls the checksum used to verify data integrity.
 The default value is
@@ -774,6 +774,7 @@ a recommended practice.
 The
 .Sy sha512 ,
 .Sy skein ,
+.Sy blake3 ,
 and
 .Sy edonr
 checksum algorithms require enabling the appropriate features on the pool.
@@ -981,7 +982,7 @@ mount options.
 .It Xo
 .Sy dedup Ns = Ns Sy off Ns | Ns Sy on Ns | Ns Sy verify Ns | Ns
 .Sy sha256[,verify] Ns | Ns Sy sha512[,verify] Ns | Ns Sy skein[,verify] Ns | Ns
-.Sy edonr,verify
+.Sy edonr,verify Ns | Ns Sy blake3[,verify]
 .Xc
 Configures deduplication for a dataset. The default value is
 .Sy off .

module/Makefile.bsd

@@ -183,6 +183,7 @@ SRCS+=	zfeature_common.c \
 SRCS+=	abd.c \
 	aggsum.c \
 	arc.c \
+	blake3_zfs.c \
 	blkptr.c \
 	bplist.c \
 	bpobj.c \

module/icp/Makefile.in

@@ -42,13 +42,19 @@ $(MODULE)-objs += algs/modes/modes.o
 $(MODULE)-objs += algs/aes/aes_impl_generic.o
 $(MODULE)-objs += algs/aes/aes_impl.o
 $(MODULE)-objs += algs/aes/aes_modes.o
+$(MODULE)-objs += algs/blake3/blake3.o
+$(MODULE)-objs += algs/blake3/blake3_generic.o
+$(MODULE)-objs += algs/blake3/blake3_impl.o
+$(MODULE)-objs += algs/blake3/blake3_x86-64.o
 $(MODULE)-objs += algs/edonr/edonr.o
 $(MODULE)-objs += algs/sha1/sha1.o
 $(MODULE)-objs += algs/sha2/sha2.o
 $(MODULE)-objs += algs/skein/skein.o
 $(MODULE)-objs += algs/skein/skein_block.o
 $(MODULE)-objs += algs/skein/skein_iv.o
 
+$(MODULE)-$(CONFIG_ARM64) += algs/blake3/blake3_neon.o
+
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/aes/aeskey.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/aes/aes_amd64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/aes/aes_aesni.o
@@ -58,6 +64,10 @@ $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/modes/ghash-x86_64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha1/sha1-x86_64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_impl.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha512_impl.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/blake3/blake3_avx2.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/blake3/blake3_avx512.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/blake3/blake3_sse2.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/blake3/blake3_sse41.o
 
 $(MODULE)-$(CONFIG_X86) += algs/modes/gcm_pclmulqdq.o
 $(MODULE)-$(CONFIG_X86) += algs/aes/aes_impl_aesni.o
@@ -78,13 +88,15 @@ ICP_DIRS = \
 	os \
 	algs \
 	algs/aes \
+	algs/blake3 \
 	algs/edonr \
 	algs/modes \
 	algs/sha1 \
 	algs/sha2 \
 	algs/skein \
 	asm-x86_64 \
 	asm-x86_64/aes \
+	asm-x86_64/blake3 \
 	asm-x86_64/modes \
 	asm-x86_64/sha1 \
 	asm-x86_64/sha2 \