Encryption patch follow-up

(* PBKDF2 implementation changed to OpenSSL implementation.)
OpenSSL changes ignored - ZOL changed for License reasons only.

* HKDF implementation moved to its own file and tests added to ensure
  correctness.

* Removed libzfs's now unnecessary dependency on libzpool and libicp.

* Ztest can now create and test encrypted datasets. This is currently
  disabled until issue #6526 is resolved, but otherwise functions as
  advertised.

* Several small bug fixes discovered after enabling ztest to run on
  encrypted datasets.

* Fixed coverity defects added by the encryption patch.

* Updated man pages for encrypted send / receive behavior.

* Fixed a bug where encrypted datasets could receive
  DRR_WRITE_EMBEDDED records.

* Minor code cleanups / consolidation.

* Fix for # 6703

Signed-off-by: Tom Caputi <tcaputi@datto.com>

* Decreasing stack usage in module/icp added due to panic with --enable-debug
- lundman

Also contains:

added comment about # 6706 fix
Fix for root cause of # 6706
Fix for send issue in # 6706
Fix for receive issue in # 6706

ZFSin/ZFSin.vcxproj

@@ -298,6 +298,7 @@
     <ClCompile Include="zfs\module\zfs\edonr_zfs.c" />
     <ClCompile Include="zfs\module\zfs\fm.c" />
     <ClCompile Include="zfs\module\zfs\gzip.c" />
+    <ClCompile Include="zfs\module\zfs\hkdf.c" />
     <ClCompile Include="zfs\module\zfs\lz4.c" />
     <ClCompile Include="zfs\module\zfs\lzjb.c" />
     <ClCompile Include="zfs\module\zfs\metaslab.c" />

ZFSin/ZFSin.vcxproj.filters

@@ -654,6 +654,9 @@
     <ClCompile Include="zfs\module\zfs\zio_crypt.c">
       <Filter>Source Files\ZFS\module\zfs</Filter>
     </ClCompile>
+    <ClCompile Include="zfs\module\zfs\hkdf.c">
+      <Filter>Source Files\ZFS\module\zfs</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="zfs\module\icp\algs\edonr\edonr_byteorder.h">

ZFSin/zfs/cmd/zfs/zfs_main.c

@@ -7447,28 +7447,27 @@ zfs_do_change_key(int argc, char **argv)
 		keystatus = zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS);
 		if (keystatus != ZFS_KEYSTATUS_AVAILABLE) {
 			ret = zfs_crypto_load_key(zhp, B_FALSE, NULL);
-			if (ret != 0)
-				goto error;
+			if (ret != 0) {
+				nvlist_free(props);
+				zfs_close(zhp);
+				return (-1);
+			}
 		}
 
-		/* refresh the properties so the new keystatus is visable */
+		/* refresh the properties so the new keystatus is visible */
 		zfs_refresh_properties(zhp);
 	}
 
 	ret = zfs_crypto_rewrap(zhp, props, inheritkey);
-	if (ret != 0)
-		goto error;
+	if (ret != 0) {
+		nvlist_free(props);
+		zfs_close(zhp);
+		return (-1);
+	}
 
 	nvlist_free(props);
 	zfs_close(zhp);
 	return (0);
-
-error:
-	if (props != NULL)
-		nvlist_free(props);
-	if (zhp != NULL)
-		zfs_close(zhp);
-	return (-1);
 }
 
 int

ZFSin/zfs/cmd/ztest/ztest.c

@@ -3327,6 +3327,9 @@ static int
 ztest_dataset_create(char *dsname)
 {
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
+=======
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	int err;
 	uint64_t rand;
 	dsl_crypto_params_t *dcp = NULL;
@@ -3373,10 +3376,13 @@ ztest_dataset_create(char *dsname)
 	}
 
 	err = dmu_objset_create(dsname, DMU_OST_OTHER, 0, dcp,
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 =======
 	uint64_t zilset = ztest_random(100);
 	int err = dmu_objset_create(dsname, DMU_OST_OTHER, 0, NULL,
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	    ztest_objset_create_cb, NULL);
 	dsl_crypto_params_free(dcp, !!err);
 
@@ -3401,12 +3407,17 @@ ztest_objset_destroy_cb(const char *name, void *arg)
 	/*
 	 * Verify that the dataset contains a directory object.
 	 */
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
 	    B_TRUE, FTAG, &os));
 =======
 	VERIFY0(dmu_objset_own(name, DMU_OST_OTHER, B_TRUE, B_TRUE, FTAG, &os));
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
+	    B_TRUE, FTAG, &os));
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	error = dmu_object_info(os, ZTEST_DIROBJ, &doi);
 	if (error != ENOENT) {
 		/* We could have crashed in the middle of destroying it */
@@ -3491,19 +3502,27 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 	 * (invoked from ztest_objset_destroy_cb()) should just throw it away.
 	 */
 	if (ztest_random(2) == 0 &&
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	    ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
 =======
 	    dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	    ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	    B_TRUE, FTAG, &os) == 0) {
 		ztest_zd_init(zdtmp, NULL, os);
 		zil_replay(os, zdtmp, ztest_replay_vector);
 		ztest_zd_fini(zdtmp);
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 		txg_wait_synced(dmu_objset_pool(os), 0);
 =======
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+		txg_wait_synced(dmu_objset_pool(os), 0);
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 		dmu_objset_disown(os, B_TRUE, FTAG);
 	}
 
@@ -3518,13 +3537,18 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 	/*
 	 * Verify that the destroyed dataset is no longer in the namespace.
 	 */
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	VERIFY3U(ENOENT, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
 	    B_TRUE, FTAG, &os));
 =======
 	VERIFY3U(ENOENT, ==, dmu_objset_own(name, DMU_OST_OTHER, B_TRUE, B_TRUE,
 	    FTAG, &os));
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	VERIFY3U(ENOENT, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
+	    B_TRUE, FTAG, &os));
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 
 	/*
 	 * Verify that we can create a new dataset.
@@ -3538,11 +3562,15 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 		fatal(0, "dmu_objset_create(%s) = %d", name, error);
 	}
 
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
 =======
 	VERIFY0(dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	    FTAG, &os));
 
 	ztest_zd_init(zdtmp, NULL, os);
@@ -3578,6 +3606,7 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 	/*
 	 * Verify that we cannot own an objset that is already owned.
 	 */
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	VERIFY3U(EBUSY, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER,
 	    B_FALSE, B_TRUE, FTAG, &os2));
@@ -3590,6 +3619,13 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 
 	zil_close(zilog);
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	VERIFY3U(EBUSY, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER,
+	    B_FALSE, B_TRUE, FTAG, &os2));
+
+	zil_close(zilog);
+	txg_wait_synced(spa_get_dsl(os->os_spa), 0);
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	dmu_objset_disown(os, B_TRUE, FTAG);
 	ztest_zd_fini(zdtmp);
 out:
@@ -3745,11 +3781,15 @@ ztest_dsl_dataset_promote_busy(ztest_ds_t *zd, uint64_t id)
 		fatal(0, "dmu_objset_create(%s) = %d", clone2name, error);
 	}
 
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	error = ztest_dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
 =======
 	error = dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	error = ztest_dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	    FTAG, &os);
 	if (error)
 		fatal(0, "dmu_objset_own(%s) = %d", snap2name, error);
@@ -5935,12 +5975,17 @@ ztest_dataset_open(int d)
 	}
 	ASSERT(error == 0 || error == EEXIST);
 
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
 	    B_TRUE, zd, &os));
 =======
 	VERIFY0(dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE, zd, &os));
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
+	    B_TRUE, zd, &os));
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	(void) rw_unlock(&ztest_name_lock);
 
 	ztest_zd_init(zd, ZTEST_GET_SHARED_DS(d), os);
@@ -5981,10 +6026,14 @@ ztest_dataset_close(int d)
 	ztest_ds_t *zd = &ztest_ds[d];
 
 	zil_close(zd->zd_zilog);
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	txg_wait_synced(spa_get_dsl(zd->zd_os->os_spa), 0);
 =======
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	txg_wait_synced(spa_get_dsl(zd->zd_os->os_spa), 0);
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 	dmu_objset_disown(zd->zd_os, B_TRUE, zd);
 
 	ztest_zd_fini(zd);
@@ -6037,11 +6086,15 @@ ztest_run(ztest_shared_t *zs)
 	ztest_spa = spa;
 
 	dmu_objset_stats_t dds;
+<<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 <<<<<<< HEAD:ZFSin/zfs/cmd/ztest/ztest.c
 	VERIFY0(ztest_dmu_objset_own(ztest_opts.zo_pool,
 =======
 	VERIFY0(dmu_objset_own(ztest_opts.zo_pool,
 >>>>>>> 4644f687... Native data and metadata encryption for zfs:cmd/ztest/ztest.c
+=======
+	VERIFY0(ztest_dmu_objset_own(ztest_opts.zo_pool,
+>>>>>>> f2e3bf1a... Encryption patch follow-up:cmd/ztest/ztest.c
 		DMU_OST_ANY, B_TRUE, B_TRUE, FTAG, &os));
 	dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
 	dmu_objset_fast_stat(os, &dds);

ZFSin/zfs/include/sys/arc_impl.h

@@ -96,6 +96,7 @@ struct arc_callback {
 	boolean_t		acb_encrypted;
 	boolean_t		acb_compressed;
 	boolean_t		acb_noauth;
+	uint64_t		acb_dsobj;
 	zio_t			*acb_zio_dummy;
 	arc_callback_t		*acb_next;
 };

ZFSin/zfs/include/sys/dmu.h

@@ -756,10 +756,13 @@ void dmu_tx_callback_register(dmu_tx_t *tx, dmu_tx_callback_func_t *dcb_func,
  * -1, the range from offset to end-of-file is freed.
  */
 int dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
-	uint64_t size, dmu_tx_t *tx);
+    uint64_t size, dmu_tx_t *tx);
 int dmu_free_long_range(objset_t *os, uint64_t object, uint64_t offset,
-	uint64_t size);
+    uint64_t size);
+int dmu_free_long_range_raw(objset_t *os, uint64_t object, uint64_t offset,
+    uint64_t size);
 int dmu_free_long_object(objset_t *os, uint64_t object);
+int dmu_free_long_object_raw(objset_t *os, uint64_t object);
 
 /*
  * Convenience functions.
@@ -808,10 +811,11 @@ struct blkptr *dmu_buf_get_blkptr(dmu_buf_t *db);
 #endif
 struct arc_buf *dmu_request_arcbuf(dmu_buf_t *handle, int size);
 void dmu_return_arcbuf(struct arc_buf *buf);
-void dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, struct arc_buf *buf,
-    dmu_tx_t *tx);
-void dmu_assign_arcbuf_impl(dmu_buf_t *handle, struct arc_buf *buf,
-    dmu_tx_t *tx);
+void dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset,
+    struct arc_buf *buf, dmu_tx_t *tx);
+void dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset,
+    struct arc_buf *buf, dmu_tx_t *tx);
+#define	dmu_assign_arcbuf	dmu_assign_arcbuf_by_dbuf
 void dmu_convert_to_raw(dmu_buf_t *handle, boolean_t byteorder,
     const uint8_t *salt, const uint8_t *iv, const uint8_t *mac, dmu_tx_t *tx);
 void dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,

ZFSin/zfs/include/sys/hkdf.h

@@ -0,0 +1,29 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ */
+
+#ifndef	_SYS_HKDF_H_
+#define	_SYS_HKDF_H_
+
+#include <sys/types.h>
+
+int hkdf_sha512(uint8_t *key_material, uint_t km_len, uint8_t *salt,
+    uint_t salt_len, uint8_t *info, uint_t info_len, uint8_t *output_key,
+    uint_t out_len);
+
+#endif	/* _SYS_HKDF_H_ */

ZFSin/zfs/include/sys/zio_crypt.h

@@ -32,18 +32,10 @@ struct zbookmark_phys;
 
 #define	WRAPPING_KEY_LEN	32
 #define	WRAPPING_IV_LEN		ZIO_DATA_IV_LEN
-#define	WRAPPING_MAC_LEN	16
-
-#define	SHA1_DIGEST_LEN		20
-#define	SHA512_DIGEST_LEN	64
-#define	SHA512_HMAC_KEYLEN	64
-
+#define	WRAPPING_MAC_LEN	ZIO_DATA_MAC_LEN
 #define	MASTER_KEY_MAX_LEN	32
-#define	L2ARC_DEFAULT_CRYPT ZIO_CRYPT_AES_256_CCM
-
-/* utility macros */
-#define	BITS_TO_BYTES(x) ((x + NBBY - 1) / NBBY)
-#define	BYTES_TO_BITS(x) (x * NBBY)
+#define	SHA512_HMAC_KEYLEN	64
+#define	SHA1_DIGEST_LEN		20
 
 typedef enum zio_crypt_type {
 	ZC_TYPE_NONE = 0,
@@ -133,7 +125,7 @@ int zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf,
 int zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd,
     uint_t datalen, boolean_t byteswap, uint8_t *cksum);
 int zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
-    uint8_t *digestbuf);
+    uint8_t *digestbuf, uint_t digestlen);
 int zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
     boolean_t byteswap, uint8_t *portable_mac, uint8_t *local_mac);
 int zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt,