Persistent L2ARC

This commit makes the L2ARC persistent across reboots. It is largely
based on issue 3525 in Illumos. This feature implements a light-weight
persistent L2ARC metadata structure that allows L2ARC contents to be
recovered after a reboot. This significantly eases the impact a reboot
has on read performance on systems with large caches.

Co-authored-by: Saso Kiselkov <skiselkov@gmail.com>
Co-authored-by: Jorgen Lundman <lundman@lundman.net>
Co-authored-by: George Amanakis <gamanakis@gmail.com>
Ported-by: Yuxuan Shui <yshuiv7@gmail.com>
Signed-off-by: George Amanakis <gamanakis@gmail.com>

cmd/zdb/zdb.c

@@ -62,6 +62,7 @@
 #include <sys/zio_compress.h>
 #include <sys/zfs_fuid.h>
 #include <sys/arc.h>
+#include <sys/arc_impl.h>
 #include <sys/ddt.h>
 #include <sys/zfeature.h>
 #include <sys/abd.h>
@@ -3474,6 +3475,56 @@ print_label_header(zdb_label_t *label, int l)
 	label->header_printed = B_TRUE;
 }
 
+static void
+print_l2arc_header(void)
+{
+	(void) printf("------------------------------------\n");
+	(void) printf("L2ARC device header\n");
+	(void) printf("------------------------------------\n");
+}
+
+static void
+dump_l2arc_header(int fd)
+{
+	l2arc_dev_hdr_phys_t hdr;
+
+	if (pread64(fd, &hdr, sizeof (hdr),
+	    VDEV_LABEL_START_SIZE) == sizeof (hdr)) {
+
+		if (hdr.dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC))
+			byteswap_uint64_array(&hdr, sizeof (hdr));
+
+		if (hdr.dh_magic == L2ARC_DEV_HDR_MAGIC) {
+
+			if (dump_opt['q'])
+				return;
+
+			print_l2arc_header();
+			(void) printf("    magic: %llu\n",
+			    (u_longlong_t)hdr.dh_magic);
+			(void) printf("    version: %llu\n",
+			    (u_longlong_t)hdr.dh_version);
+			(void) printf("    pool_guid: %llu\n",
+			    (u_longlong_t)hdr.dh_spa_guid);
+			(void) printf("    flags: %llu\n",
+			    (u_longlong_t)hdr.dh_flags);
+			(void) printf("    start_lbps[0]: %llu\n",
+			    (u_longlong_t)
+			    hdr.dh_start_lbps[0].lbp_daddr);
+			(void) printf("    start_lbps[1]: %llu\n",
+			    (u_longlong_t)
+			    hdr.dh_start_lbps[1].lbp_daddr);
+			(void) printf("    log_blk_count: %llu\n",
+			    (u_longlong_t)hdr.dh_log_blk_count);
+			(void) printf("    log_blk_ent: %llu\n",
+			    (u_longlong_t)hdr.dh_log_blk_ent);
+			(void) printf("    evict: %llu\n",
+			    (u_longlong_t)hdr.dh_evict);
+			(void) printf("\n");
+		}
+	}
+}
+
 static void
 dump_config_from_label(zdb_label_t *label, size_t buflen, int l)
 {
@@ -3762,6 +3813,11 @@ dump_label(const char *dev)
 		}
 	}
 
+	/*
+	 * Dump the L2ARC header, if existent.
+	 */
+	dump_l2arc_header(fd);
+
 	/*
 	 * Dump the label and uberblocks.
 	 */

cmd/ztest/ztest.c

@@ -1117,6 +1117,10 @@ make_vdev_root(char *path, char *aux, char *pool, size_t size, uint64_t ashift,
 		    r, m);
 		VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 		    log) == 0);
+		if (aux != NULL && strcmp(aux, ZPOOL_CONFIG_L2CACHE) == 0) {
+			VERIFY(nvlist_add_boolean_value(child[c],
+			    ZPOOL_CONFIG_L2CACHE_PERSISTENT, B_TRUE) == 0);
+		}
 
 		if (class != NULL && class[0] != '\0') {
 			ASSERT(m > 1 || log);   /* expecting a mirror */

configure.ac

@@ -334,6 +334,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/no_space/Makefile
 	tests/zfs-tests/tests/functional/nopwrite/Makefile
 	tests/zfs-tests/tests/functional/online_offline/Makefile
+	tests/zfs-tests/tests/functional/persist_l2arc/Makefile
 	tests/zfs-tests/tests/functional/pool_checkpoint/Makefile
 	tests/zfs-tests/tests/functional/pool_names/Makefile
 	tests/zfs-tests/tests/functional/poolversion/Makefile

include/sys/arc.h

@@ -300,13 +300,14 @@ void arc_fini(void);
  * Level 2 ARC
  */
 
-void l2arc_add_vdev(spa_t *spa, vdev_t *vd);
+void l2arc_add_vdev(spa_t *spa, vdev_t *vd, boolean_t rebuild);
 void l2arc_remove_vdev(vdev_t *vd);
 boolean_t l2arc_vdev_present(vdev_t *vd);
 void l2arc_init(void);
 void l2arc_fini(void);
 void l2arc_start(void);
 void l2arc_stop(void);
+void l2arc_spa_rebuild_start(spa_t *spa);
 
 #ifndef _KERNEL
 extern boolean_t arc_watch;