Debug zio_wait() hangs (zio->io_magic)

One theory for openzfs#2523 is that somehow the spinlock at the start of
the mutex is being accidentally overwritten.  To test this theory
a magic value is added before the mutex.  By checking this value
periodically against a known magic value we can detect if this
is somehow happening.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Issue openzfs#2523

include/sys/zio.h

@@ -45,6 +45,11 @@ extern "C" {
  */
 #define	ZEC_MAGIC	0x210da7ab10c7a11ULL
 
+/*
+ * Debug zio redzone magic
+ */
+#define	ZIO_MAGIC	0x210210210210210ULL
+
 typedef struct zio_eck {
 	uint64_t	zec_magic;	/* for validation, endianness	*/
 	zio_cksum_t	zec_cksum;	/* 256-bit checksum		*/
@@ -445,6 +450,7 @@ struct zio {
 	zio_gang_node_t	*io_gang_tree;
 	void		*io_executor;
 	void		*io_waiter;
+	uint64_t	io_magic;
 	kmutex_t	io_lock;
 	kcondvar_t	io_cv;
 

module/zfs/zio.c

@@ -98,7 +98,10 @@ zio_cons(void *arg, void *unused, int kmflag)
 {
 	zio_t *zio = arg;
 
+	/* Verify the constructor never runs twice on the same object. */
+	VERIFY3U(zio->io_magic, !=, ZIO_MAGIC);
 	bzero(zio, sizeof (zio_t));
+	zio->io_magic = ZIO_MAGIC;
 
 	mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL);
@@ -116,6 +119,10 @@ zio_dest(void *arg, void *unused)
 {
 	zio_t *zio = arg;
 
+	/* Verify the destructor never runs twice on the same object. */
+	VERIFY3U(zio->io_magic, !=, ~ZIO_MAGIC);
+	zio->io_magic = ~ZIO_MAGIC;
+
 	mutex_destroy(&zio->io_lock);
 	cv_destroy(&zio->io_cv);
 	list_destroy(&zio->io_parent_list);
@@ -553,6 +560,7 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
 	ASSERT(vd || stage == ZIO_STAGE_OPEN);
 
 	zio = kmem_cache_alloc(zio_cache, KM_PUSHPAGE);
+	VERIFY3U(zio->io_magic, ==, ZIO_MAGIC);
 
 	if (vd != NULL)
 		zio->io_child_type = ZIO_CHILD_VDEV;
@@ -647,6 +655,7 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
 static void
 zio_destroy(zio_t *zio)
 {
+	VERIFY3U(zio->io_magic, ==, ZIO_MAGIC);
 	kmem_cache_free(zio_cache, zio);
 }
 
@@ -1380,6 +1389,7 @@ __zio_execute(zio_t *zio)
 		ASSERT(!MUTEX_HELD(&zio->io_lock));
 		ASSERT(ISP2(stage));
 		ASSERT(zio->io_stall == NULL);
+		VERIFY3U(zio->io_magic, ==, ZIO_MAGIC);
 
 		do {
 			stage <<= 1;
@@ -1450,8 +1460,10 @@ zio_wait(zio_t *zio)
 	__zio_execute(zio);
 
 	mutex_enter(&zio->io_lock);
-	while (zio->io_executor != NULL)
+	while (zio->io_executor != NULL) {
+		VERIFY3U(zio->io_magic, ==, ZIO_MAGIC);
 		cv_wait_io(&zio->io_cv, &zio->io_lock);
+	}
 	mutex_exit(&zio->io_lock);
 
 	error = zio->io_error;