Ignore special vdev ashift for spa ashift min/max

The removal of a vdev in the normal class would fail if there was a
special or deup vdev that had a different ashift than the vdevs in
the normal class.

Moved the initialization of spa_min_ashift / spa_max_ashift from
vdev_open so that it occurs after the vdev allocation bias was
initialized (i.e. after vdev_load).

Caveat -- In order to remove a special/dedup vdev it must have the
same ashift as the normal pool vdevs.  This could perhaps be lifted
in the future (i.e. for the case where there is ample space in any
surviving special class vdevs)

Reviewed-by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Don Brady <don.brady@delphix.com>
Closes #9363
Closes #9364
Closes #11053

Author: Don Brady

module/zfs/vdev.c

@@ -1286,9 +1286,9 @@ vdev_metaslab_group_create(vdev_t *vd)
 		    spa->spa_alloc_count);
 
 		/*
-		 * The spa ashift values currently only reflect the
-		 * general vdev classes. Class destination is late
-		 * binding so ashift checking had to wait until now
+		 * The spa ashift min/max only apply for the normal metaslab
+		 * class. Class destination is late binding so ashift boundry
+		 * setting had to wait until now.
 		 */
 		if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
 		    mc == spa_normal_class(spa) && vd->vdev_aux == NULL) {
@@ -1952,18 +1952,6 @@ vdev_open(vdev_t *vd)
 		return (error);
 	}
 
-	/*
-	 * Track the min and max ashift values for normal data devices.
-	 */
-	if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
-	    vd->vdev_alloc_bias == VDEV_BIAS_NONE &&
-	    vd->vdev_islog == 0 && vd->vdev_aux == NULL) {
-		if (vd->vdev_ashift > spa->spa_max_ashift)
-			spa->spa_max_ashift = vd->vdev_ashift;
-		if (vd->vdev_ashift < spa->spa_min_ashift)
-			spa->spa_min_ashift = vd->vdev_ashift;
-	}
-
 	/*
 	 * If this is a leaf vdev, assess whether a resilver is needed.
 	 * But don't do this if we are doing a reopen for a scrub, since

module/zfs/vdev_removal.c

@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2019 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
  */
 
@@ -2030,6 +2030,15 @@ spa_vdev_remove_top_check(vdev_t *vd)
 		return (SET_ERROR(EINVAL));
 	}
 
+	/*
+	 * A removed special/dedup vdev must have same ashift as normal class.
+	 */
+	ASSERT(!vd->vdev_islog);
+	if (vd->vdev_alloc_bias != VDEV_BIAS_NONE &&
+	    vd->vdev_ashift != spa->spa_max_ashift) {
+		return (SET_ERROR(EINVAL));
+	}
+
 	/*
 	 * All vdevs in normal class must have the same ashift
 	 * and not be raidz.
@@ -2038,7 +2047,18 @@ spa_vdev_remove_top_check(vdev_t *vd)
 	int num_indirect = 0;
 	for (uint64_t id = 0; id < rvd->vdev_children; id++) {
 		vdev_t *cvd = rvd->vdev_child[id];
-		if (cvd->vdev_ashift != 0 && !cvd->vdev_islog)
+
+		/*
+		 * A removed special/dedup vdev must have the same ashift
+		 * across all vdevs in its class.
+		 */
+		if (vd->vdev_alloc_bias != VDEV_BIAS_NONE &&
+		    cvd->vdev_alloc_bias == vd->vdev_alloc_bias &&
+		    cvd->vdev_ashift != vd->vdev_ashift) {
+			return (SET_ERROR(EINVAL));
+		}
+		if (cvd->vdev_ashift != 0 &&
+		    cvd->vdev_alloc_bias == VDEV_BIAS_NONE)
 			ASSERT3U(cvd->vdev_ashift, ==, spa->spa_max_ashift);
 		if (cvd->vdev_ops == &vdev_indirect_ops)
 			num_indirect++;