diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c
index a94101485..6af61cdcd 100644
--- a/module/zfs/vdev.c
+++ b/module/zfs/vdev.c
@@ -1286,9 +1286,9 @@ vdev_metaslab_group_create(vdev_t *vd)
 		    spa->spa_alloc_count);
 
 		/*
-		 * The spa ashift values currently only reflect the
-		 * general vdev classes. Class destination is late
-		 * binding so ashift checking had to wait until now
+		 * The spa ashift min/max only apply for the normal metaslab
+		 * class. Class destination is late binding so ashift boundry
+		 * setting had to wait until now.
 		 */
 		if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
 		    mc == spa_normal_class(spa) && vd->vdev_aux == NULL) {
@@ -1952,18 +1952,6 @@ vdev_open(vdev_t *vd)
 		return (error);
 	}
 
-	/*
-	 * Track the min and max ashift values for normal data devices.
-	 */
-	if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
-	    vd->vdev_alloc_bias == VDEV_BIAS_NONE &&
-	    vd->vdev_islog == 0 && vd->vdev_aux == NULL) {
-		if (vd->vdev_ashift > spa->spa_max_ashift)
-			spa->spa_max_ashift = vd->vdev_ashift;
-		if (vd->vdev_ashift < spa->spa_min_ashift)
-			spa->spa_min_ashift = vd->vdev_ashift;
-	}
-
 	/*
 	 * If this is a leaf vdev, assess whether a resilver is needed.
 	 * But don't do this if we are doing a reopen for a scrub, since
diff --git a/module/zfs/vdev_removal.c b/module/zfs/vdev_removal.c
index fdeca7ab3..ed7d1d4b3 100644
--- a/module/zfs/vdev_removal.c
+++ b/module/zfs/vdev_removal.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2019 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
  */
 
@@ -2030,6 +2030,15 @@ spa_vdev_remove_top_check(vdev_t *vd)
 		return (SET_ERROR(EINVAL));
 	}
 
+	/*
+	 * A removed special/dedup vdev must have same ashift as normal class.
+	 */
+	ASSERT(!vd->vdev_islog);
+	if (vd->vdev_alloc_bias != VDEV_BIAS_NONE &&
+	    vd->vdev_ashift != spa->spa_max_ashift) {
+		return (SET_ERROR(EINVAL));
+	}
+
 	/*
 	 * All vdevs in normal class must have the same ashift
 	 * and not be raidz.
@@ -2038,7 +2047,18 @@ spa_vdev_remove_top_check(vdev_t *vd)
 	int num_indirect = 0;
 	for (uint64_t id = 0; id < rvd->vdev_children; id++) {
 		vdev_t *cvd = rvd->vdev_child[id];
-		if (cvd->vdev_ashift != 0 && !cvd->vdev_islog)
+
+		/*
+		 * A removed special/dedup vdev must have the same ashift
+		 * across all vdevs in its class.
+		 */
+		if (vd->vdev_alloc_bias != VDEV_BIAS_NONE &&
+		    cvd->vdev_alloc_bias == vd->vdev_alloc_bias &&
+		    cvd->vdev_ashift != vd->vdev_ashift) {
+			return (SET_ERROR(EINVAL));
+		}
+		if (cvd->vdev_ashift != 0 &&
+		    cvd->vdev_alloc_bias == VDEV_BIAS_NONE)
 			ASSERT3U(cvd->vdev_ashift, ==, spa->spa_max_ashift);
 		if (cvd->vdev_ops == &vdev_indirect_ops)
 			num_indirect++;