OpenZFS 6531 - Provide mechanism to artificially limit disk performance

Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Ported by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>

OpenZFS-issue: https://www.illumos.org/issues/6531
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/97e8130

Porting notes:
- Added new IO delay tracepoints, and moved common ZIO tracepoint macros
  to a new trace_common.h file.
- Used zio_delay_taskq() in place of OpenZFS's timeout_generic() function.
- Updated zinject man page
- Updated zpool_scrub test files

tests/zfs-tests/tests/functional/cli_root/zpool_scrub/setup.ksh

@@ -38,10 +38,7 @@ verify_disk_count "$DISKS" 2
 default_mirror_setup_noexit $DISK1 $DISK2
 
 mntpnt=$(get_prop mountpoint $TESTPOOL)
-typeset -i i=0
-while ((i < 10)); do
-	log_must $DD if=/dev/urandom of=$mntpnt/bigfile.$i bs=1024k count=100
-	((i += 1))
-done
 
+# Create 100MB of data
+log_must $FILE_WRITE -b 1048576 -c 100 -o create -d 0 -f $mntpnt/bigfile
 log_pass

tests/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_002_pos.ksh

@@ -38,16 +38,15 @@
 #	3. Verify zpool scrub -s succeed when the system is scrubbing.
 #
 # NOTES:
-#	A 1 second delay is added to 10% of zio's in order to ensure that
-#	the scrub does not complete before it has a chance to be cancelled.
+#	A 10ms delay is added to the ZIOs in order to ensure that the
+#	scrub does not complete before it has a chance to be cancelled.
 #	This can occur when testing with small pools or very fast hardware.
 #
 
 verify_runnable "global"
 
 log_assert "Verify scrub -s works correctly."
-
-log_must $ZINJECT -d $DISK1 -f10 -D1 $TESTPOOL
+log_must $ZINJECT -d $DISK1 -D10:1 $TESTPOOL
 log_must $ZPOOL scrub $TESTPOOL
 log_must $ZPOOL scrub -s $TESTPOOL
 log_must is_pool_scrub_stopped $TESTPOOL

tests/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_003_pos.ksh

@@ -40,8 +40,8 @@
 #	4. Check the percent again, verify a new scrub started.
 #
 # NOTES:
-#	A 1 second delay is added to 10% of zio's in order to ensure that
-#	the scrub does not complete before it has a chance to be restarted.
+#	A 10ms delay is added to the ZIOs in order to ensure that the
+#	scrub does not complete before it has a chance to be restarted.
 #	This can occur when testing with small pools or very fast hardware.
 #
 
@@ -61,7 +61,7 @@ function get_scrub_percent
 log_assert "scrub command terminates the existing scrub process and starts" \
 	"a new scrub."
 
-log_must $ZINJECT -d $DISK1 -f10 -D1 $TESTPOOL
+log_must $ZINJECT -d $DISK1 -D10:1 $TESTPOOL
 log_must $ZPOOL scrub $TESTPOOL
 typeset -i PERCENT=30 percent=0
 while ((percent < PERCENT)) ; do

tests/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_004_pos.ksh

@@ -42,16 +42,16 @@
 #	3. Verify scrub failed until the resilver completed
 #
 # NOTES:
-#	A 1 second delay is added to 10% of zio's in order to ensure that
-#	the resilver does not complete before the scrub can be issue.  This
+#	A 10ms delay is added to 10% of zio's in order to ensure that the
+#	resilver does not complete before the scrub can be issued.  This
 #	can occur when testing with small pools or very fast hardware.
 
 verify_runnable "global"
 
 log_assert "Resilver prevent scrub from starting until the resilver completes"
 
-log_must $ZINJECT -d $DISK1 -f10 -D1 $TESTPOOL
 log_must $ZPOOL detach $TESTPOOL $DISK2
+log_must $ZINJECT -d $DISK1 -D10:1 $TESTPOOL
 log_must $ZPOOL attach $TESTPOOL $DISK1 $DISK2
 log_must is_pool_resilvering $TESTPOOL
 log_mustnot $ZPOOL scrub $TESTPOOL