zvol: Enable zvol threading functionality on FreeBSD

Make zvol I/O requests processing asynchronous on FreeBSD side in some
cases. Clone zvol threading logic and required module parameters from
Linux side. Make zvol threadpool creation/destruction logic shared for
both Linux and FreeBSD.
The IO requests are processed asynchronously in next cases:
- volmode=geom: if IO request thread is geom thread or cannot sleep.
- volmode=cdev: if IO request passed thru struct cdevsw .d_strategy
routine, mean is AIO request.
In all other cases the IO requests are processed synchronously. The
volthreading zvol property is ignored on FreeBSD side.

Sponsored-by: vStack, Inc.
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: @ImAwsumm
Signed-off-by: Fedor Uporov <fuporov.vstack@gmail.com>
Closes #17169

tests/zfs-tests/tests/functional/zvol/zvol_common.shlib

@@ -140,3 +140,11 @@ function set_blk_mq
 		log_must set_tunable32 VOL_USE_BLK_MQ $1
 	fi
 }
+
+# enable/disable zvol sync mode
+#
+# $1: 1 = enable, 0 = disable
+function set_zvol_sync
+{
+	log_must set_tunable32 VOL_REQUEST_SYNC $1
+}

tests/zfs-tests/tests/functional/zvol/zvol_stress/zvol_stress.ksh

@@ -60,6 +60,9 @@ typeset -f each_zvol_size=$(( floor($biggest_zvol_size_possible * 0.9 / \
 
 typeset tmpdir="$(mktemp -t -d zvol_stress_fio_state.XXXXXX)"
 
+log_must save_tunable VOL_USE_BLK_MQ
+log_must save_tunable VOL_REQUEST_SYNC
+
 function create_zvols
 {
 	log_note "Creating $num_zvols zvols that are ${each_zvol_size}B each"
@@ -124,7 +127,8 @@ function cleanup
 	log_must zinject -c all
 	log_must zpool clear $TESTPOOL
 	destroy_zvols
-	set_blk_mq 0
+	log_must restore_tunable VOL_USE_BLK_MQ
+	log_must restore_tunable VOL_REQUEST_SYNC
 
 	# Remove all fio's leftover state files
 	if [ -n "$tmpdir" ] ; then
@@ -146,6 +150,18 @@ destroy_zvols
 set_blk_mq 1
 create_zvols
 do_zvol_stress
+destroy_zvols
+
+# Disable zvol sync mode, and re-run test
+set_zvol_sync 0
+create_zvols
+do_zvol_stress
+destroy_zvols
+
+# Same for enabled zvol sync mode
+set_zvol_sync 1
+create_zvols
+do_zvol_stress
 
 # Inject some errors, and verify we see some IO errors in zpool status
 sync_pool $TESTPOOL