zvol: Implement zvol threading as a Property

Currently, zvol threading can be switched through the zvol_request_sync
module parameter system-wide. By making it a zvol property, zvol
threading can be switched per zvol.

Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Signed-off-by: Ameer Hamza <ahamza@ixsystems.com>
Closes #15409

include/sys/fs/zfs.h

@@ -192,6 +192,7 @@ typedef enum {
 	ZFS_PROP_REDACT_SNAPS,
 	ZFS_PROP_SNAPSHOTS_CHANGED,
 	ZFS_PROP_PREFETCH,
+	ZFS_PROP_VOLTHREADING,
 	ZFS_NUM_PROPS
 } zfs_prop_t;
 

include/sys/zvol.h

@@ -50,6 +50,7 @@ extern int zvol_get_stats(objset_t *, nvlist_t *);
 extern boolean_t zvol_is_zvol(const char *);
 extern void zvol_create_cb(objset_t *, void *, cred_t *, dmu_tx_t *);
 extern int zvol_set_volsize(const char *, uint64_t);
+extern int zvol_set_volthreading(const char *, boolean_t);
 extern int zvol_set_common(const char *, zfs_prop_t, zprop_source_t, uint64_t);
 extern zvol_state_handle_t *zvol_suspend(const char *);
 extern int zvol_resume(zvol_state_handle_t *);

include/sys/zvol_impl.h

@@ -58,6 +58,7 @@ typedef struct zvol_state {
 	atomic_t		zv_suspend_ref;	/* refcount for suspend */
 	krwlock_t		zv_suspend_lock;	/* suspend lock */
 	struct zvol_state_os	*zv_zso;	/* private platform state */
+	boolean_t		zv_threading;	/* volthreading property */
 } zvol_state_t;
 
 

lib/libzfs/libzfs.abi

@@ -1868,7 +1868,8 @@
       <enumerator name='ZFS_PROP_REDACT_SNAPS' value='94'/>
       <enumerator name='ZFS_PROP_SNAPSHOTS_CHANGED' value='95'/>
       <enumerator name='ZFS_PROP_PREFETCH' value='96'/>
-      <enumerator name='ZFS_NUM_PROPS' value='97'/>
+      <enumerator name='ZFS_PROP_VOLTHREADING' value='97'/>
+      <enumerator name='ZFS_NUM_PROPS' value='98'/>
     </enum-decl>
     <typedef-decl name='zfs_prop_t' type-id='4b000d60' id='58603c44'/>
     <enum-decl name='zprop_source_t' naming-typedef-id='a2256d42' id='5903f80e'>

man/man7/zfsprops.7

@@ -1197,6 +1197,18 @@ are equivalent to the
 and
 .Sy noexec
 mount options.
+.It Sy volthreading Ns = Ns Sy on Ns | Ns Sy off
+Controls internal zvol threading.
+The value
+.Sy off
+disables zvol threading, and zvol relies on application threads.
+The default value is
+.Sy on ,
+which enables threading within a zvol.
+Please note that this property will be overridden by
+.Sy zvol_request_sync
+module parameter.
+This property is only applicable to Linux.
 .It Sy filesystem_limit Ns = Ns Ar count Ns | Ns Sy none
 Limits the number of filesystems and volumes that can exist under this point in
 the dataset tree.

module/os/linux/zfs/zvol_os.c

@@ -512,7 +512,7 @@ zvol_request_impl(zvol_state_t *zv, struct bio *bio, struct request *rq,
 	uint64_t size = io_size(bio, rq);
 	int rw = io_data_dir(bio, rq);
 
-	if (zvol_request_sync)
+	if (zvol_request_sync || zv->zv_threading == B_FALSE)
 		force_sync = 1;
 
 	zv_request_t zvr = {
@@ -1304,6 +1304,7 @@ zvol_os_create_minor(const char *name)
 	int error = 0;
 	int idx;
 	uint64_t hash = zvol_name_hash(name);
+	uint64_t volthreading;
 	bool replayed_zil = B_FALSE;
 
 	if (zvol_inhibit_dev)
@@ -1350,6 +1351,12 @@ zvol_os_create_minor(const char *name)
 	zv->zv_volsize = volsize;
 	zv->zv_objset = os;
 
+	/* Default */
+	zv->zv_threading = B_TRUE;
+	if (dsl_prop_get_integer(name, "volthreading", &volthreading, NULL)
+	    == 0)
+		zv->zv_threading = volthreading;
+
 	set_capacity(zv->zv_zso->zvo_disk, zv->zv_volsize >> 9);
 
 	blk_queue_max_hw_sectors(zv->zv_zso->zvo_queue,

module/zcommon/zfs_prop.c

@@ -628,6 +628,9 @@ zfs_prop_init(void)
 	    ZVOL_DEFAULT_BLOCKSIZE, PROP_ONETIME,
 	    ZFS_TYPE_VOLUME, "512 to 128k, power of 2",	"VOLBLOCK", B_FALSE,
 	    sfeatures);
+	zprop_register_index(ZFS_PROP_VOLTHREADING, "volthreading",
+	    1, PROP_DEFAULT, ZFS_TYPE_VOLUME, "on | off", "zvol threading",
+	    boolean_table, sfeatures);
 	zprop_register_number(ZFS_PROP_USEDSNAP, "usedbysnapshots", 0,
 	    PROP_READONLY, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, "<size>",
 	    "USEDSNAP", B_FALSE, sfeatures);

module/zfs/zfs_ioctl.c

@@ -2522,6 +2522,15 @@ zfs_prop_set_special(const char *dsname, zprop_source_t source,
 	case ZFS_PROP_VOLSIZE:
 		err = zvol_set_volsize(dsname, intval);
 		break;
+	case ZFS_PROP_VOLTHREADING:
+		err = zvol_set_volthreading(dsname, intval);
+		/*
+		 * Set err to -1 to force the zfs_set_prop_nvlist code down the
+		 * default path to set the value in the nvlist.
+		 */
+		if (err == 0)
+			err = -1;
+		break;
 	case ZFS_PROP_SNAPDEV:
 	case ZFS_PROP_VOLMODE:
 		err = zvol_set_common(dsname, prop, source, intval);

module/zfs/zvol.c

@@ -369,6 +369,20 @@ out:
 	return (SET_ERROR(error));
 }
 
+/*
+ * Update volthreading.
+ */
+int
+zvol_set_volthreading(const char *name, boolean_t value)
+{
+	zvol_state_t *zv = zvol_find_by_name(name, RW_NONE);
+	if (zv == NULL)
+		return (ENOENT);
+	zv->zv_threading = value;
+	mutex_exit(&zv->zv_state_lock);
+	return (0);
+}
+
 /*
  * Sanity check volume block size.
  */