Add port of FreeBSD 'volmode' property

The volmode property may be set to control the visibility of ZVOL
block devices.

This allow switching ZVOL between three modes:
   full - existing fully functional behaviour (default)
   dev  - hide partitions on ZVOL block devices
   none - not exposing volumes outside ZFS

Additionally the new zvol_volmode module parameter can be used to
control the default behaviour.

This functionality can be used, for instance, on "backup" pools to
avoid cluttering /dev with unneeded zd* devices.

Original-patch-by: mav <mav@FreeBSD.org>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Ported-by: loli10K <ezomori.nozomu@gmail.com>
Signed-off-by: loli10K <ezomori.nozomu@gmail.com>

FreeBSD-commit: https://github.com/freebsd/freebsd/commit/dd28e6bb
Closes #1796 
Closes #3438 
Closes #6233

tests/runfiles/linux.run

@@ -569,7 +569,7 @@ tests = ['zvol_cli_001_pos', 'zvol_cli_002_pos', 'zvol_cli_003_neg']
 [tests/functional/zvol/zvol_misc]
 tests = ['zvol_misc_001_neg', 'zvol_misc_002_pos', 'zvol_misc_003_neg',
     'zvol_misc_004_pos', 'zvol_misc_005_neg', 'zvol_misc_006_pos',
-    'zvol_misc_snapdev']
+    'zvol_misc_snapdev', 'zvol_misc_volmode']
 
 [tests/functional/zvol/zvol_swap]
 tests = ['zvol_swap_001_pos', 'zvol_swap_002_pos', 'zvol_swap_003_pos',

tests/zfs-tests/include/libtest.shlib

@@ -748,7 +748,7 @@ function zero_partitions #<whole_disk_name>
 	else
 		for i in 0 1 3 4 5 6 7
 		do
-			set_partition $i "" 0mb $diskname
+			log_must set_partition $i "" 0mb $diskname
 		done
 	fi
 
@@ -788,7 +788,11 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
 		parted $DEV_DSKDIR/$disk -s -- print 1 >/dev/null
 		typeset ret_val=$?
 		if [[ $slicenum -eq 0 || $ret_val -ne 0 ]]; then
-			log_must parted $DEV_DSKDIR/$disk -s -- mklabel gpt
+			parted $DEV_DSKDIR/$disk -s -- mklabel gpt
+			if [[ $? -ne 0 ]]; then
+				log_note "Failed to create GPT partition table on $disk"
+				return 1
+			fi
 		fi
 
 		# When no start is given align on the first cylinder.
@@ -804,8 +808,12 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
 			awk -F '[:k.]' '{print $4}')
 		((end = (size_mb * 1024 / cly_size_kb) + start))
 
-		log_must parted $DEV_DSKDIR/$disk -s -- \
+		parted $DEV_DSKDIR/$disk -s -- \
 		    mkpart part$slicenum ${start}cyl ${end}cyl
+		if [[ $? -ne 0 ]]; then
+			log_note "Failed to create partition $slicenum on $disk"
+			return 1
+		fi
 
 		blockdev --rereadpt $DEV_DSKDIR/$disk 2>/dev/null
 		block_device_wait
@@ -828,8 +836,10 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
 
 	typeset ret_val=$?
 	rm -f $format_file
-	[[ $ret_val -ne 0 ]] && \
-	    log_fail "Unable to format $disk slice $slicenum to $size"
+	if [[ $ret_val -ne 0 ]]; then
+		log_note "Unable to format $disk slice $slicenum to $size"
+		return 1
+	fi
 	return 0
 }
 
@@ -959,7 +969,7 @@ function partition_disk	#<slice_size> <whole_disk_name>	<total_slices>
 				continue
 			fi
 		fi
-		set_partition $i "$cyl" $slice_size $disk_name
+		log_must set_partition $i "$cyl" $slice_size $disk_name
 		cyl=$(get_endslice $disk_name $i)
 		((i = i+1))
 	done

tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_011_neg.ksh

@@ -81,13 +81,13 @@ vfstab_dev=$(find_vfstab_dev)
 if is_linux; then
 	partition_disk $SIZE $disk 7
 	cyl=$(get_endslice $disk $SLICE5)
-	set_partition $SLICE6 "$cyl" $SIZE1 $disk
+	log_must set_partition $SLICE6 "$cyl" $SIZE1 $disk
 else
 	specified_dump_dev=${disk}${SLICE_PREFIX}${SLICE0}
 	saved_dump_dev=$(save_dump_dev)
 
 	cyl=$(get_endslice $disk $SLICE6)
-	set_partition $SLICE7 "$cyl" $SIZE1 $disk
+	log_must set_partition $SLICE7 "$cyl" $SIZE1 $disk
 fi
 create_pool "$TESTPOOL" "$pooldev1"
 

tests/zfs-tests/tests/functional/zvol/zvol_misc/Makefile.am

@@ -1,5 +1,6 @@
 pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/zvol/zvol_misc
 dist_pkgdata_SCRIPTS = \
+	zvol_misc_common.kshlib \
 	cleanup.ksh \
 	setup.ksh \
 	zvol_misc_001_neg.ksh \
@@ -8,4 +9,5 @@ dist_pkgdata_SCRIPTS = \
 	zvol_misc_004_pos.ksh \
 	zvol_misc_005_neg.ksh \
 	zvol_misc_006_pos.ksh \
-	zvol_misc_snapdev.ksh
+	zvol_misc_snapdev.ksh \
+	zvol_misc_volmode.ksh

tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib

@@ -0,0 +1,141 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2017, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
+. $STF_SUITE/tests/functional/zvol/zvol_common.shlib
+
+#
+# Wait for udev to settle, completely.
+# This is quite discomforting, but there's a race condition here
+# (Amazon 2015.09 x86_64 Release (TEST) is good at triggering this)  where the
+# kernel tries to remove zvol device nodes while they're open by [blkid],
+# [zvol_id] or other udev related processes.
+# Calling 'udevadm settle' is not enough: wait for those processes "manually".
+#
+function udev_wait
+{
+	sleep 1
+	udevadm trigger --action=change
+	udevadm settle
+	for i in {1..3}; do
+		blkid="$(pgrep blkid | wc -l)"
+		zvol_id="$(pgrep zvol_id | wc -l)"
+		[[ "0" == "$zvol_id" && "0" == "$blkid" ]] && return
+		udevadm settle
+	done
+	log_fail "Wait timeout reached for udev_wait"
+}
+
+#
+# Clean up udev status
+# This is also a problem on "Amazon 2015.09 x86_64 Release (TEST)" where udev,
+# sometimes, does not clean up /dev/zvol symlinks correctly for removed ZVOLs.
+# Prune those links manually, then tell udev to forget them.
+#
+function udev_cleanup
+{
+	log_note "Pruning broken ZVOL symlinks ..."
+	udevadm settle
+	# find all dangling links and delete them
+	find -L "${ZVOL_DEVDIR}" -type l -print -delete
+	# purge those links from udev database
+	udevadm info --cleanup-db
+}
+
+#
+# Verify $device exists and is a block device
+#
+function blockdev_exists # device
+{
+	typeset device="$1"
+
+	# we wait here instead of doing it in a wrapper around 'zfs set snapdev'
+	# because there are other commands (zfs snap, zfs inherit, zfs destroy)
+	# that can affect device nodes
+	for i in {1..3}; do
+		udev_wait
+		[[ -b "$device" ]] && return 0
+	done
+	log_fail "$device does not exist as a block device"
+}
+
+#
+# Verify $device does not exist
+#
+function blockdev_missing # device
+{
+	typeset device="$1"
+
+	# we wait here instead of doing it in a wrapper around 'zfs set snapdev'
+	# because there are other commands (zfs snap, zfs inherit, zfs destroy)
+	# that can affect device nodes
+	for i in {1..3}; do
+		udev_wait
+		[[ ! -e "$device" ]] && return 0
+	done
+	log_fail "$device exists when not expected"
+}
+
+#
+# Verify $property on $dataset is inherited by $parent and is set to $value
+#
+function verify_inherited # property value dataset parent
+{
+	typeset property="$1"
+	typeset value="$2"
+	typeset dataset="$3"
+	typeset parent="$4"
+
+	typeset val=$(get_prop "$property" "$dataset")
+	typeset src=$(get_source "$property" "$dataset")
+	if [[ "$val" != "$value" || "$src" != "inherited from $parent" ]]; then
+		log_fail "Dataset $dataset did not inherit $property properly:"\
+		    "expected=$value, value=$val, source=$src."
+	fi
+}
+
+#
+# Create a small partition on $device, then verify if we can access it
+#
+function verify_partition # device
+{
+	typeset device="$1"
+
+	if [[ ! -b "$device" ]]; then
+		log_fail "$device is not a block device"
+	fi
+	# create a small dummy partition
+	set_partition 0 1 1m $device
+	# verify we can access the partition on the device
+	devname="$(readlink -f "$device")"
+	if is_linux; then
+		[[ -b "$devname""p1" ]]
+	else
+		[[ -b "$devname""s0" ]]
+	fi
+	return $?
+}

tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh

@@ -27,6 +27,7 @@
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
 . $STF_SUITE/tests/functional/zvol/zvol_common.shlib
+. $STF_SUITE/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
 
 #
 # DESCRIPTION:
@@ -46,60 +47,7 @@ function cleanup
 	datasetexists $ZVOL && log_must zfs destroy -r $ZVOL
 	log_must zfs inherit snapdev $TESTPOOL
 	block_device_wait
-}
-
-#
-# Verify $device exists and is a block device
-#
-function blockdev_exists # device
-{
-	typeset device="$1"
-
-	# we wait here instead of doing it in a wrapper around 'zfs set snapdev'
-	# because there are other commands (zfs snap, zfs inherit, zfs destroy)
-	# that can affect device nodes
-	block_device_wait
-
-	if [[ ! -b "$device" ]]; then
-		log_fail "$device does not exist as a block device"
-	fi
-}
-
-#
-# Verify $device does not exist
-#
-function check_missing # device
-{
-	typeset device="$1"
-
-	# we wait here instead of doing it in a wrapper around 'zfs set snapdev'
-	# because there are other commands (zfs snap, zfs inherit, zfs destroy)
-	# that can affect device nodes
-	block_device_wait
-
-	if [[ -e "$device" ]]; then
-		log_fail "$device exists when not expected"
-	fi
-}
-
-#
-# Verify $property on $dataset is inherited by $parent and is set to $value
-#
-function verify_inherited # property value dataset parent
-{
-	typeset property="$1"
-	typeset value="$2"
-	typeset dataset="$3"
-	typeset parent="$4"
-
-	typeset val=$(get_prop "$property" "$dataset")
-	typeset src=$(get_source "$property" "$dataset")
-	if [[ "$val" != "$value" || "$src" != "inherited from $parent" ]]
-	then
-		log_fail "Dataset $dataset did not inherit $property properly:"\
-		    "expected=$value, value=$val, source=$src."
-	fi
-
+	udev_cleanup
 }
 
 log_assert "Verify that ZFS volume property 'snapdev' works as expected."
@@ -130,14 +78,14 @@ log_must zfs snapshot $SNAP
 log_must zfs set snapdev=visible $ZVOL
 blockdev_exists $SNAPDEV
 log_must zfs set snapdev=hidden $ZVOL
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 log_must zfs destroy $SNAP
 # 2.2 First set snapdev property then create a snapshot
 log_must zfs set snapdev=visible $ZVOL
 log_must zfs snapshot $SNAP
 blockdev_exists $SNAPDEV
 log_must zfs destroy $SNAP
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 # 2.3 Verify setting to the same value multiple times does not lead to issues
 log_must zfs snapshot $SNAP
 log_must zfs set snapdev=visible $ZVOL
@@ -145,9 +93,9 @@ blockdev_exists $SNAPDEV
 log_must zfs set snapdev=visible $ZVOL
 blockdev_exists $SNAPDEV
 log_must zfs set snapdev=hidden $ZVOL
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 log_must zfs set snapdev=hidden $ZVOL
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 log_must zfs destroy $SNAP
 
 # 3. Verify "snapdev" is inherited correctly
@@ -160,14 +108,14 @@ blockdev_exists $SNAPDEV
 # 3.2 Check snapdev=hidden case
 log_must zfs set snapdev=hidden $TESTPOOL
 verify_inherited 'snapdev' 'hidden' $ZVOL $TESTPOOL
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 # 3.3 Check inheritance on multiple levels
 log_must zfs snapshot $SUBSNAP
 log_must zfs inherit snapdev $SUBZVOL
 log_must zfs set snapdev=hidden $VOLFS
 log_must zfs set snapdev=visible $TESTPOOL
 verify_inherited 'snapdev' 'hidden' $SUBZVOL $VOLFS
-check_missing $SUBSNAPDEV
+blockdev_missing $SUBSNAPDEV
 blockdev_exists $SNAPDEV
 
 log_pass "ZFS volume property 'snapdev' works as expected"

tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh

@@ -0,0 +1,225 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2017, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
+. $STF_SUITE/tests/functional/zvol/zvol_common.shlib
+. $STF_SUITE/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
+
+#
+# DESCRIPTION:
+# Verify that ZFS volume property "volmode" works as intended.
+#
+# STRATEGY:
+# 1. Verify "volmode" property does not accept invalid values
+# 2. Verify "volmode=none" hides ZVOL device nodes
+# 3. Verify "volmode=full" exposes a fully functional device
+# 4. Verify "volmode=dev" hides partition info on the device
+# 5. Verify "volmode=default" behaves accordingly to "volmode" module parameter
+# 6. Verify "volmode" property is inherited correctly
+# 7. Verify "volmode" behaves accordingly to zvol_inhibit_dev (Linux only)
+#
+# NOTE: changing volmode may need to remove minors, which could be open, so call
+#       udev_wait() before we "zfs set volmode=<value>".
+
+verify_runnable "global"
+
+function cleanup
+{
+	datasetexists $VOLFS && log_must_busy zfs destroy -r $VOLFS
+	datasetexists $ZVOL && log_must_busy zfs destroy -r $ZVOL
+	log_must zfs inherit volmode $TESTPOOL
+	udev_wait
+	sysctl_inhibit_dev 0
+	udev_cleanup
+}
+
+#
+# Set zvol_inhibit_dev tunable to $value
+#
+function sysctl_inhibit_dev # value
+{
+	typeset value="$1"
+
+	if is_linux; then
+		log_note "Setting zvol_inhibit_dev tunable to $value"
+		log_must eval "echo $value > "\
+		    "/sys/module/zfs/parameters/zvol_inhibit_dev"
+	fi
+}
+
+#
+# Set volmode tunable to $value
+#
+function sysctl_volmode # value
+{
+	typeset value="$1"
+
+	log_note "Setting volmode tunable to $value"
+	if is_linux; then
+		echo "$value" > '/sys/module/zfs/parameters/zvol_volmode'
+	else
+		sysctl 'vfs.zfs.vol.mode' "$value"
+	fi
+	if [[ $? -ne 0 ]]; then
+		log_fail "Unable to set volmode tunable to $value"
+	fi
+}
+
+log_assert "Verify that ZFS volume property 'volmode' works as intended"
+log_onexit cleanup
+
+VOLFS="$TESTPOOL/volfs"
+ZVOL="$TESTPOOL/vol"
+ZDEV="${ZVOL_DEVDIR}/$ZVOL"
+SUBZVOL="$VOLFS/subvol"
+SUBZDEV="${ZVOL_DEVDIR}/$SUBZVOL"
+
+log_must zfs create -o mountpoint=none $VOLFS
+log_must zfs create -V $VOLSIZE -s $SUBZVOL
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+
+# 1. Verify "volmode" property does not accept invalid values
+typeset badvals=("off" "on" "1" "nope" "-")
+for badval in ${badvals[@]}
+do
+	log_mustnot zfs set volmode="$badval" $ZVOL
+done
+
+# 2. Verify "volmode=none" hides ZVOL device nodes
+log_must zfs set volmode=none $ZVOL
+blockdev_missing $ZDEV
+log_must_busy zfs destroy $ZVOL
+
+# 3. Verify "volmode=full" exposes a fully functional device
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=full $ZVOL
+blockdev_exists $ZDEV
+log_must verify_partition $ZDEV
+udev_wait
+# 3.1 Verify "volmode=geom" is an alias for "volmode=full"
+log_must zfs set volmode=geom $ZVOL
+blockdev_exists $ZDEV
+if [[ "$(get_prop 'volmode' $ZVOL)" != "full" ]]; then
+	log_fail " Volmode value 'geom' is not an alias for 'full'"
+fi
+udev_wait
+log_must_busy zfs destroy $ZVOL
+
+# 4. Verify "volmode=dev" hides partition info on the device
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=dev $ZVOL
+blockdev_exists $ZDEV
+log_mustnot verify_partition $ZDEV
+udev_wait
+log_must_busy zfs destroy $ZVOL
+
+# 5. Verify "volmode=default" behaves accordingly to "volmode" module parameter
+# 5.1 Verify sysctl "volmode=full"
+sysctl_volmode 1
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=default $ZVOL
+blockdev_exists $ZDEV
+log_must verify_partition $ZDEV
+udev_wait
+log_must_busy zfs destroy $ZVOL
+# 5.2 Verify sysctl "volmode=dev"
+sysctl_volmode 2
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=default $ZVOL
+blockdev_exists $ZDEV
+log_mustnot verify_partition $ZDEV
+udev_wait
+log_must_busy zfs destroy $ZVOL
+# 5.2 Verify sysctl "volmode=none"
+sysctl_volmode 3
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=default $ZVOL
+blockdev_missing $ZDEV
+
+# 6. Verify "volmode" property is inherited correctly
+log_must zfs inherit volmode $ZVOL
+# 6.1 Check volmode=full case
+log_must zfs set volmode=full $TESTPOOL
+verify_inherited 'volmode' 'full' $ZVOL $TESTPOOL
+blockdev_exists $ZDEV
+# 6.2 Check volmode=none case
+log_must zfs set volmode=none $TESTPOOL
+verify_inherited 'volmode' 'none' $ZVOL $TESTPOOL
+blockdev_missing $ZDEV
+# 6.3 Check volmode=dev case
+log_must zfs set volmode=dev $TESTPOOL
+verify_inherited 'volmode' 'dev' $ZVOL $TESTPOOL
+blockdev_exists $ZDEV
+# 6.4 Check volmode=default case
+sysctl_volmode 1
+log_must zfs set volmode=default $TESTPOOL
+verify_inherited 'volmode' 'default' $ZVOL $TESTPOOL
+blockdev_exists $ZDEV
+# 6.5 Check inheritance on multiple levels
+log_must zfs inherit volmode $SUBZVOL
+udev_wait
+log_must zfs set volmode=none $VOLFS
+udev_wait
+log_must zfs set volmode=full $TESTPOOL
+verify_inherited 'volmode' 'none' $SUBZVOL $VOLFS
+blockdev_missing $SUBZDEV
+blockdev_exists $ZDEV
+log_must_busy zfs destroy $ZVOL
+log_must_busy zfs destroy $SUBZVOL
+
+# 7. Verify "volmode" behaves accordingly to zvol_inhibit_dev (Linux only)
+if is_linux; then
+	sysctl_inhibit_dev 1
+	# 7.1 Verify device nodes not are not created with "volmode=full"
+	sysctl_volmode 1
+	log_must zfs create -V $VOLSIZE -s $ZVOL
+	blockdev_missing $ZDEV
+	log_must zfs set volmode=full $ZVOL
+	blockdev_missing $ZDEV
+	log_must_busy zfs destroy $ZVOL
+	# 7.1 Verify device nodes not are not created with "volmode=dev"
+	sysctl_volmode 2
+	log_must zfs create -V $VOLSIZE -s $ZVOL
+	blockdev_missing $ZDEV
+	log_must zfs set volmode=dev $ZVOL
+	blockdev_missing $ZDEV
+	log_must_busy zfs destroy $ZVOL
+	# 7.1 Verify device nodes not are not created with "volmode=none"
+	sysctl_volmode 3
+	log_must zfs create -V $VOLSIZE -s $ZVOL
+	blockdev_missing $ZDEV
+	log_must zfs set volmode=none $ZVOL
+	blockdev_missing $ZDEV
+fi
+
+log_pass "Verify that ZFS volume property 'volmode' works as intended"