Add support for user/group dnode accounting & quota

This patch tracks dnode usage for each user/group in the
DMU_USER/GROUPUSED_OBJECT ZAPs. ZAP entries dedicated to dnode
accounting have the key prefixed with "obj-" followed by the UID/GID
in string format (as done for the block accounting).
A new SPA feature has been added for dnode accounting as well as
a new ZPL version. The SPA feature must be enabled in the pool
before upgrading the zfs filesystem. During the zfs version upgrade,
a "quotacheck" will be executed by marking all dnode as dirty.

ZoL-bug-id: https://github.com/zfsonlinux/zfs/issues/3500

Signed-off-by: Jinshan Xiong <jinshan.xiong@intel.com>
Signed-off-by: Johann Lombardi <johann.lombardi@intel.com>

tests/zfs-tests/cmd/mkfiles/mkfiles.c

@@ -48,10 +48,7 @@ main(int argc, char **argv)
 	if (argc == 4 && sscanf(argv[3], "%u", &first_file) != 1)
 		usage("Invalid first file", -3);
 
-	if (numfiles < first_file)
-		usage("First file larger than last file", -3);
-
-	for (i = first_file; i <= numfiles; i++) {
+	for (i = first_file; i < first_file + numfiles; i++) {
 		int fd;
 		(void) snprintf(buf, MAXPATHLEN, "%s%u", argv[1], i);
 		if ((fd = open(buf, O_CREAT | O_EXCL, O_RDWR)) == -1) {

tests/zfs-tests/include/libtest.shlib

@@ -2857,3 +2857,29 @@ function block_device_wait
 		$UDEVADM settle
 	fi
 }
+
+#
+# Synchronize all the data in pool
+#
+# $1 pool name
+#
+function sync_pool #pool
+{
+	typeset pool=${1:-$TESTPOOL}
+
+	log_must $SYNC
+	log_must $SLEEP 2
+	# Flush all the pool data.
+	typeset -i ret
+	$ZPOOL scrub $pool >/dev/null 2>&1
+	ret=$?
+	(( $ret != 0 )) && \
+	log_fail "$ZPOOL scrub $pool failed."
+
+	while ! is_pool_scrubbed $pool; do
+		if is_pool_resilvered $pool ; then
+			log_fail "$pool should not be resilver completed."
+		fi
+		log_must $SLEEP 2
+	done
+}

tests/zfs-tests/tests/functional/Makefile.am

@@ -51,6 +51,7 @@ SUBDIRS = \
 	sparse \
 	threadsappend \
 	truncate \
+	upgrade \
 	userquota \
 	vdev_zaps \
 	write_dirs \

tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg

@@ -38,7 +38,8 @@ typeset -a properties=("size" "capacity" "altroot" "health" "guid" "version"
     "feature@large_blocks" "feature@large_dnode" "feature@filesystem_limits"
     "feature@spacemap_histogram" "feature@enabled_txg" "feature@hole_birth"
     "feature@extensible_dataset" "feature@bookmarks" "feature@embedded_data"
-    "feature@sha512" "feature@skein" "feature@edonr")
+    "feature@sha512" "feature@skein" "feature@edonr"
+    "feature@userobj_accounting")
 else
 typeset -a properties=("size" "capacity" "altroot" "health" "guid" "version"
     "bootfs" ""leaked" delegation" "autoreplace" "cachefile" "dedupditto" "dedupratio"

tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib

@@ -213,32 +213,6 @@ function get_vdevs #pool cnt
 	$ECHO "$vdevs"
 }
 
-#
-# Synchronize all the data in pool
-#
-# $1 pool name
-#
-function sync_pool #pool
-{
-	typeset pool=$1
-
-	log_must $SYNC
-	log_must $SLEEP 2
-	# Flush all the pool data.
-	typeset -i ret
-	$ZPOOL scrub $pool >/dev/null 2>&1
-	ret=$?
-	(( $ret != 0 )) && \
-		log_fail "$ZPOOL scrub $pool failed."
-
-	while ! is_pool_scrubbed $pool; do
-		if is_pool_resilvered $pool ; then
-			log_fail "$pool should not be resilver completed."
-		fi
-		log_must $SLEEP 2
-	done
-}
-
 #
 # Create and replace the same name virtual device files
 #

tests/zfs-tests/tests/functional/upgrade/Makefile.am

@@ -0,0 +1,5 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/upgrade
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	upgrade_userobj_001_pos.ksh

tests/zfs-tests/tests/functional/upgrade/cleanup.ksh

@@ -0,0 +1,44 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+#
+# Copyright (c) 2016 by Jinshan Xiong. No rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_must $ZPOOL destroy $TESTPOOL
+
+log_must $RM /tmp/zpool_upgrade_test.dat
+
+default_cleanup

tests/zfs-tests/tests/functional/upgrade/setup.ksh

@@ -0,0 +1,44 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+#
+# Copyright (c) 2016 by Jinshan Xiong. No rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+# create a pool without any features
+log_must $MKFILE 128m /tmp/zpool_upgrade_test.dat
+log_must $ZPOOL create -d -m $TESTDIR $TESTPOOL /tmp/zpool_upgrade_test.dat
+
+log_pass

tests/zfs-tests/tests/functional/upgrade/upgrade_userobj_001_pos.ksh

@@ -0,0 +1,98 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2013 by Jinshan Xiong. No rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+#
+# Check that zfs upgrade for object count accounting works.
+# Since userobjaccounting is a per dataset feature, this test case
+# will create multiple dataset and try different upgrade method.
+#
+# STRATEGY:
+# 1. Create a pool with all features disabled
+# 2. Create a few dataset for testing
+# 3. Make sure automatic upgrade work
+# 4. Make sure manual upgrade work
+#
+
+function cleanup
+{
+	datasetexists $TESTPOOL/fs1 && log_must $ZFS destroy $TESTPOOL/fs1
+	datasetexists $TESTPOOL/fs2 && log_must $ZFS destroy $TESTPOOL/fs2
+}
+
+verify_runnable "global"
+
+log_assert "pool upgrade for userobj accounting should work"
+log_onexit cleanup
+
+log_must $MKFILES $TESTDIR/tf $((RANDOM % 1000 + 1))
+log_must $ZFS create $TESTPOOL/fs1
+log_must $MKFILES $TESTDIR/fs1/tf $((RANDOM % 1000 + 1))
+log_must $ZFS create $TESTPOOL/fs2
+log_must $MKFILES $TESTDIR/fs2/tf $((RANDOM % 1000 + 1))
+log_must $ZFS umount $TESTPOOL/fs2
+
+# Make sure userobj accounting is disabled
+$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" ||
+	log_fail "userobj accounting should be disabled initially"
+
+# Upgrade zpool to support all features
+log_must $ZPOOL upgrade $TESTPOOL
+
+# Make sure userobj accounting is disabled again
+$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" ||
+	log_fail "userobj accounting should be disabled after pool upgrade"
+
+# Create a file in fs1 should trigger dataset upgrade
+log_must $MKFILE 1m $TESTDIR/fs1/tf
+sync_pool
+
+# Make sure userobj accounting is working for fs1
+$ZFS userspace -o objused -H $TESTPOOL/fs1 | $HEAD -n 1 | $GREP -q "-" &&
+	log_fail "userobj accounting should be enabled for $TESTPOOL/fs1"
+
+# Mount a dataset should trigger upgrade
+log_must $ZFS mount $TESTPOOL/fs2
+sync_pool
+
+# Make sure userobj accounting is working for fs2
+$ZFS userspace -o objused -H $TESTPOOL/fs2 | $HEAD -n 1 | $GREP -q "-" &&
+	log_fail "userobj accounting should be enabled for $TESTPOOL/fs2"
+
+# All in all, after having been through this, the dataset for testpool
+# still shouldn't be upgraded
+$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" ||
+	log_fail "userobj accounting should be disabled for $TESTPOOL"
+
+# Manual upgrade root dataset
+log_must $ZFS set version=current $TESTPOOL
+$ZFS userspace -o objused -H $TESTPOOL | $HEAD -n 1 | $GREP -q "-" &&
+	log_fail "userobj accounting should be enabled for $TESTPOOL"
+
+log_pass "all tests passed - what a lucky day!"

tests/zfs-tests/tests/functional/userquota/Makefile.am

@@ -6,6 +6,7 @@ dist_pkgdata_SCRIPTS = \
 	cleanup.ksh \
 	groupspace_001_pos.ksh \
 	groupspace_002_pos.ksh \
+	groupspace_003_pos.ksh \
 	userquota_001_pos.ksh \
 	userquota_002_pos.ksh \
 	userquota_003_pos.ksh \
@@ -18,5 +19,7 @@ dist_pkgdata_SCRIPTS = \
 	userquota_010_pos.ksh \
 	userquota_011_pos.ksh \
 	userquota_012_neg.ksh \
+	userquota_013_pos.ksh \
 	userspace_001_pos.ksh \
-	userspace_002_pos.ksh
+	userspace_002_pos.ksh \
+	userspace_003_pos.ksh

tests/zfs-tests/tests/functional/userquota/groupspace_003_pos.ksh

@@ -0,0 +1,103 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib
+
+#
+# DESCRIPTION:
+#       Check the user used and groupspace object counts in zfs groupspace
+#
+#
+# STRATEGY:
+#	1. set zfs groupquota to a fs
+#	2. create objects for different users in the same group
+#	3. use zfs groupspace to check the object count
+#
+
+function cleanup
+{
+	if datasetexists $snapfs; then
+		log_must $ZFS destroy $snapfs
+	fi
+
+	log_must $RM -f ${QFILE}_*
+	log_must cleanup_quota
+}
+
+function group_object_count
+{
+	typeset fs=$1
+	typeset user=$2
+	typeset cnt=$($ZFS groupspace -oname,objused $fs | $GREP $user |
+			$AWK '{print $2}')
+	echo $cnt
+}
+
+log_onexit cleanup
+
+log_assert "Check the zfs groupspace object used"
+
+mkmount_writable $QFS
+log_must $ZFS set xattr=sa $QFS
+
+((user1_cnt = RANDOM % 100 + 1))
+((user2_cnt = RANDOM % 100 + 1))
+log_must user_run $QUSER1 $MKFILES ${QFILE}_1 $user1_cnt
+log_must user_run $QUSER2 $MKFILES ${QFILE}_2 $user2_cnt
+((grp_cnt = user1_cnt + user2_cnt))
+sync_pool
+
+typeset snapfs=$QFS@snap
+
+log_must $ZFS snapshot $snapfs
+
+log_must eval "$ZFS groupspace $QFS >/dev/null 2>&1"
+log_must eval "$ZFS groupspace $snapfs >/dev/null 2>&1"
+
+for fs in "$QFS" "$snapfs"; do
+	log_note "check the object count in zfs groupspace $fs"
+        [[ $(group_object_count $fs $QGROUP) -eq $grp_cnt ]] ||
+                log_fail "expected $grp_cnt"
+done
+
+log_note "file removal"
+log_must $RM ${QFILE}_*
+sync_pool
+
+[[ $(group_object_count $QFS $QGROUP) -eq 0 ]] ||
+        log_fail "expected 0 files for $QGROUP"
+
+[[ $(group_object_count $snapfs $QGROUP) -eq $grp_cnt ]] ||
+        log_fail "expected $grp_cnt files for $QGROUP"
+
+cleanup
+log_pass "Check the zfs groupspace object used pass as expect"

tests/zfs-tests/tests/functional/userquota/userquota_001_pos.ksh

@@ -58,7 +58,7 @@ mkmount_writable $QFS
 log_note "Check the userquota@$QUSER1"
 log_must $ZFS set userquota@$QUSER1=$UQUOTA_SIZE $QFS
 log_must user_run $QUSER1 $MKFILE $UQUOTA_SIZE $QFILE
-$SYNC
+sync_pool
 log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE
 cleanup_quota
 
@@ -66,7 +66,7 @@ log_note "Check the groupquota@$QGROUP"
 log_must $ZFS set groupquota@$QGROUP=$GQUOTA_SIZE $QFS
 mkmount_writable $QFS
 log_must user_run $QUSER1 $MKFILE $GQUOTA_SIZE $QFILE
-$SYNC
+sync_pool
 log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE
 
 cleanup_quota

tests/zfs-tests/tests/functional/userquota/userquota_004_pos.ksh

@@ -50,6 +50,7 @@ log_onexit cleanup
 
 log_assert "Check the basic function of {user|group} used"
 
+sync_pool
 typeset user_used=$(get_value "userused@$QUSER1" $QFS)
 typeset group_used=$(get_value "groupused@$QGROUP" $QFS)
 
@@ -62,7 +63,7 @@ fi
 
 mkmount_writable $QFS
 log_must user_run $QUSER1 $MKFILE 100m $QFILE
-$SYNC
+sync_pool
 
 user_used=$(get_value "userused@$QUSER1" $QFS)
 group_used=$(get_value "groupused@$QGROUP" $QFS)

tests/zfs-tests/tests/functional/userquota/userquota_010_pos.ksh

@@ -57,7 +57,7 @@ log_must $ZFS set groupquota@$QGROUP=$GQUOTA_SIZE $QFS
 
 mkmount_writable $QFS
 log_must user_run $QUSER1 $MKFILE $UQUOTA_SIZE $QFILE
-$SYNC
+sync_pool
 
 log_must eval "$ZFS get -p userused@$QUSER1 $QFS >/dev/null 2>&1"
 log_must eval "$ZFS get -p groupused@$GROUPUSED $QFS >/dev/null 2>&1"

tests/zfs-tests/tests/functional/userquota/userquota_013_pos.ksh

@@ -0,0 +1,77 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2016 by Jinshan Xiong. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib
+
+#
+#
+# DESCRIPTION:
+#       Check the basic function of the userobjquota and groupobjquota
+#
+#
+# STRATEGY:
+#       1. Set userobjquota and overwrite the quota size
+#       2. Creating new object should fail with Disc quota exceeded
+#       3. Set groupobjquota and overwrite the quota size
+#       4. Creating new object should fail with Disc quota exceeded
+#
+#
+
+function cleanup
+{
+	log_must $RM -f ${QFILE}_*
+	cleanup_quota
+}
+
+log_onexit cleanup
+
+log_assert "If creating object exceeds {user|group}objquota count, it will fail"
+
+mkmount_writable $QFS
+log_must $ZFS set xattr=sa $QFS
+
+log_note "Check the userobjquota@$QUSER1"
+log_must $ZFS set userobjquota@$QUSER1=100 $QFS
+log_must user_run $QUSER1 $MKFILES ${QFILE}_1 100
+sync_pool
+log_mustnot user_run $QUSER1 $MKFILE 1 $OFILE
+cleanup_quota
+
+log_note "Check the groupobjquota@$QGROUP"
+log_must $ZFS set groupobjquota@$QGROUP=200 $QFS
+mkmount_writable $QFS
+log_must user_run $QUSER1 $MKFILES ${QFILE}_2 100
+sync_pool
+log_mustnot user_run $QUSER2 $MKFILE 1 $OFILE
+
+cleanup
+log_pass "Creating objects exceeds {user|group}objquota count, it as expect"

tests/zfs-tests/tests/functional/userquota/userquota_common.kshlib

@@ -38,8 +38,11 @@ function cleanup_quota
 {
 	if datasetexists $QFS; then
 		log_must $ZFS set userquota@$QUSER1=none $QFS
+		log_must $ZFS set userobjquota@$QUSER1=none $QFS
 		log_must $ZFS set userquota@$QUSER2=none $QFS
+		log_must $ZFS set userobjquota@$QUSER2=none $QFS
 		log_must $ZFS set groupquota@$QGROUP=none $QFS
+		log_must $ZFS set groupobjquota@$QGROUP=none $QFS
 		recovery_writable $QFS
 	fi
 
@@ -47,7 +50,7 @@ function cleanup_quota
 	[[ -f $OFILE ]] && log_must $RM -f $OFILE
 	$SYNC
 
-	 return 0
+	return 0
 }
 
 #

tests/zfs-tests/tests/functional/userquota/userspace_002_pos.ksh

@@ -75,7 +75,7 @@ for fs in "$QFS" "$snapfs"; do
 	log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 100M"
 
 	log_note "check the user used size in zfs userspace $fs"
-	log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 50.0M"
+	log_must eval "$ZFS userspace $fs | $GREP $QUSER1 | $GREP 50\\.\*M"
 done
 
 log_pass "Check the zfs userspace used and quota"

tests/zfs-tests/tests/functional/userquota/userspace_003_pos.ksh

@@ -0,0 +1,116 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2016 by Jinshan Xiong. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib
+
+#
+# DESCRIPTION:
+#       Check the user used object accounting in zfs userspace
+#
+#
+# STRATEGY:
+#       1. create a bunch of files by specific users
+#	2. use zfs userspace to check the used objects
+#	3. change the owner of test files and verify object count
+#	4. delete files and verify object count
+#
+
+function cleanup
+{
+	if datasetexists $snapfs; then
+		log_must $ZFS destroy $snapfs
+	fi
+
+	log_must $RM -f ${QFILE}_*
+	log_must cleanup_quota
+}
+
+function user_object_count
+{
+	typeset fs=$1
+	typeset user=$2
+	typeset cnt=$($ZFS userspace -oname,objused $fs |
+	    $AWK /$user/'{print $2}')
+	echo $cnt
+}
+
+log_onexit cleanup
+
+log_assert "Check the zfs userspace object used"
+
+mkmount_writable $QFS
+log_must $ZFS set xattr=sa $QFS
+
+((user1_cnt = RANDOM % 100 + 1))
+((user2_cnt = RANDOM % 100 + 1))
+
+log_must user_run $QUSER1 $MKFILES ${QFILE}_1 $user1_cnt
+log_must user_run $QUSER2 $MKFILES ${QFILE}_2 $user2_cnt
+sync_pool
+
+typeset snapfs=$QFS@snap
+
+log_must $ZFS snapshot $snapfs
+
+log_must eval "$ZFS userspace $QFS >/dev/null 2>&1"
+log_must eval "$ZFS userspace $snapfs >/dev/null 2>&1"
+
+for fs in "$QFS" "$snapfs"; do
+	log_note "check the user used objects in zfs userspace $fs"
+	[[ $(user_object_count $fs $QUSER1) -eq $user1_cnt ]] ||
+		log_fail "expected $user1_cnt"
+	[[ $(user_object_count $fs $QUSER2) -eq $user2_cnt ]] ||
+		log_fail "expected $user2_cnt"
+done
+
+log_note "change the owner of files"
+log_must $CHOWN $QUSER2 ${QFILE}_1*
+sync_pool
+
+[[ $(user_object_count $QFS $QUSER1) -eq 0 ]] ||
+	log_fail "expected 0 files for $QUSER1"
+
+[[ $(user_object_count $snapfs $QUSER1) -eq $user1_cnt ]] ||
+	log_fail "expected $user_cnt files for $QUSER1 in snapfs"
+
+[[ $(user_object_count $QFS $QUSER2) -eq $((user1_cnt+user2_cnt)) ]] ||
+	log_fail "expected $((user1_cnt+user2_cnt)) files for $QUSER2"
+
+log_note "file removal"
+log_must $RM ${QFILE}_*
+sync_pool
+
+[[ $(user_object_count $QFS $QUSER2) -eq 0 ]] ||
+        log_fail "expected 0 files for $QUSER2"
+
+cleanup
+log_pass "Check the zfs userspace object used"