OpenZFS 9185 - Enable testing over NFS in ZFS performance tests

This change makes additions to the ZFS test suite that allows the
performance tests to run over NFS. The test is run and performance data
collected from the server side, while IO is generated on the NFS client.

This has been tested with Linux and illumos NFS clients.

Authored by: Ahmed Ghanem <ahmedg@delphix.com>
Reviewed by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Reviewed by: Kevin Greene <kevin.greene@delphix.com>
Reviewed-by: Richard Elling <Richard.Elling@RichardElling.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Ported-by: John Kennedy <john.kennedy@delphix.com>
Signed-off-by: John Kennedy <john.kennedy@delphix.com>

OpenZFS-issue: https://www.illumos.org/issues/9185
Closes #8367

scripts/zfs-tests.sh

@@ -273,6 +273,7 @@ OPTIONS:
 	-f          Use files only, disables block device tests
 	-S          Enable stack tracer (negative performance impact)
 	-c          Only create and populate constrained path
+	-n NFSFILE  Use the nfsfile to determine the NFS configuration
 	-I NUM      Number of iterations
 	-d DIR      Use DIR for files and loopback devices
 	-s SIZE     Use vdevs of SIZE (default: 4G)
@@ -295,7 +296,7 @@ $0 -x
 EOF
 }
 
-while getopts 'hvqxkfScd:s:r:?t:T:u:I:' OPTION; do
+while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do
 	case $OPTION in
 	h)
 		usage
@@ -324,6 +325,12 @@ while getopts 'hvqxkfScd:s:r:?t:T:u:I:' OPTION; do
 		constrain_path
 		exit
 		;;
+	n)
+		nfsfile=$OPTARG
+		[[ -f $nfsfile ]] || fail "Cannot read file: $nfsfile"
+		export NFS=1
+		. "$nfsfile"
+		;;
 	d)
 		FILEDIR="$OPTARG"
 		;;

tests/zfs-tests/include/commands.cfg

@@ -100,6 +100,7 @@ export SYSTEM_FILES='arp
     readlink
     rm
     rmdir
+    scp
     sed
     seq
     setenforce
@@ -110,6 +111,7 @@ export SYSTEM_FILES='arp
     shuf
     sleep
     sort
+    ssh
     stat
     strings
     su

tests/zfs-tests/tests/perf/Makefile.am

@@ -1,5 +1,7 @@
 pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/perf
-dist_pkgdata_DATA = perf.shlib
+dist_pkgdata_SCRIPTS = \
+	nfs-sample.cfg \
+	perf.shlib
 
 SUBDIRS = \
 	fio \

tests/zfs-tests/tests/perf/fio/random_reads.fio

@@ -26,6 +26,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 
 [job]

tests/zfs-tests/tests/perf/fio/random_readwrite.fio

@@ -28,6 +28,7 @@ runtime=${RUNTIME}
 bssplit=4k/50:8k/30:128k/10:1m/10
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 buffer_compress_percentage=66
 buffer_compress_chunk=4096

tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio

@@ -28,6 +28,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 buffer_compress_percentage=66
 buffer_compress_chunk=4096

tests/zfs-tests/tests/perf/fio/random_writes.fio

@@ -25,6 +25,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
 buffer_compress_percentage=66

tests/zfs-tests/tests/perf/fio/sequential_reads.fio

@@ -26,6 +26,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 
 [job]

tests/zfs-tests/tests/perf/fio/sequential_writes.fio

@@ -25,6 +25,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
 buffer_compress_percentage=66

tests/zfs-tests/tests/perf/nfs-sample.cfg

@@ -0,0 +1,46 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2016 by Delphix. All rights reserved.
+#
+
+#
+# This file is a sample NFS configuration for the performance tests. To use the
+# performance tests over NFS you must have:
+#   - a client machine with fio and sudo installed
+#   - passwordless SSH set up from this host
+#     for delphix and root users to the client
+#   - passwordless sudo for the user on the client
+#
+
+
+# The IP address for the server
+export NFS_SERVER=127.0.0.1
+
+# The IP address for the client
+export NFS_CLIENT=127.0.0.1
+
+# The mountpoint to use inside the client
+export NFS_MOUNT=/var/tmp/nfs
+
+# The user to run the tests as on the client
+export NFS_USER=delphix
+
+# Common NFS client mount options
+export NFS_OPTIONS="-o rw,nosuid,bg,hard,rsize=1048576,wsize=1048576,"
+NFS_OPTIONS+="nointr,timeo=600,proto=tcp,actimeo=0,port=2049"
+
+# illumos NFS client mount options
+# export NFS_OPTIONS="$NFS_OPTIONS,vers=3"
+
+# Linux NFS client mount options
+export NFS_OPTIONS="-t nfs $NFS_OPTIONS,noacl,nfsvers=3"

tests/zfs-tests/tests/perf/perf.shlib

@@ -112,6 +112,16 @@ function do_fio_run_impl
 	export BLOCKSIZE=$iosize
 	sync
 
+	# When running locally, we want to keep the default behavior of
+	# DIRECT == 0, so only set it when we're running over NFS to
+	# disable client cache for reads.
+	if [[ $NFS -eq 1 ]]; then
+		export DIRECT=1
+		do_setup_nfs $script
+	else
+		export DIRECT=0
+	fi
+
 	# This will be part of the output filename.
 	typeset suffix=$(get_suffix $threads $sync $iosize)
 
@@ -124,7 +134,14 @@ function do_fio_run_impl
 	typeset outfile="$logbase.fio.$suffix"
 
 	# Start the load
-	log_must fio --output $outfile $FIO_SCRIPTS/$script
+	if [[ $NFS -eq 1 ]]; then
+		log_must ssh -t $NFS_USER@$NFS_CLIENT "
+			fio --output /tmp/fio.out /tmp/test.fio
+		"
+		log_must scp $NFS_USER@$NFS_CLIENT:/tmp/fio.out $outfile
+	else
+		log_must fio --output $outfile $FIO_SCRIPTS/$script
+	fi
 }
 
 #
@@ -166,6 +183,34 @@ function do_fio_run
 	done
 }
 
+# This function sets NFS mount on the client and make sure all correct
+# permissions are in place
+#
+function do_setup_nfs
+{
+	typeset script=$1
+	zfs  set sharenfs=on $TESTFS
+	log_must chmod  -R 777 /$TESTFS
+
+	ssh -t $NFS_USER@$NFS_CLIENT "mkdir -m 777 -p $NFS_MOUNT"
+	ssh -t $NFS_USER@$NFS_CLIENT "sudo -S umount $NFS_MOUNT"
+	log_must ssh -t $NFS_USER@$NFS_CLIENT "
+		sudo -S mount $NFS_OPTIONS $NFS_SERVER:/$TESTFS $NFS_MOUNT
+	"
+	#
+	# The variables in the fio script are only available in our current
+	# shell session, so we have to evaluate them here before copying
+	# the resulting script over to the target machine.
+	#
+	export jobnum='$jobnum'
+	while read line; do
+		eval echo "$line"
+	done < $FIO_SCRIPTS/$script > /tmp/test.fio
+	log_must sed -i -e "s%directory.*%directory=$NFS_MOUNT%" /tmp/test.fio
+	log_must scp /tmp/test.fio $NFS_USER@$NFS_CLIENT:/tmp
+	log_must rm /tmp/test.fio
+}
+
 #
 # This function iterates through the value pairs in $PERF_COLLECT_SCRIPTS.
 # The script at index N is launched in the background, with its output

tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh

@@ -60,6 +60,10 @@ elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
 fi
 
+# Until the performance tests over NFS can deal with multiple file systems,
+# force the use of only one file system when testing over NFS.
+[[ $NFS -eq 1 ]] && PERF_NTHREADS_PER_FS='0'
+
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then