Reset l2ad_hand and l2ad_first in l2arc_evict

Increasing l2arc_write_size or l2arc_write_boost can result in
l2arc_write_buffers() not having enough space to perform its writes and
panic zio_write_phys().

Instead of resetting l2ad_hand to l2ad_start at the end of
l2arc_write_buffers() and not taking into account a possible
user-mediated increase of l2arc_write_max, we do this in l2arc_evict(),
right after l2arc_write_size() has run. If there is not enough space to
evict (ie we will exceed l2ad_end) we evict to the end of the device,
reset l2ad_hand to l2ad_start, set l2ad_first to 0 and iterate
l2arc_evict(). We avoid infinite iteration of l2arc_evict() by making
sure in l2arc_write_size() that l2ad_start + size does not exceed
l2ad_end.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Ryan Moeller <ryan@iXsystems.com>
Signed-off-by: George Amanakis <gamanakis@gmail.com>
Closes #10154

tests/runfiles/common.run

@@ -49,7 +49,7 @@ post =
 [tests/functional/cache]
 tests = ['cache_001_pos', 'cache_002_pos', 'cache_003_pos', 'cache_004_neg',
     'cache_005_neg', 'cache_006_pos', 'cache_007_neg', 'cache_008_neg',
-    'cache_009_pos', 'cache_010_neg', 'cache_011_pos']
+    'cache_009_pos', 'cache_010_neg', 'cache_011_pos', 'cache_012_pos']
 tags = ['functional', 'cache']
 
 [tests/functional/cachefile]

tests/zfs-tests/include/libtest.shlib

@@ -4053,3 +4053,16 @@ function ls_xattr # path
 		;;
 	esac
 }
+
+function get_arcstat # stat
+{
+	if is_linux; then
+		typeset stat=$1
+		typeset zfs_arcstats="/proc/spl/kstat/zfs/arcstats"
+		[[ -f "$zfs_arcstats" ]] || return 1
+		grep $stat $zfs_arcstats | awk '{print $3}'
+		return $?
+	else
+		return 1
+	fi
+}

tests/zfs-tests/include/tunables.cfg

@@ -35,6 +35,9 @@ DISABLE_IVSET_GUID_CHECK	disable_ivset_guid_check	zfs_disable_ivset_guid_check
 INITIALIZE_CHUNK_SIZE		initialize_chunk_size		zfs_initialize_chunk_size
 INITIALIZE_VALUE		initialize_value		zfs_initialize_value
 KEEP_LOG_SPACEMAPS_AT_EXPORT	keep_log_spacemaps_at_export	zfs_keep_log_spacemaps_at_export
+L2ARC_NOPREFETCH		l2arc.noprefetch		l2arc_noprefetch
+L2ARC_WRITE_BOOST		l2arc.write_boost		l2arc_write_boost
+L2ARC_WRITE_MAX			l2arc.write_max			l2arc_write_max
 LIVELIST_CONDENSE_NEW_ALLOC	livelist.condense.new_alloc	zfs_livelist_condense_new_alloc
 LIVELIST_CONDENSE_SYNC_CANCEL	livelist.condense.sync_cancel	zfs_livelist_condense_sync_cancel
 LIVELIST_CONDENSE_SYNC_PAUSE	livelist.condense.sync_pause	zfs_livelist_condense_sync_pause

tests/zfs-tests/tests/functional/cache/Makefile.am

@@ -12,7 +12,8 @@ dist_pkgdata_SCRIPTS = \
 	cache_008_neg.ksh \
 	cache_009_pos.ksh \
 	cache_010_neg.ksh \
-	cache_011_pos.ksh
+	cache_011_pos.ksh \
+	cache_012_pos.ksh
 
 dist_pkgdata_DATA = \
 	cache.cfg \

tests/zfs-tests/tests/functional/cache/cache_012_pos.ksh

@@ -0,0 +1,110 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/cache/cache.cfg
+. $STF_SUITE/tests/functional/cache/cache.kshlib
+
+#
+# DESCRIPTION:
+#	Looping around a cache device with l2arc_write_size exceeding
+#	the device size succeeds.
+#
+# STRATEGY:
+#	1. Create pool with a cache device.
+#	2. Set l2arc_write_max to a value larger than the cache device.
+#	3. Create a file larger than the cache device and random read
+#		for 10 sec.
+#	4. Verify that l2arc_write_max is set back to the default.
+#	5. Set l2arc_write_max to a value less than the cache device size but
+#		larger than the default (64MB).
+#	6. Record the l2_size.
+#	7. Random read for 1 sec.
+#	8. Record the l2_size again.
+#	9. If (6) <= (8) then we have not looped around yet.
+#	10. If (6) > (8) then we looped around. Break out of the loop and test.
+#	11. Destroy pool.
+#
+
+verify_runnable "global"
+
+log_assert "Looping around a cache device succeeds."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_WRITE_MAX $write_max
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+}
+log_onexit cleanup
+
+typeset write_max=$(get_tunable L2ARC_WRITE_MAX)
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+
+typeset VDEV="$VDIR/vdev.disk"
+typeset VDEV_SZ=$(( 4 * 1024 * 1024 * 1024 ))
+typeset VCACHE="$VDIR/vdev.cache"
+typeset VCACHE_SZ=$(( $VDEV_SZ / 2 ))
+
+typeset fill_mb=$(( floor($VDEV_SZ * 3 / 4 ) ))
+export DIRECTORY=/$TESTPOOL
+export NUMJOBS=4
+export RUNTIME=10
+export PERF_RANDSEED=1234
+export PERF_COMPPERCENT=66
+export PERF_COMPCHUNK=0
+export BLOCKSIZE=128K
+export SYNC_TYPE=0
+export DIRECT=1
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))
+
+log_must set_tunable32 L2ARC_WRITE_MAX $(( $VCACHE_SZ * 2 ))
+
+log_must truncate -s $VCACHE_SZ $VCACHE
+log_must truncate -s $VDEV_SZ $VDEV
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VCACHE
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+typeset write_max2=$(get_tunable L2ARC_WRITE_MAX)
+
+log_must test $write_max2 -eq $write_max
+
+log_must set_tunable32 L2ARC_WRITE_MAX $(( 64 * 1024 * 1024 ))
+export RUNTIME=1
+
+typeset do_once=true
+while $do_once || [[ $l2_size1 -le $l2_size2 ]]; do
+	typeset l2_size1=$(get_arcstat l2_size)
+	log_must fio $FIO_SCRIPTS/random_reads.fio
+	typeset l2_size2=$(get_arcstat l2_size)
+	do_once=false
+done
+
+log_must test $l2_size1 -gt $l2_size2
+
+log_must zpool destroy $TESTPOOL
+
+log_pass "Looping around a cache device succeeds."