mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2025-06-26 19:17:36 +03:00
8fb1ede146
The intent of this patch is extend the existing deadman code
such that it's flexible enough to be used by both ztest and
on production systems. The proposed changes include:
* Added a new `zfs_deadman_failmode` module option which is
used to dynamically control the behavior of the deadman. It's
loosely modeled after, but independant from, the pool failmode
property. It can be set to wait, continue, or panic.
* wait - Wait for the "hung" I/O (default)
* continue - Attempt to recover from a "hung" I/O
* panic - Panic the system
* Added a new `zfs_deadman_ziotime_ms` module option which is
analogous to `zfs_deadman_synctime_ms` except instead of
applying to a pool TXG sync it applies to zio_wait(). A
default value of 300s is used to define a "hung" zio.
* The ztest deadman thread has been re-enabled by default,
aligned with the upstream OpenZFS code, and then extended
to terminate the process when it takes significantly longer
to complete than expected.
* The -G option was added to ztest to print the internal debug
log when a fatal error is encountered. This same option was
previously added to zdb in commit fa603f82. Update zloop.sh
to unconditionally pass -G to obtain additional debugging.
* The FM_EREPORT_ZFS_DELAY event which was previously posted
when the deadman detect a "hung" pool has been replaced by
a new dedicated FM_EREPORT_ZFS_DEADMAN event.
* The proposed recovery logic attempts to restart a "hung"
zio by calling zio_interrupt() on any outstanding leaf zios.
We may want to further restrict this to zios in either the
ZIO_STAGE_VDEV_IO_START or ZIO_STAGE_VDEV_IO_DONE stages.
Calling zio_interrupt() is expected to only be useful for
cases when an IO has been submitted to the physical device
but for some reasonable the completion callback hasn't been
called by the lower layers. This shouldn't be possible but
has been observed and may be caused by kernel/driver bugs.
* The 'zfs_deadman_synctime_ms' default value was reduced from
1000s to 600s.
* Depending on how ztest fails there may be no cache file to
move. This should not be considered fatal, collect the logs
which are available and carry on.
* Add deadman test cases for spa_deadman() and zio_wait().
* Increase default zfs_deadman_checktime_ms to 60s.
Reviewed-by: Tim Chase <tim@chase2k.com>
Reviewed by: Thomas Caputi <tcaputi@datto.com>
Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #6999
87 lines
2.7 KiB
Bash
Executable File
87 lines
2.7 KiB
Bash
Executable File
#!/bin/ksh -p
|
|
#
|
|
# CDDL HEADER START
|
|
#
|
|
# The contents of this file are subject to the terms of the
|
|
# Common Development and Distribution License (the "License").
|
|
# You may not use this file except in compliance with the License.
|
|
#
|
|
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
|
# or http://www.opensolaris.org/os/licensing.
|
|
# See the License for the specific language governing permissions
|
|
# and limitations under the License.
|
|
#
|
|
# When distributing Covered Code, include this CDDL HEADER in each
|
|
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
|
# If applicable, add the following below this CDDL HEADER, with the
|
|
# fields enclosed by brackets "[]" replaced with your own identifying
|
|
# information: Portions Copyright [yyyy] [name of copyright owner]
|
|
#
|
|
# CDDL HEADER END
|
|
#
|
|
|
|
#
|
|
# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
|
|
# Use is subject to license terms.
|
|
#
|
|
|
|
# DESCRIPTION:
|
|
# Verify spa deadman detects a hung txg
|
|
#
|
|
# STRATEGY:
|
|
# 1. Reduce the zfs_deadman_synctime_ms to 5s.
|
|
# 2. Reduce the zfs_deadman_checktime_ms to 1s.
|
|
# 3. Inject a 10s zio delay to force long IOs.
|
|
# 4. Write enough data to force a long txg sync time due to the delay.
|
|
# 5. Verify a "deadman" event is posted.
|
|
#
|
|
|
|
. $STF_SUITE/include/libtest.shlib
|
|
. $STF_SUITE/tests/functional/deadman/deadman.cfg
|
|
|
|
verify_runnable "both"
|
|
|
|
function cleanup
|
|
{
|
|
log_must zinject -c all
|
|
default_cleanup_noexit
|
|
|
|
log_must set_tunable64 zfs_deadman_synctime_ms $SYNCTIME_DEFAULT
|
|
log_must set_tunable64 zfs_deadman_checktime_ms $CHECKTIME_DEFAULT
|
|
log_must set_tunable64 zfs_deadman_failmode $FAILMODE_DEFAULT
|
|
}
|
|
|
|
log_assert "Verify spa deadman detects a hung txg"
|
|
log_onexit cleanup
|
|
|
|
log_must set_tunable64 zfs_deadman_synctime_ms 5000
|
|
log_must set_tunable64 zfs_deadman_checktime_ms 1000
|
|
log_must set_tunable64 zfs_deadman_failmode "wait"
|
|
|
|
# Create a new pool in order to use the updated deadman settings.
|
|
default_setup_noexit $DISK1
|
|
log_must zpool events -c
|
|
|
|
# Force each IO to take 10s by allow them to run concurrently.
|
|
log_must zinject -d $DISK1 -D10000:10 $TESTPOOL
|
|
|
|
mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS)
|
|
log_must file_write -b 1048576 -c 8 -o create -d 0 -f $mntpnt/file
|
|
sleep 10
|
|
|
|
log_must zinject -c all
|
|
log_must zpool sync
|
|
|
|
# Log txg sync times for reference and the zpool event summary.
|
|
log_must cat /proc/spl/kstat/zfs/$TESTPOOL/txgs
|
|
log_must zpool events
|
|
|
|
# Verify at least 5 deadman events were logged. The first after 5 seconds,
|
|
# and another each second thereafter until the delay is clearer.
|
|
events=$(zpool events | grep -c ereport.fs.zfs.deadman)
|
|
if [ "$events" -lt 5 ]; then
|
|
log_fail "Expect >=5 deadman events, $events found"
|
|
fi
|
|
|
|
log_pass "Verify spa deadman detected a hung txg and $events deadman events"
|