mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-24 03:08:51 +03:00
Add zfault zpool configurations and tests
Eleven new zpool configurations were added to allow testing of various failure cases. The first 5 zpool configurations leverage the 'faulty' md device type which allow us to simuluate IO errors at the block layer. The last 6 zpool configurations leverage the scsi_debug module provided by modern kernels. This device allows you to create virtual scsi devices which are backed by a ram disk. With this setup we can verify the full IO stack by injecting faults at the lowest layer. Both methods of fault injection are important to verifying the IO stack. The zfs code itself also provides a mechanism for error injection via the zinject command line tool. While we should also take advantage of this appraoch to validate the code it does not address any of the Linux integration issues which are the most concerning. For the moment we're trusting that the upstream Solaris guys are running zinject and would have caught internal zfs logic errors. Currently, there are 6 r/w test cases layered on top of the 'faulty' md devices. They include 3 writes tests for soft/transient errors, hard/permenant errors, and all writes error to the device. There are 3 matching read tests for soft/transient errors, hard/permenant errors, and fixable read error with a write. Although for this last case zfs doesn't do anything special. The seventh test case verifies zfs detects and corrects checksum errors. In this case one of the drives is extensively damaged and by dd'ing over large sections of it. We then ensure zfs logs the issue and correctly rebuilds the damage. The next test cases use the scsi_debug configuration to injects error at the bottom of the scsi stack. This ensures we find any flaws in the scsi midlayer or our usage of it. Plus it stresses the device specific retry, timeout, and error handling outside of zfs's control. The eighth test case is to verify that the system correctly handles an intermittent device timeout. Here the scsi_debug device drops 1 in N requests resulting in a retry either at the block level. The ZFS code does specify the FAILFAST option but it turns out that for this case the Linux IO stack with still retry the command. The FAILFAST logic located in scsi_noretry_cmd() does no seem to apply to the simply timeout case. It appears to be more targeted to specific device or transport errors from the lower layers. The ninth test case handles a persistent failure in which the device is removed from the system by Linux. The test verifies that the failure is detected, the device is made unavailable, and then can be successfully re-add when brought back online. Additionally, it ensures that errors and events are logged to the correct places and the no data corruption has occured due to the failure.
This commit is contained in:
Brian Behlendorf
+184
-12
@@ -61,6 +61,11 @@ LSMOD=${LSMOD:-/sbin/lsmod}
|
||||
RMMOD=${RMMOD:-/sbin/rmmod}
|
||||
INFOMOD=${INFOMOD:-/sbin/modinfo}
|
||||
LOSETUP=${LOSETUP:-/sbin/losetup}
|
||||
MDADM=${MDADM:-/sbin/mdadm}
|
||||
PARTED=${PARTED:-/sbin/parted}
|
||||
BLOCKDEV=${BLOCKDEV:-/sbin/blockdev}
|
||||
LSSCSI=${LSSCSI:-/usr/bin/lsscsi}
|
||||
SCSIRESCAN=${SCSIRESCAN:-/usr/bin/scsi-rescan}
|
||||
SYSCTL=${SYSCTL:-/sbin/sysctl}
|
||||
UDEVADM=${UDEVADM:-/sbin/udevadm}
|
||||
AWK=${AWK:-/usr/bin/awk}
|
||||
@@ -225,8 +230,19 @@ unload_modules() {
|
||||
return 0
|
||||
}
|
||||
|
||||
#
|
||||
# Check that the mdadm utilities are installed.
|
||||
#
|
||||
check_loop_utils() {
|
||||
test -f ${LOSETUP} || die "${LOSETUP} utility must be installed"
|
||||
}
|
||||
|
||||
|
||||
#
|
||||
# Find and return an unused loopback device.
|
||||
#
|
||||
unused_loop_device() {
|
||||
for DEVICE in `ls -1 /dev/loop*`; do
|
||||
for DEVICE in `ls -1 /dev/loop* 2>/dev/null`; do
|
||||
${LOSETUP} ${DEVICE} &>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo ${DEVICE}
|
||||
@@ -239,7 +255,7 @@ unused_loop_device() {
|
||||
|
||||
#
|
||||
# This can be slightly dangerous because the loop devices we are
|
||||
# cleanup up may not be ours. However, if the devices are currently
|
||||
# cleaning up may not be ours. However, if the devices are currently
|
||||
# in use we will not be able to remove them, and we only remove
|
||||
# devices which include 'zpool' in the name. So any damage we might
|
||||
# do should be limited to other zfs related testing.
|
||||
@@ -255,6 +271,168 @@ cleanup_loop_devices() {
|
||||
rm -f ${TMP_FILE}
|
||||
}
|
||||
|
||||
#
|
||||
# Destroy the passed loopback devices, this is used when you know
|
||||
# the names of the loopback devices.
|
||||
#
|
||||
destroy_loop_devices() {
|
||||
local LODEVICES="$1"
|
||||
|
||||
msg "Destroying ${LODEVICES}"
|
||||
${LOSETUP} -d ${LODEVICES} || \
|
||||
die "Error $? destroying ${FILE} -> ${DEVICE} loopback"
|
||||
|
||||
rm -f ${FILES}
|
||||
return 0
|
||||
}
|
||||
|
||||
#
|
||||
# Check that the mdadm utilities are installed.
|
||||
#
|
||||
check_md_utils() {
|
||||
test -f ${MDADM} || die "${MDADM} utility must be installed"
|
||||
test -f ${PARTED} || die "${PARTED} utility must be installed"
|
||||
}
|
||||
|
||||
check_md_partitionable() {
|
||||
local LOFILE=`mktemp -p /tmp zpool-lo.XXXXXXXX`
|
||||
local LODEVICE=`unused_loop_device`
|
||||
local MDDEVICE=`unused_md_device`
|
||||
local RESULT=1
|
||||
|
||||
check_md_utils
|
||||
|
||||
rm -f ${LOFILE}
|
||||
dd if=/dev/zero of=${LOFILE} bs=1M count=0 seek=16 \
|
||||
&>/dev/null || return ${RESULT}
|
||||
|
||||
msg "Creating ${LODEVICE} using ${LOFILE}"
|
||||
${LOSETUP} ${LODEVICE} ${LOFILE}
|
||||
if [ $? -ne 0 ]; then
|
||||
rm -f ${LOFILE}
|
||||
return ${RESULT}
|
||||
fi
|
||||
|
||||
msg "Creating ${MDDEVICE} using ${LODEVICE}"
|
||||
${MDADM} --build ${MDDEVICE} --level=faulty \
|
||||
--raid-devices=1 ${LODEVICE} &>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
destroy_loop_devices ${LODEVICE}
|
||||
rm -f ${LOFILE}
|
||||
return ${RESULT}
|
||||
fi
|
||||
wait_udev ${MDDEVICE} 30
|
||||
|
||||
${BLOCKDEV} --rereadpt ${MDDEVICE} 2>/dev/null
|
||||
RESULT=$?
|
||||
|
||||
destroy_md_devices ${MDDEVICE}
|
||||
destroy_loop_devices ${LODEVICE}
|
||||
rm -f ${LOFILE}
|
||||
|
||||
return ${RESULT}
|
||||
}
|
||||
|
||||
#
|
||||
# Find and return an unused md device.
|
||||
#
|
||||
unused_md_device() {
|
||||
for (( i=0; i<32; i++ )); do
|
||||
MDDEVICE=md${i}
|
||||
|
||||
# Skip active devicesudo in /proc/mdstat.
|
||||
grep -q "${MDDEVICE} " /proc/mdstat && continue
|
||||
|
||||
# Device doesn't exist, use it.
|
||||
if [ ! -e $/dev/{MDDEVICE} ]; then
|
||||
echo /dev/${MDDEVICE}
|
||||
return
|
||||
fi
|
||||
|
||||
# Device exists but may not be in use.
|
||||
if [ -b /dev/${MDDEVICE} ]; then
|
||||
${MDADM} --detail /dev/${MDDEVICE} &>/dev/null
|
||||
if [ $? -eq 1 ]; then
|
||||
echo /dev/${MDDEVICE}
|
||||
return
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
die "Error: Unable to find unused md device"
|
||||
}
|
||||
|
||||
#
|
||||
# This can be slightly dangerous because it is possible the md devices
|
||||
# we are cleaning up may not be ours. However, if the devices are
|
||||
# currently in use we will not be able to remove them, and even if
|
||||
# we remove devices which were not out we do not zero the super block
|
||||
# so you should be able to reconstruct them.
|
||||
#
|
||||
cleanup_md_devices() {
|
||||
destroy_md_devices "`ls /dev/md* 2>/dev/null | grep -v p`"
|
||||
udev_trigger
|
||||
}
|
||||
|
||||
#
|
||||
# Destroy the passed md devices, this is used when you know
|
||||
# the names of the md devices.
|
||||
#
|
||||
destroy_md_devices() {
|
||||
local MDDEVICES="$1"
|
||||
|
||||
msg "Destroying ${MDDEVICES}"
|
||||
for MDDEVICE in ${MDDEVICES}; do
|
||||
${MDADM} --stop ${MDDEVICE} &>/dev/null
|
||||
${MDADM} --remove ${MDDEVICE} &>/dev/null
|
||||
${MDADM} --detail ${MDDEVICE} &>/dev/null
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
#
|
||||
# Check that the scsi utilities are installed.
|
||||
#
|
||||
check_sd_utils() {
|
||||
${INFOMOD} scsi_debug &>/dev/null || die "scsi_debug module required"
|
||||
test -f ${LSSCSI} || die "${LSSCSI} utility must be installed"
|
||||
}
|
||||
|
||||
#
|
||||
# Rescan the scsi bus for scsi_debug devices. It is preferable to use the
|
||||
# scsi-rescan tool if it is installed, but if it's not we can fall back to
|
||||
# removing and readding the device manually. This rescan will only effect
|
||||
# the first scsi_debug device if scsi-rescan is missing.
|
||||
#
|
||||
scsi_rescan() {
|
||||
local AWK_SCRIPT="/scsi_debug/ { print \$1; exit }"
|
||||
|
||||
if [ -f ${SCSIRESCAN} ]; then
|
||||
${SCSIRESCAN} --forcerescan --remove &>/dev/null
|
||||
else
|
||||
local SCSIID=`${LSSCSI} | ${AWK} "${AWK_SCRIPT}" | tr -d '[]'`
|
||||
local SCSIHOST=`echo ${SCSIID} | cut -f1 -d':'`
|
||||
echo 1 >"/sys/class/scsi_device/${SCSIID}/device/delete"
|
||||
udev_trigger
|
||||
echo "- - -" >/sys/class/scsi_host/host${SCSIHOST}/scan
|
||||
udev_trigger
|
||||
fi
|
||||
}
|
||||
|
||||
#
|
||||
# Trigger udev and wait for it to settle.
|
||||
#
|
||||
udev_trigger() {
|
||||
if [ -f ${UDEVADM} ]; then
|
||||
${UDEVADM} trigger
|
||||
${UDEVADM} settle
|
||||
else
|
||||
/sbin/udevtrigger
|
||||
/sbin/udevsettle
|
||||
fi
|
||||
}
|
||||
|
||||
#
|
||||
# The following udev helper functions assume that the provided
|
||||
# udev rules file will create a /dev/disk/zpool/<CHANNEL><RANK>
|
||||
@@ -292,14 +470,7 @@ udev_setup() {
|
||||
fi
|
||||
|
||||
cp ${SRC_PATH} ${DST_PATH}
|
||||
|
||||
if [ -f ${UDEVADM} ]; then
|
||||
${UDEVADM} trigger
|
||||
${UDEVADM} settle
|
||||
else
|
||||
/sbin/udevtrigger
|
||||
/sbin/udevsettle
|
||||
fi
|
||||
udev_trigger
|
||||
fi
|
||||
|
||||
return 0
|
||||
@@ -405,7 +576,7 @@ run_one_test() {
|
||||
local TEST_NUM=$1
|
||||
local TEST_NAME=$2
|
||||
|
||||
printf "%-4d %-36s " ${TEST_NUM} "${TEST_NAME}"
|
||||
printf "%-4d %-34s " ${TEST_NUM} "${TEST_NAME}"
|
||||
test_${TEST_NUM}
|
||||
}
|
||||
|
||||
@@ -413,7 +584,7 @@ skip_one_test() {
|
||||
local TEST_NUM=$1
|
||||
local TEST_NAME=$2
|
||||
|
||||
printf "%-4d %-36s " ${TEST_NUM} "${TEST_NAME}"
|
||||
printf "%-4d %-34s " ${TEST_NUM} "${TEST_NAME}"
|
||||
skip
|
||||
}
|
||||
|
||||
@@ -447,6 +618,7 @@ wait_udev() {
|
||||
local DELAY=$2
|
||||
local COUNT=0
|
||||
|
||||
udev_trigger
|
||||
while [ ! -e ${DEVICE} ]; do
|
||||
if [ ${COUNT} -gt ${DELAY} ]; then
|
||||
return 1
|
||||
|
||||
Reference in New Issue
Block a user