mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-31 11:14:09 +03:00
Various ZED fixes
* Teach ZED to handle spares usingi the configured ashift: if the zpool 'ashift' property is set then ZED should use its value when kicking in a hotspare; with this change 512e disks can be used as spares for VDEVs that were created with ashift=9, even if ZFS natively detects them as 4K block devices. * Introduce an additional auto_spare test case which verifies that in the face of multiple device failures an appropiate number of spares are kicked in. * Fix zed_stop() in "libtest.shlib" which did not correctly wait the target pid. * Fix ZED crashing on startup caused by a race condition in libzfs when used in multi-threaded context. * Convert ZED over to using the tpool library which is already present in the Illumos FMA code. Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: loli10K <ezomori.nozomu@gmail.com> Closes #2562 Closes #6858
This commit is contained in:
LOLi
@@ -27,7 +27,7 @@ if is_linux; then
|
||||
for SDDEVICE in $(get_debug_device); do
|
||||
unplug $SDDEVICE
|
||||
done
|
||||
modprobe -r scsi_debug
|
||||
unload_scsi_debug
|
||||
fi
|
||||
|
||||
log_pass
|
||||
|
||||
@@ -22,7 +22,7 @@ verify_runnable "global"
|
||||
|
||||
# Create scsi_debug devices for the reopen tests
|
||||
if is_linux; then
|
||||
load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
|
||||
load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
|
||||
else
|
||||
log_unsupported "scsi debug module unsupported"
|
||||
fi
|
||||
|
||||
@@ -6,4 +6,6 @@ dist_pkgdata_SCRIPTS = \
|
||||
auto_online_001_pos.ksh \
|
||||
auto_replace_001_pos.ksh \
|
||||
auto_spare_001_pos.ksh \
|
||||
auto_spare_002_pos.ksh
|
||||
auto_spare_002_pos.ksh \
|
||||
auto_spare_ashift.ksh \
|
||||
auto_spare_multiple.ksh
|
||||
|
||||
@@ -54,9 +54,8 @@ fi
|
||||
|
||||
function cleanup
|
||||
{
|
||||
#online last disk before fail
|
||||
insert_disk $offline_disk $host
|
||||
poolexists $TESTPOOL && destroy_pool $TESTPOOL
|
||||
destroy_pool $TESTPOOL
|
||||
unload_scsi_debug
|
||||
}
|
||||
|
||||
log_assert "Testing automated auto-online FMA test"
|
||||
@@ -65,8 +64,8 @@ log_onexit cleanup
|
||||
|
||||
# If using the default loop devices, need a scsi_debug device for auto-online
|
||||
if is_loop_device $DISK1; then
|
||||
SD=$(lsscsi | nawk '/scsi_debug/ {print $6; exit}')
|
||||
SDDEVICE=$(echo $SD | nawk -F / '{print $3}')
|
||||
load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
|
||||
SDDEVICE=$(get_debug_device)
|
||||
SDDEVICE_ID=$(get_persistent_disk_name $SDDEVICE)
|
||||
autoonline_disks="$SDDEVICE"
|
||||
else
|
||||
|
||||
@@ -57,27 +57,23 @@ fi
|
||||
|
||||
function setup
|
||||
{
|
||||
lsmod | egrep scsi_debug > /dev/null
|
||||
if (($? == 1)); then
|
||||
load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
|
||||
fi
|
||||
load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
|
||||
SD=$(get_debug_device)
|
||||
SDDEVICE_ID=$(get_persistent_disk_name $SD)
|
||||
# Register vdev_id alias rule for scsi_debug device to create a
|
||||
# persistent path
|
||||
SD=$(lsscsi | nawk '/scsi_debug/ {print $6; exit}' \
|
||||
| nawk -F / '{print $3}')
|
||||
SDDEVICE_ID=$(get_persistent_disk_name $SD)
|
||||
log_must eval "echo "alias scsidebug /dev/disk/by-id/$SDDEVICE_ID" \
|
||||
>> $VDEVID_CONF"
|
||||
block_device_wait
|
||||
|
||||
SDDEVICE=$(udevadm info -q all -n $DEV_DSKDIR/$SD | egrep ID_VDEV \
|
||||
| nawk '{print $2; exit}' | nawk -F = '{print $2; exit}')
|
||||
SDDEVICE=$(udevadm info -q all -n $DEV_DSKDIR/$SD \
|
||||
| awk -F'=' '/ID_VDEV=/{print $2; exit}')
|
||||
[[ -z $SDDEVICE ]] && log_fail "vdev rule was not registered properly"
|
||||
}
|
||||
|
||||
function cleanup
|
||||
{
|
||||
poolexists $TESTPOOL && destroy_pool $TESTPOOL
|
||||
destroy_pool $TESTPOOL
|
||||
unload_scsi_debug
|
||||
}
|
||||
|
||||
log_assert "Testing automated auto-replace FMA test"
|
||||
@@ -112,7 +108,7 @@ log_must zpool export -F $TESTPOOL
|
||||
# Offline disk
|
||||
remove_disk $SD
|
||||
block_device_wait
|
||||
log_must modprobe -r scsi_debug
|
||||
unload_scsi_debug
|
||||
|
||||
# Reimport pool with drive missing
|
||||
log_must zpool import $TESTPOOL
|
||||
|
||||
@@ -42,7 +42,7 @@ verify_runnable "both"
|
||||
function cleanup
|
||||
{
|
||||
log_must zinject -c all
|
||||
poolexists $TESTPOOL && destroy_pool $TESTPOOL
|
||||
destroy_pool $TESTPOOL
|
||||
rm -f $VDEV_FILES $SPARE_FILE
|
||||
}
|
||||
|
||||
@@ -50,6 +50,9 @@ log_assert "Testing automated auto-spare FMA test"
|
||||
|
||||
log_onexit cleanup
|
||||
|
||||
# Clear events from previous runs
|
||||
zed_events_drain
|
||||
|
||||
TESTFILE="/$TESTPOOL/$TESTFS/testfile"
|
||||
|
||||
for type in "mirror" "raidz" "raidz2"; do
|
||||
|
||||
@@ -42,7 +42,7 @@ verify_runnable "both"
|
||||
function cleanup
|
||||
{
|
||||
log_must zinject -c all
|
||||
poolexists $TESTPOOL && destroy_pool $TESTPOOL
|
||||
destroy_pool $TESTPOOL
|
||||
rm -f $VDEV_FILES $SPARE_FILE
|
||||
}
|
||||
|
||||
@@ -50,6 +50,9 @@ log_assert "Testing automated auto-spare FMA test"
|
||||
|
||||
log_onexit cleanup
|
||||
|
||||
# Clear events from previous runs
|
||||
zed_events_drain
|
||||
|
||||
TESTFILE="/$TESTPOOL/$TESTFS/testfile"
|
||||
|
||||
for type in "mirror" "raidz" "raidz2"; do
|
||||
@@ -65,8 +68,14 @@ for type in "mirror" "raidz" "raidz2"; do
|
||||
log_must dd if=/dev/urandom of=$TESTFILE bs=1M count=16
|
||||
|
||||
# 4. Inject CHECKSUM ERRORS on read with a zinject error handler
|
||||
# NOTE: checksum events are ratelimited to max 5 per second, ZED needs
|
||||
# 10 to kick in a spare
|
||||
log_must zinject -d $FAULT_FILE -e corrupt -f 50 -T read $TESTPOOL
|
||||
log_must cp $TESTFILE /dev/null
|
||||
log_must sleep 1
|
||||
log_must cp $TESTFILE /dev/null
|
||||
log_must sleep 1
|
||||
log_must cp $TESTFILE /dev/null
|
||||
|
||||
# 5. Verify the ZED kicks in a hot spare and expected pool/device status
|
||||
log_note "Wait for ZED to auto-spare"
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/bin/ksh -p
|
||||
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# This file and its contents are supplied under the terms of the
|
||||
# Common Development and Distribution License ("CDDL"), version 1.0.
|
||||
# You may only use this file in accordance with the terms of version
|
||||
# 1.0 of the CDDL.
|
||||
#
|
||||
# A full copy of the text of the CDDL should have accompanied this
|
||||
# source. A copy of the CDDL is also available via the Internet at
|
||||
# http://www.illumos.org/license/CDDL.
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2017 by Intel Corporation. All rights reserved.
|
||||
# Copyright 2017, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
|
||||
#
|
||||
|
||||
. $STF_SUITE/include/libtest.shlib
|
||||
. $STF_SUITE/include/math.shlib
|
||||
. $STF_SUITE/tests/functional/fault/fault.cfg
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Testing Fault Management Agent ZED Logic - Automated Auto-Spare Test when
|
||||
# drive is faulted and a custom ashift value needs to be provided to replace it.
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create a pool from 512b devices and set "ashift" pool property accordingly
|
||||
# 2. Add one 512e spare device (4Kn would generate IO errors on replace)
|
||||
# 3. Inject IO errors with a zinject error handler
|
||||
# 4. Start a scrub
|
||||
# 5. Verify the ZED kicks in the hot spare and expected pool/device status
|
||||
# 6. Clear the fault
|
||||
# 7. Verify the hot spare is available and expected pool/device status
|
||||
#
|
||||
|
||||
verify_runnable "both"
|
||||
|
||||
function cleanup
|
||||
{
|
||||
log_must zinject -c all
|
||||
destroy_pool $TESTPOOL
|
||||
unload_scsi_debug
|
||||
rm -f $SAFE_DEVICE $FAIL_DEVICE
|
||||
}
|
||||
|
||||
log_assert "ZED should replace a device using the configured ashift property"
|
||||
log_onexit cleanup
|
||||
|
||||
# Clear events from previous runs
|
||||
zed_events_drain
|
||||
|
||||
SAFE_DEVICE="$TEST_BASE_DIR/safe-dev"
|
||||
FAIL_DEVICE="$TEST_BASE_DIR/fail-dev"
|
||||
|
||||
# 1. Create a pool from 512b devices and set "ashift" pool property accordingly
|
||||
for vdev in $SAFE_DEVICE $FAIL_DEVICE; do
|
||||
truncate -s $SPA_MINDEVSIZE $vdev
|
||||
done
|
||||
log_must zpool create -f $TESTPOOL mirror $SAFE_DEVICE $FAIL_DEVICE
|
||||
# NOTE: file VDEVs should be added as 512b devices, verify this "just in case"
|
||||
for vdev in $SAFE_DEVICE $FAIL_DEVICE; do
|
||||
verify_eq "9" "$(zdb -e -l $vdev | awk '/ashift: /{print $2}')" "ashift"
|
||||
done
|
||||
log_must zpool set ashift=9 $TESTPOOL
|
||||
|
||||
# 2. Add one 512e spare device (4Kn would generate IO errors on replace)
|
||||
# NOTE: must be larger than the existing 512b devices, add 32m of fudge
|
||||
load_scsi_debug $(($SPA_MINDEVSIZE/1024/1024+32)) $SDHOSTS $SDTGTS $SDLUNS '512e'
|
||||
SPARE_DEVICE=$(get_debug_device)
|
||||
log_must_busy zpool add $TESTPOOL spare $SPARE_DEVICE
|
||||
|
||||
# 3. Inject IO errors with a zinject error handler
|
||||
log_must zinject -d $FAIL_DEVICE -e io -T all -f 100 $TESTPOOL
|
||||
|
||||
# 4. Start a scrub
|
||||
log_must zpool scrub $TESTPOOL
|
||||
|
||||
# 5. Verify the ZED kicks in a hot spare and expected pool/device status
|
||||
log_note "Wait for ZED to auto-spare"
|
||||
log_must wait_vdev_state $TESTPOOL $FAIL_DEVICE "FAULTED" 60
|
||||
log_must wait_vdev_state $TESTPOOL $SPARE_DEVICE "ONLINE" 60
|
||||
log_must wait_hotspare_state $TESTPOOL $SPARE_DEVICE "INUSE"
|
||||
log_must check_state $TESTPOOL "" "DEGRADED"
|
||||
|
||||
# 6. Clear the fault
|
||||
log_must zinject -c all
|
||||
log_must zpool clear $TESTPOOL $FAIL_DEVICE
|
||||
|
||||
# 7. Verify the hot spare is available and expected pool/device status
|
||||
log_must wait_vdev_state $TESTPOOL $FAIL_DEVICE "ONLINE" 60
|
||||
log_must wait_hotspare_state $TESTPOOL $SPARE_DEVICE "AVAIL"
|
||||
log_must is_pool_resilvered $TESTPOOL
|
||||
log_must check_state $TESTPOOL "" "ONLINE"
|
||||
|
||||
log_pass "ZED successfully replaces a device using the configured ashift property"
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/bin/ksh -p
|
||||
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# This file and its contents are supplied under the terms of the
|
||||
# Common Development and Distribution License ("CDDL"), version 1.0.
|
||||
# You may only use this file in accordance with the terms of version
|
||||
# 1.0 of the CDDL.
|
||||
#
|
||||
# A full copy of the text of the CDDL should have accompanied this
|
||||
# source. A copy of the CDDL is also available via the Internet at
|
||||
# http://www.illumos.org/license/CDDL.
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2017 by Intel Corporation. All rights reserved.
|
||||
# Copyright 2017, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
|
||||
#
|
||||
|
||||
. $STF_SUITE/include/libtest.shlib
|
||||
. $STF_SUITE/tests/functional/fault/fault.cfg
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Testing Fault Management Agent ZED Logic - Automated Auto-Spare Test when
|
||||
# multiple drives are faulted.
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create a pool with two hot spares
|
||||
# 2. Inject IO ERRORS with a zinject error handler on the first device
|
||||
# 3. Start a scrub
|
||||
# 4. Verify the ZED kicks in a hot spare and expected pool/device status
|
||||
# 5. Inject IO ERRORS on a second device
|
||||
# 6. Start a scrub
|
||||
# 7. Verify the ZED kicks in a second hot spare
|
||||
# 8. Clear the fault on both devices
|
||||
# 9. Verify the hot spares are available and expected pool/device status
|
||||
# 10. Rinse and repeat, this time faulting both devices at the same time
|
||||
#
|
||||
|
||||
verify_runnable "both"
|
||||
|
||||
function cleanup
|
||||
{
|
||||
log_must zinject -c all
|
||||
destroy_pool $TESTPOOL
|
||||
rm -f $DATA_DEVS $SPARE_DEVS
|
||||
}
|
||||
|
||||
log_assert "ZED should be able to handle multiple faulted devices"
|
||||
log_onexit cleanup
|
||||
|
||||
# Clear events from previous runs
|
||||
zed_events_drain
|
||||
|
||||
FAULT_DEV1="$TEST_BASE_DIR/fault-dev1"
|
||||
FAULT_DEV2="$TEST_BASE_DIR/fault-dev2"
|
||||
SAFE_DEV1="$TEST_BASE_DIR/safe-dev1"
|
||||
SAFE_DEV2="$TEST_BASE_DIR/safe-dev2"
|
||||
DATA_DEVS="$FAULT_DEV1 $FAULT_DEV2 $SAFE_DEV1 $SAFE_DEV2"
|
||||
SPARE_DEV1="$TEST_BASE_DIR/spare-dev1"
|
||||
SPARE_DEV2="$TEST_BASE_DIR/spare-dev2"
|
||||
SPARE_DEVS="$SPARE_DEV1 $SPARE_DEV2"
|
||||
|
||||
for type in "mirror" "raidz" "raidz2" "raidz3"; do
|
||||
# 1. Create a pool with two hot spares
|
||||
truncate -s $SPA_MINDEVSIZE $DATA_DEVS $SPARE_DEVS
|
||||
log_must zpool create -f $TESTPOOL $type $DATA_DEVS spare $SPARE_DEVS
|
||||
|
||||
# 2. Inject IO ERRORS with a zinject error handler on the first device
|
||||
log_must zinject -d $FAULT_DEV1 -e io -T all -f 100 $TESTPOOL
|
||||
|
||||
# 3. Start a scrub
|
||||
log_must zpool scrub $TESTPOOL
|
||||
|
||||
# 4. Verify the ZED kicks in a hot spare and expected pool/device status
|
||||
log_note "Wait for ZED to auto-spare"
|
||||
log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "FAULTED" 60
|
||||
log_must wait_vdev_state $TESTPOOL $SPARE_DEV1 "ONLINE" 60
|
||||
log_must wait_hotspare_state $TESTPOOL $SPARE_DEV1 "INUSE"
|
||||
log_must check_state $TESTPOOL "" "DEGRADED"
|
||||
|
||||
# 5. Inject IO ERRORS on a second device
|
||||
log_must zinject -d $FAULT_DEV2 -e io -T all -f 100 $TESTPOOL
|
||||
|
||||
# 6. Start a scrub
|
||||
while is_pool_scrubbing $TESTPOOL || is_pool_resilvering $TESTPOOL; do
|
||||
sleep 1
|
||||
done
|
||||
log_must zpool scrub $TESTPOOL
|
||||
|
||||
# 7. Verify the ZED kicks in a second hot spare
|
||||
log_note "Wait for ZED to auto-spare"
|
||||
log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "FAULTED" 60
|
||||
log_must wait_vdev_state $TESTPOOL $SPARE_DEV2 "ONLINE" 60
|
||||
log_must wait_hotspare_state $TESTPOOL $SPARE_DEV2 "INUSE"
|
||||
log_must check_state $TESTPOOL "" "DEGRADED"
|
||||
|
||||
# 8. Clear the fault on both devices
|
||||
log_must zinject -c all
|
||||
log_must zpool clear $TESTPOOL $FAULT_DEV1
|
||||
log_must zpool clear $TESTPOOL $FAULT_DEV2
|
||||
|
||||
# 9. Verify the hot spares are available and expected pool/device status
|
||||
log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "ONLINE" 60
|
||||
log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "ONLINE" 60
|
||||
log_must wait_hotspare_state $TESTPOOL $SPARE_DEV1 "AVAIL"
|
||||
log_must wait_hotspare_state $TESTPOOL $SPARE_DEV2 "AVAIL"
|
||||
log_must check_state $TESTPOOL "" "ONLINE"
|
||||
|
||||
# Cleanup
|
||||
cleanup
|
||||
done
|
||||
|
||||
# Rinse and repeat, this time faulting both devices at the same time
|
||||
# NOTE: "raidz" is exluded since it cannot survive 2 faulted devices
|
||||
# NOTE: "mirror" is a 4-way mirror here and should survive this test
|
||||
for type in "mirror" "raidz2" "raidz3"; do
|
||||
# 1. Create a pool with two hot spares
|
||||
truncate -s $SPA_MINDEVSIZE $DATA_DEVS $SPARE_DEVS
|
||||
log_must zpool create -f $TESTPOOL $type $DATA_DEVS spare $SPARE_DEVS
|
||||
|
||||
# 2. Inject IO ERRORS with a zinject error handler on two devices
|
||||
log_must eval "zinject -d $FAULT_DEV1 -e io -T all -f 100 $TESTPOOL &"
|
||||
log_must eval "zinject -d $FAULT_DEV2 -e io -T all -f 100 $TESTPOOL &"
|
||||
|
||||
# 3. Start a scrub
|
||||
log_must zpool scrub $TESTPOOL
|
||||
|
||||
# 4. Verify the ZED kicks in two hot spares and expected pool/device status
|
||||
log_note "Wait for ZED to auto-spare"
|
||||
log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "FAULTED" 60
|
||||
log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "FAULTED" 60
|
||||
log_must wait_vdev_state $TESTPOOL $SPARE_DEV1 "ONLINE" 60
|
||||
log_must wait_vdev_state $TESTPOOL $SPARE_DEV2 "ONLINE" 60
|
||||
log_must wait_hotspare_state $TESTPOOL $SPARE_DEV1 "INUSE"
|
||||
log_must wait_hotspare_state $TESTPOOL $SPARE_DEV2 "INUSE"
|
||||
log_must check_state $TESTPOOL "" "DEGRADED"
|
||||
|
||||
# 5. Clear the fault on both devices
|
||||
log_must zinject -c all
|
||||
log_must zpool clear $TESTPOOL $FAULT_DEV1
|
||||
log_must zpool clear $TESTPOOL $FAULT_DEV2
|
||||
|
||||
# Cleanup
|
||||
cleanup
|
||||
done
|
||||
|
||||
log_pass "ZED successfully handles multiple faulted devices"
|
||||
@@ -33,14 +33,4 @@ cleanup_devices $DISKS
|
||||
zed_stop
|
||||
zed_cleanup
|
||||
|
||||
SDDEVICE=$(get_debug_device)
|
||||
|
||||
# Offline disk and remove scsi_debug module
|
||||
if is_linux; then
|
||||
if [ -n "$SDDEVICE" ]; then
|
||||
remove_disk $SDDEVICE
|
||||
fi
|
||||
modprobe -r scsi_debug
|
||||
fi
|
||||
|
||||
log_pass
|
||||
|
||||
@@ -31,8 +31,4 @@ verify_runnable "global"
|
||||
zed_setup
|
||||
zed_start
|
||||
|
||||
# Create a scsi_debug device to be used with auto-online (if using loop devices)
|
||||
# and auto-replace regardless of other devices
|
||||
load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
|
||||
|
||||
log_pass
|
||||
|
||||
Reference in New Issue
Block a user