mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-26 12:12:13 +03:00
Add illumos FMD ZFS logic to ZED -- phase 2
The phase 2 work primarily entails the Diagnosis Engine and the Retire Agent modules. It also includes infrastructure to support a crude FMD environment to host these modules. The Diagnosis Engine consumes I/O and checksum ereports and feeds them into a SERD engine which will generate a corres- ponding fault diagnosis when the SERD engine fires. All the diagnosis state data is collected into cases, one case per vdev being tracked. The Retire Agent responds to diagnosed faults by isolating the faulty VDEV. It will notify the ZFS kernel module of the new VDEV state (degraded or faulted). This agent is also responsible for managing hot spares across pools. When it encounters a device fault or a device removal it replaces the device with an appropriate spare if available. Reviewed-by: Tony Hutter <hutter2@llnl.gov> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Don Brady <don.brady@intel.com> Closes #5343
This commit is contained in:
Don Brady
committed by
Brian Behlendorf
Brian Behlendorf
parent
f4bae2ed63
commit
976246fadd
@@ -1 +0,0 @@
|
||||
io-spare.sh
|
||||
@@ -1,239 +0,0 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Replace a device with a hot spare in response to IO or CHECKSUM errors.
|
||||
# The following actions will be performed automatically when the number
|
||||
# of errors exceed the limit set by ZED_SPARE_ON_IO_ERRORS or
|
||||
# ZED_SPARE_ON_CHECKSUM_ERRORS.
|
||||
#
|
||||
# 1) FAULT the device on IO errors, no futher IO will be attempted.
|
||||
# DEGRADE the device on checksum errors, the device is still
|
||||
# functional and can be used to service IO requests.
|
||||
# 2) Set the SES fault beacon for the device.
|
||||
# 3) Replace the device with a hot spare if any are available.
|
||||
#
|
||||
# Once the hot sparing operation is complete either the failed device or
|
||||
# the hot spare must be manually retired using the 'zpool detach' command.
|
||||
# The 'autoreplace' functionality which would normally take care of this
|
||||
# under Illumos has not yet been implemented.
|
||||
#
|
||||
# Full support for autoreplace is planned, but it requires that the full
|
||||
# ZFS Diagnosis Engine be ported. In the meanwhile this script provides
|
||||
# the majority of the expected hot spare functionality.
|
||||
#
|
||||
# Exit codes:
|
||||
# 0: hot spare replacement successful
|
||||
# 1: hot spare device not available
|
||||
# 2: hot sparing disabled or threshold not reached
|
||||
# 3: device already faulted or degraded
|
||||
# 9: internal error
|
||||
|
||||
[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
|
||||
. "${ZED_ZEDLET_DIR}/zed-functions.sh"
|
||||
|
||||
# Disabled by default. Enable in the zed.rc file.
|
||||
: "${ZED_SPARE_ON_CHECKSUM_ERRORS:=0}"
|
||||
: "${ZED_SPARE_ON_IO_ERRORS:=0}"
|
||||
|
||||
|
||||
# query_vdev_status (pool, vdev)
|
||||
#
|
||||
# Given a [pool] and [vdev], return the matching vdev path & status on stdout.
|
||||
#
|
||||
# Warning: This function does not handle the case of [pool] or [vdev]
|
||||
# containing whitespace. Beware of ShellCheck SC2046. Caveat emptor.
|
||||
#
|
||||
# Arguments
|
||||
# pool: pool name
|
||||
# vdev: virtual device name
|
||||
#
|
||||
# StdOut
|
||||
# arg1: vdev pathname
|
||||
# arg2: vdev status
|
||||
#
|
||||
query_vdev_status()
|
||||
{
|
||||
local pool="$1"
|
||||
local vdev="$2"
|
||||
local t
|
||||
|
||||
vdev="$(basename -- "${vdev}")"
|
||||
([ -n "${pool}" ] && [ -n "${vdev}" ]) || return
|
||||
t="$(printf '\t')"
|
||||
|
||||
"${ZPOOL}" status "${pool}" 2>/dev/null | sed -n -e \
|
||||
"s,^[ $t]*\(.*${vdev}\(-part[0-9]\+\)\?\)[ $t]*\([A-Z]\+\).*,\1 \3,p" \
|
||||
| tail -1
|
||||
}
|
||||
|
||||
|
||||
# notify (old_vdev, new_vdev, num_errors)
|
||||
#
|
||||
# Send a notification regarding the hot spare replacement.
|
||||
#
|
||||
# Arguments
|
||||
# old_vdev: path of old vdev that has failed
|
||||
# new_vdev: path of new vdev used as the hot spare replacement
|
||||
# num_errors: number of errors that triggered this replacement
|
||||
#
|
||||
notify()
|
||||
{
|
||||
local old_vdev="$1"
|
||||
local new_vdev="$2"
|
||||
local num_errors="$3"
|
||||
local note_subject
|
||||
local note_pathname
|
||||
local s
|
||||
local rv
|
||||
|
||||
umask 077
|
||||
note_subject="ZFS hot spare replacement for ${ZEVENT_POOL} on $(hostname)"
|
||||
note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
|
||||
{
|
||||
[ "${num_errors}" -ne 1 ] 2>/dev/null && s="s"
|
||||
|
||||
echo "ZFS has replaced a failing device with a hot spare after" \
|
||||
"${num_errors} ${ZEVENT_SUBCLASS} error${s}:"
|
||||
echo
|
||||
echo " eid: ${ZEVENT_EID}"
|
||||
echo " class: ${ZEVENT_SUBCLASS}"
|
||||
echo " host: $(hostname)"
|
||||
echo " time: ${ZEVENT_TIME_STRING}"
|
||||
echo " old: ${old_vdev}"
|
||||
echo " new: ${new_vdev}"
|
||||
|
||||
"${ZPOOL}" status "${ZEVENT_POOL}"
|
||||
|
||||
} > "${note_pathname}"
|
||||
|
||||
zed_notify "${note_subject}" "${note_pathname}"; rv=$?
|
||||
rm -f "${note_pathname}"
|
||||
return "${rv}"
|
||||
}
|
||||
|
||||
|
||||
# main
|
||||
#
|
||||
# Arguments
|
||||
# none
|
||||
#
|
||||
# Return
|
||||
# see above
|
||||
#
|
||||
main()
|
||||
{
|
||||
local num_errors
|
||||
local action
|
||||
local lockfile
|
||||
local vdev_path
|
||||
local vdev_status
|
||||
local spare
|
||||
local spare_path
|
||||
local spare_status
|
||||
local zpool_err
|
||||
local zpool_rv
|
||||
local rv
|
||||
|
||||
# Avoid hot-sparing a hot-spare.
|
||||
#
|
||||
# Note: ZEVENT_VDEV_PATH is not defined for ZEVENT_VDEV_TYPE=spare.
|
||||
#
|
||||
[ "${ZEVENT_VDEV_TYPE}" = "spare" ] && exit 2
|
||||
|
||||
[ -n "${ZEVENT_POOL}" ] || exit 9
|
||||
[ -n "${ZEVENT_VDEV_GUID}" ] || exit 9
|
||||
[ -n "${ZEVENT_VDEV_PATH}" ] || exit 9
|
||||
|
||||
zed_check_cmd "${ZPOOL}" "${ZINJECT}" || exit 9
|
||||
|
||||
# Fault the device after a given number of I/O errors.
|
||||
#
|
||||
if [ "${ZEVENT_SUBCLASS}" = "io" ]; then
|
||||
if [ "${ZED_SPARE_ON_IO_ERRORS}" -gt 0 ]; then
|
||||
num_errors=$((ZEVENT_VDEV_READ_ERRORS + ZEVENT_VDEV_WRITE_ERRORS))
|
||||
[ "${num_errors}" -ge "${ZED_SPARE_ON_IO_ERRORS}" ] \
|
||||
&& action="fault"
|
||||
fi 2>/dev/null
|
||||
|
||||
# Degrade the device after a given number of checksum errors.
|
||||
#
|
||||
elif [ "${ZEVENT_SUBCLASS}" = "checksum" ]; then
|
||||
if [ "${ZED_SPARE_ON_CHECKSUM_ERRORS}" -gt 0 ]; then
|
||||
num_errors="${ZEVENT_VDEV_CKSUM_ERRORS}"
|
||||
[ "${num_errors}" -ge "${ZED_SPARE_ON_CHECKSUM_ERRORS}" ] \
|
||||
&& action="degrade"
|
||||
fi 2>/dev/null
|
||||
|
||||
else
|
||||
zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
|
||||
exit 9
|
||||
fi
|
||||
|
||||
# Error threshold not reached.
|
||||
#
|
||||
if [ -z "${action}" ]; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
lockfile="zed.spare.lock"
|
||||
zed_lock "${lockfile}"
|
||||
|
||||
# shellcheck disable=SC2046
|
||||
set -- $(query_vdev_status "${ZEVENT_POOL}" "${ZEVENT_VDEV_PATH}")
|
||||
vdev_path="$1"
|
||||
vdev_status="$2"
|
||||
|
||||
# Device is already FAULTED or DEGRADED.
|
||||
#
|
||||
if [ "${vdev_status}" = "FAULTED" ] \
|
||||
|| [ "${vdev_status}" = "DEGRADED" ]; then
|
||||
rv=3
|
||||
|
||||
else
|
||||
rv=1
|
||||
|
||||
# 1) FAULT or DEGRADE the device.
|
||||
#
|
||||
"${ZINJECT}" -d "${ZEVENT_VDEV_GUID}" -A "${action}" "${ZEVENT_POOL}"
|
||||
|
||||
# 2) Set the SES fault beacon.
|
||||
#
|
||||
# TODO: Set the 'fault' or 'ident' beacon for the device. This can
|
||||
# be done through the sg_ses utility. The only hard part is to map
|
||||
# the sd device to its corresponding enclosure and slot. We may
|
||||
# be able to leverage the existing vdev_id scripts for this.
|
||||
#
|
||||
# $ sg_ses --dev-slot-num=0 --set=ident /dev/sg3
|
||||
# $ sg_ses --dev-slot-num=0 --clear=ident /dev/sg3
|
||||
|
||||
# 3) Replace the device with a hot spare.
|
||||
#
|
||||
# Round-robin through the spares trying those that are available.
|
||||
#
|
||||
for spare in ${ZEVENT_VDEV_SPARE_PATHS}; do
|
||||
|
||||
# shellcheck disable=SC2046
|
||||
set -- $(query_vdev_status "${ZEVENT_POOL}" "${spare}")
|
||||
spare_path="$1"
|
||||
spare_status="$2"
|
||||
|
||||
[ "${spare_status}" = "AVAIL" ] || continue
|
||||
|
||||
zpool_err="$("${ZPOOL}" replace "${ZEVENT_POOL}" \
|
||||
"${ZEVENT_VDEV_GUID}" "${spare_path}" 2>&1)"; zpool_rv=$?
|
||||
|
||||
if [ "${zpool_rv}" -ne 0 ]; then
|
||||
[ -n "${zpool_err}" ] && zed_log_err "zpool ${zpool_err}"
|
||||
else
|
||||
notify "${vdev_path}" "${spare_path}" "${num_errors}"
|
||||
rv=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
zed_unlock "${lockfile}"
|
||||
exit "${rv}"
|
||||
}
|
||||
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user