mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-24 11:18:52 +03:00
Initial implementation of zed (ZFS Event Daemon)
zed monitors ZFS events. When a zevent is posted, zed will run any scripts that have been enabled for the corresponding zevent class. Multiple scripts may be invoked for a given zevent. The zevent nvpairs are passed to the scripts as environment variables. Events are processed synchronously by the single thread, and there is no maximum timeout for script execution. Consequently, a misbehaving script can delay (or forever block) the processing of subsequent zevents. Plans are to address this in future commits. Initial scripts have been developed to log events to syslog and send email in response to checksum/data/io errors and resilver.finish/scrub.finish events. By default, email will only be sent if the ZED_EMAIL variable is configured in zed.rc (which is serving as a config file of sorts until a proper configuration file is implemented). Signed-off-by: Chris Dunlap <cdunlap@llnl.gov> Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Issue #2
This commit is contained in:
Chris Dunlap
committed by
Brian Behlendorf
Brian Behlendorf
parent
8c7aa0cfc4
commit
9e246ac3d8
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Log all environment variables to ZED_DEBUG_LOG.
|
||||
#
|
||||
test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
|
||||
|
||||
# Override the default umask to restrict access to a newly-created logfile.
|
||||
umask 077
|
||||
|
||||
# Append stdout to the logfile after obtaining an advisory lock.
|
||||
exec >> "${ZED_DEBUG_LOG:=/tmp/zed.debug.log}"
|
||||
flock -x 1
|
||||
|
||||
printenv | sort
|
||||
echo
|
||||
|
||||
exit 0
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Log the zevent via syslog.
|
||||
#
|
||||
test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
|
||||
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" -p "${ZED_SYSLOG_PRIORITY:=daemon.notice}" \
|
||||
eid="${ZEVENT_EID}" class="${ZEVENT_SUBCLASS}" \
|
||||
"${ZEVENT_POOL:+pool=$ZEVENT_POOL}"
|
||||
|
||||
exit 0
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
io-email.sh
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Send email to ZED_EMAIL in response to a DATA zevent.
|
||||
# Only one message per ZED_EMAIL_INTERVAL_SECS will be sent for a given
|
||||
# class/pool combination. This protects against spamming the recipient
|
||||
# should multiple events occur together in time for the same pool.
|
||||
# Exit codes:
|
||||
# 0: email sent
|
||||
# 1: email failed
|
||||
# 2: email suppressed
|
||||
# 3: missing executable
|
||||
# 4: unsupported event class
|
||||
# 5: internal error
|
||||
# State File Format:
|
||||
# POOL:TIME_OF_LAST_EMAIL
|
||||
#
|
||||
test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
|
||||
|
||||
test -n "${ZEVENT_POOL}" || exit 5
|
||||
test -n "${ZEVENT_SUBCLASS}" || exit 5
|
||||
|
||||
if test "${ZEVENT_SUBCLASS}" != "data"; then \
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: unsupported event class \"${ZEVENT_SUBCLASS}\"
|
||||
exit 4
|
||||
fi
|
||||
|
||||
# Only send email if ZED_EMAIL has been configured.
|
||||
test -n "${ZED_EMAIL}" || exit 2
|
||||
|
||||
# Ensure requisite executables are installed.
|
||||
if ! command -v "${MAIL:=mail}" >/dev/null 2>&1; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${MAIL}" not installed
|
||||
exit 3
|
||||
fi
|
||||
|
||||
NAME="zed.${ZEVENT_SUBCLASS}.email"
|
||||
LOCKFILE="${ZED_LOCKDIR:=/var/lock}/${NAME}.lock"
|
||||
STATEFILE="${ZED_RUNDIR:=/var/run}/${NAME}.state"
|
||||
|
||||
# Obtain lock to ensure mutual exclusion for accessing state.
|
||||
exec 8> "${LOCKFILE}"
|
||||
flock -x 8
|
||||
|
||||
# Query state for last time email was sent for this pool.
|
||||
TIME_NOW=`date +%s`
|
||||
TIME_LAST=`egrep "^${ZEVENT_POOL}:" "${STATEFILE}" 2>/dev/null | cut -d: -f2`
|
||||
if test -n "${TIME_LAST}"; then
|
||||
TIME_DELTA=`expr "${TIME_NOW}" - "${TIME_LAST}"`
|
||||
if test "${TIME_DELTA}" -lt "${ZED_EMAIL_INTERVAL_SECS:=3600}"; then
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
"${MAIL}" -s "ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on `hostname`" \
|
||||
"${ZED_EMAIL}" <<EOF
|
||||
A ZFS ${ZEVENT_SUBCLASS} error has been detected:
|
||||
|
||||
eid: ${ZEVENT_EID}
|
||||
host: `hostname`
|
||||
time: ${ZEVENT_TIME_STRING}
|
||||
pool: ${ZEVENT_POOL}
|
||||
EOF
|
||||
MAIL_STATUS=$?
|
||||
|
||||
# Update state.
|
||||
egrep -v "^${ZEVENT_POOL}:" "${STATEFILE}" 2>/dev/null > "${STATEFILE}.$$"
|
||||
echo "${ZEVENT_POOL}:${TIME_NOW}" >> "${STATEFILE}.$$"
|
||||
mv -f "${STATEFILE}.$$" "${STATEFILE}"
|
||||
|
||||
if test "${MAIL_STATUS}" -ne 0; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${MAIL}" exit="${MAIL_STATUS}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Send email to ZED_EMAIL in response to a given zevent.
|
||||
# This is a generic script than can be symlinked to a file in the zed
|
||||
# enabled-scripts directory in order to have email sent when a particular
|
||||
# class of zevents occurs. The symlink filename must begin with the zevent
|
||||
# (sub)class string (eg, "probe_failure-email.sh" for the "probe_failure"
|
||||
# subclass). Refer to the zed(8) manpage for details.
|
||||
# Exit codes:
|
||||
# 0: email sent
|
||||
# 1: email failed
|
||||
# 2: email suppressed
|
||||
# 3: missing executable
|
||||
#
|
||||
test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
|
||||
|
||||
# Only send email if ZED_EMAIL has been configured.
|
||||
test -n "${ZED_EMAIL}" || exit 2
|
||||
|
||||
# Ensure requisite executables are installed.
|
||||
if ! command -v "${MAIL:=mail}" >/dev/null 2>&1; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${MAIL}" not installed
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Override the default umask to restrict access to the msgbody tmpfile.
|
||||
umask 077
|
||||
|
||||
SUBJECT="ZFS ${ZEVENT_SUBCLASS} event"
|
||||
test -n "${ZEVENT_POOL}" && SUBJECT="${SUBJECT} for ${ZEVENT_POOL}"
|
||||
SUBJECT="${SUBJECT} on `hostname`"
|
||||
|
||||
MSGBODY="${TMPDIR:=/tmp}/`basename \"$0\"`.$$"
|
||||
{
|
||||
echo "A ZFS ${ZEVENT_SUBCLASS} event has been posted:"
|
||||
echo
|
||||
echo " eid: ${ZEVENT_EID}"
|
||||
echo " host: `hostname`"
|
||||
echo " time: ${ZEVENT_TIME_STRING}"
|
||||
test -n "${ZEVENT_VDEV_TYPE}" -a -n "${ZEVENT_VDEV_PATH}" && \
|
||||
echo " vdev: ${ZEVENT_VDEV_TYPE}:${ZEVENT_VDEV_PATH}"
|
||||
test -n "${ZEVENT_POOL}" -a -x "${ZPOOL}" && \
|
||||
"${ZPOOL}" status "${ZEVENT_POOL}"
|
||||
} > "${MSGBODY}"
|
||||
|
||||
test -f "${MSGBODY}" && "${MAIL}" -s "${SUBJECT}" "${ZED_EMAIL}" < "${MSGBODY}"
|
||||
MAIL_STATUS=$?
|
||||
rm -f "${MSGBODY}"
|
||||
|
||||
if test "${MAIL_STATUS}" -ne 0; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${MAIL}" exit="${MAIL_STATUS}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Send email to ZED_EMAIL in response to a CHECKSUM or IO zevent.
|
||||
# Only one message per ZED_EMAIL_INTERVAL_SECS will be sent for a given
|
||||
# class/pool/vdev combination. This protects against spamming the recipient
|
||||
# should multiple events occur together in time for the same pool/device.
|
||||
# Exit codes:
|
||||
# 0: email sent
|
||||
# 1: email failed
|
||||
# 2: email suppressed
|
||||
# 3: missing executable
|
||||
# 4: unsupported event class
|
||||
# 5: internal error
|
||||
# State File Format:
|
||||
# POOL:VDEV_PATH:TIME_OF_LAST_EMAIL
|
||||
#
|
||||
test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
|
||||
|
||||
test -n "${ZEVENT_POOL}" || exit 5
|
||||
test -n "${ZEVENT_SUBCLASS}" || exit 5
|
||||
test -n "${ZEVENT_VDEV_PATH}" || exit 5
|
||||
|
||||
if test "${ZEVENT_SUBCLASS}" != "checksum" \
|
||||
-a "${ZEVENT_SUBCLASS}" != "io"; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: unsupported event class \"${ZEVENT_SUBCLASS}\"
|
||||
exit 4
|
||||
fi
|
||||
|
||||
# Only send email if ZED_EMAIL has been configured.
|
||||
test -n "${ZED_EMAIL}" || exit 2
|
||||
|
||||
# Ensure requisite executables are installed.
|
||||
if ! command -v "${MAIL:=mail}" >/dev/null 2>&1; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${MAIL}" not installed
|
||||
exit 3
|
||||
fi
|
||||
|
||||
NAME="zed.${ZEVENT_SUBCLASS}.email"
|
||||
LOCKFILE="${ZED_LOCKDIR:=/var/lock}/${NAME}.lock"
|
||||
STATEFILE="${ZED_RUNDIR:=/var/run}/${NAME}.state"
|
||||
|
||||
# Obtain lock to ensure mutual exclusion for accessing state.
|
||||
exec 8> "${LOCKFILE}"
|
||||
flock -x 8
|
||||
|
||||
# Query state for last time email was sent for this pool/vdev.
|
||||
TIME_NOW=`date +%s`
|
||||
TIME_LAST=`egrep "^${ZEVENT_POOL}:${ZEVENT_VDEV_PATH}:" "${STATEFILE}" \
|
||||
2>/dev/null | cut -d: -f3`
|
||||
if test -n "${TIME_LAST}"; then
|
||||
TIME_DELTA=`expr "${TIME_NOW}" - "${TIME_LAST}"`
|
||||
if test "${TIME_DELTA}" -lt "${ZED_EMAIL_INTERVAL_SECS:=3600}"; then
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
"${MAIL}" -s "ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on `hostname`" \
|
||||
"${ZED_EMAIL}" <<EOF
|
||||
A ZFS ${ZEVENT_SUBCLASS} error has been detected:
|
||||
|
||||
eid: ${ZEVENT_EID}
|
||||
host: `hostname`
|
||||
time: ${ZEVENT_TIME_STRING}
|
||||
pool: ${ZEVENT_POOL}
|
||||
vdev: ${ZEVENT_VDEV_TYPE}:${ZEVENT_VDEV_PATH}
|
||||
EOF
|
||||
MAIL_STATUS=$?
|
||||
|
||||
# Update state.
|
||||
egrep -v "^${ZEVENT_POOL}:${ZEVENT_VDEV_PATH}:" "${STATEFILE}" \
|
||||
2>/dev/null > "${STATEFILE}.$$"
|
||||
echo "${ZEVENT_POOL}:${ZEVENT_VDEV_PATH}:${TIME_NOW}" >> "${STATEFILE}.$$"
|
||||
mv -f "${STATEFILE}.$$" "${STATEFILE}"
|
||||
|
||||
if test "${MAIL_STATUS}" -ne 0; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${MAIL}" exit="${MAIL_STATUS}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+1
@@ -0,0 +1 @@
|
||||
scrub.finish-email.sh
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Send email to ZED_EMAIL in response to a RESILVER.FINISH or SCRUB.FINISH.
|
||||
# By default, "zpool status" output will only be included in the email for
|
||||
# a scrub.finish zevent if the pool is not healthy; to always include its
|
||||
# output, set ZED_EMAIL_VERBOSE=1.
|
||||
# Exit codes:
|
||||
# 0: email sent
|
||||
# 1: email failed
|
||||
# 2: email suppressed
|
||||
# 3: missing executable
|
||||
# 4: unsupported event class
|
||||
# 5: internal error
|
||||
#
|
||||
test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
|
||||
|
||||
test -n "${ZEVENT_POOL}" || exit 5
|
||||
test -n "${ZEVENT_SUBCLASS}" || exit 5
|
||||
|
||||
if test "${ZEVENT_SUBCLASS}" = "resilver.finish"; then
|
||||
ACTION="resilvering"
|
||||
elif test "${ZEVENT_SUBCLASS}" = "scrub.finish"; then
|
||||
ACTION="scrubbing"
|
||||
else
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: unsupported event class \"${ZEVENT_SUBCLASS}\"
|
||||
exit 4
|
||||
fi
|
||||
|
||||
# Only send email if ZED_EMAIL has been configured.
|
||||
test -n "${ZED_EMAIL}" || exit 2
|
||||
|
||||
# Ensure requisite executables are installed.
|
||||
if ! command -v "${MAIL:=mail}" >/dev/null 2>&1; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${MAIL}" not installed
|
||||
exit 3
|
||||
fi
|
||||
if ! test -x "${ZPOOL}"; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${ZPOOL}" not installed
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# For scrub, suppress email if pool is healthy and verbosity is not enabled.
|
||||
if test "${ZEVENT_SUBCLASS}" = "scrub.finish"; then
|
||||
HEALTHY=`"${ZPOOL}" status -x "${ZEVENT_POOL}" | \
|
||||
grep "'${ZEVENT_POOL}' is healthy"`
|
||||
test -n "${HEALTHY}" -a "${ZED_EMAIL_VERBOSE:=0}" = 0 && exit 2
|
||||
fi
|
||||
|
||||
"${MAIL}" -s "ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on `hostname`" \
|
||||
"${ZED_EMAIL}" <<EOF
|
||||
A ZFS pool has finished ${ACTION}:
|
||||
|
||||
eid: ${ZEVENT_EID}
|
||||
host: `hostname`
|
||||
time: ${ZEVENT_TIME_STRING}
|
||||
`"${ZPOOL}" status "${ZEVENT_POOL}"`
|
||||
EOF
|
||||
MAIL_STATUS=$?
|
||||
|
||||
if test "${MAIL_STATUS}" -ne 0; then
|
||||
logger -t "${ZED_SYSLOG_TAG:=zed}" \
|
||||
-p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
|
||||
`basename "$0"`: "${MAIL}" exit="${MAIL_STATUS}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,28 @@
|
||||
# zed.rc
|
||||
|
||||
# Absolute path to the debug output file.
|
||||
#ZED_DEBUG_LOG="/tmp/zed.debug.log"
|
||||
|
||||
# Email address of the zpool administrator.
|
||||
# Email will only be sent if ZED_EMAIL is defined.
|
||||
#ZED_EMAIL="root"
|
||||
|
||||
# Email verbosity.
|
||||
# If set to 0, suppress email if the pool is healthy.
|
||||
# If set to 1, send email regardless of pool health.
|
||||
#ZED_EMAIL_VERBOSE=0
|
||||
|
||||
# Minimum number of seconds between emails sent for a similar event.
|
||||
#ZED_EMAIL_INTERVAL_SECS="3600"
|
||||
|
||||
# Default directory for zed lock files.
|
||||
#ZED_LOCKDIR="/var/lock"
|
||||
|
||||
# Default directory for zed state files.
|
||||
#ZED_RUNDIR="/var/run"
|
||||
|
||||
# The syslog priority (eg, specified as a "facility.level" pair).
|
||||
#ZED_SYSLOG_PRIORITY="daemon.notice"
|
||||
|
||||
# The syslog tag for marking zed events.
|
||||
#ZED_SYSLOG_TAG="zed"
|
||||
Reference in New Issue
Block a user