Multipath autoreplace, control enclosure LEDs, event rate limiting

1. Enable multipath autoreplace support for FMA.

This extends FMA autoreplace to work with multipath disks.  This
requires libdevmapper to be installed at build time.

2. Turn on/off fault LEDs when VDEVs become degraded/faulted/online

Set ZED_USE_ENCLOSURE_LEDS=1 in zed.rc to have ZED turn on/off the enclosure
LED for a drive when a drive becomes FAULTED/DEGRADED.  Your enclosure must
be supported by the Linux SES driver for this to work.  The enclosure LED
scripts work for multipath devices as well.  The scripts will clear the LED
when the fault is cleared.

3. Rate limit ZIO delay and checksum events so as not to flood ZED

ZIO delay and checksum events are rate limited to 5/sec in the zfs module.

Reviewed-by: Richard Laager <rlaager@wiktel.com>
Reviewed by: Don Brady <don.brady@intel.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes #2449 
Closes #3017 
Closes #5159

include/libzfs.h

@@ -280,6 +280,9 @@ extern nvlist_t *zpool_find_vdev_by_physpath(zpool_handle_t *, const char *,
 extern int zpool_label_disk_wait(char *, int);
 extern int zpool_label_disk(libzfs_handle_t *, zpool_handle_t *, char *);
 
+int dev_is_dm(char *devname);
+char *get_underlying_path(libzfs_handle_t *hdl, char *dev_name);
+
 /*
  * Functions to manage pool properties
  */
@@ -827,10 +830,12 @@ extern int zpool_fru_set(zpool_handle_t *, uint64_t, const char *);
  */
 extern boolean_t is_mpath_whole_disk(const char *);
 extern void update_vdev_config_dev_strs(nvlist_t *);
-extern char *zfs_strip_partition(libzfs_handle_t *, char *);
+extern char *zfs_strip_partition(char *);
 
 #ifdef HAVE_LIBUDEV
 struct udev_device;
+
+extern boolean_t udev_is_mpath(struct udev_device *dev);
 extern int zfs_device_get_devid(struct udev_device *, char *, size_t);
 extern int zfs_device_get_physical(struct udev_device *, char *, size_t);
 #endif

include/sys/Makefile.am

@@ -96,6 +96,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/zfs_delay.h \
 	$(top_srcdir)/include/sys/zfs_dir.h \
 	$(top_srcdir)/include/sys/zfs_fuid.h \
+	$(top_srcdir)/include/sys/zfs_ratelimit.h \
 	$(top_srcdir)/include/sys/zfs_rlock.h \
 	$(top_srcdir)/include/sys/zfs_sa.h \
 	$(top_srcdir)/include/sys/zfs_stat.h \

include/sys/fm/protocol.h

@@ -361,6 +361,7 @@ extern uint64_t fm_ena_generation_get(uint64_t);
 extern uchar_t fm_ena_format_get(uint64_t);
 extern uint64_t fm_ena_id_get(uint64_t);
 extern uint64_t fm_ena_time_get(uint64_t);
+extern void fm_erpt_dropped_increment(void);
 
 #ifdef	__cplusplus
 }

include/sys/vdev_impl.h

@@ -34,6 +34,7 @@
 #include <sys/vdev.h>
 #include <sys/dkio.h>
 #include <sys/uberblock_impl.h>
+#include <sys/zfs_ratelimit.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -243,6 +244,15 @@ struct vdev {
 	kmutex_t	vdev_dtl_lock;	/* vdev_dtl_{map,resilver}	*/
 	kmutex_t	vdev_stat_lock;	/* vdev_stat			*/
 	kmutex_t	vdev_probe_lock; /* protects vdev_probe_zio	*/
+
+	/*
+	 * We rate limit ZIO delay and ZIO checksum events, since they
+	 * can flood ZED with tons of events when a drive is acting up.
+	 */
+#define	DELAYS_PER_SECOND 5
+#define	CHECKSUMS_PER_SECOND 5
+	zfs_ratelimit_t vdev_delay_rl;
+	zfs_ratelimit_t vdev_checksum_rl;
 };
 
 #define	VDEV_RAIDZ_MAXPARITY	3

include/sys/zfs_ratelimit.h

@@ -0,0 +1,37 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2016, Lawrence Livermore National Security, LLC.
+ */
+
+#ifndef _SYS_ZFS_RATELIMIT_H
+#define	_SYS_ZFS_RATELIMIT_H
+
+#include <sys/zfs_context.h>
+
+typedef struct {
+	hrtime_t start;
+	unsigned int count;
+	unsigned int burst;		/* Number to allow per interval */
+	unsigned int interval;		/* Interval length in seconds */
+	kmutex_t lock;
+} zfs_ratelimit_t;
+
+int zfs_ratelimit(zfs_ratelimit_t *rl);
+void zfs_ratelimit_init(zfs_ratelimit_t *rl, unsigned int burst,
+    unsigned int interval);
+
+#endif	/* _SYS_ZFS_RATELIMIT_H */