Add pool state /proc entry, "SUSPENDED" pools

1. Add a proc entry to display the pool's state:

$ cat /proc/spl/kstat/zfs/tank/state
ONLINE

This is done without using the spa config locks, so it will
never hang.

2. Fix 'zpool status' and 'zpool list -o health' output to print
"SUSPENDED" instead of "ONLINE" for suspended pools.

Reviewed-by: Olaf Faaland <faaland1@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: Richard Elling <Richard.Elling@RichardElling.com>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes #7331 
Closes #7563

module/spl/spl-kstat.c

@@ -389,7 +389,8 @@ kstat_seq_start(struct seq_file *f, loff_t *pos)
 
 	ksp->ks_snaptime = gethrtime();
 
-	if (!n && kstat_seq_show_headers(f))
+	if (!(ksp->ks_flags & KSTAT_FLAG_NO_HEADERS) && !n &&
+	    kstat_seq_show_headers(f))
 		return (NULL);
 
 	if (n >= ksp->ks_ndata)
@@ -539,7 +540,6 @@ __kstat_create(const char *ks_module, int ks_instance, const char *ks_name,
 	ASSERT(ks_module);
 	ASSERT(ks_instance == 0);
 	ASSERT(ks_name);
-	ASSERT(!(ks_flags & KSTAT_FLAG_UNSUPPORTED));
 
 	if ((ks_type == KSTAT_TYPE_INTR) || (ks_type == KSTAT_TYPE_IO))
 		ASSERT(ks_ndata == 1);

module/zfs/spa_misc.c

@@ -2254,6 +2254,45 @@ spa_set_missing_tvds(spa_t *spa, uint64_t missing)
 	spa->spa_missing_tvds = missing;
 }
 
+/*
+ * Return the pool state string ("ONLINE", "DEGRADED", "SUSPENDED", etc).
+ */
+const char *
+spa_state_to_name(spa_t *spa)
+{
+	vdev_state_t state = spa->spa_root_vdev->vdev_state;
+	vdev_aux_t aux = spa->spa_root_vdev->vdev_stat.vs_aux;
+
+	if (spa_suspended(spa) &&
+	    (spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE))
+		return ("SUSPENDED");
+
+	switch (state) {
+	case VDEV_STATE_CLOSED:
+	case VDEV_STATE_OFFLINE:
+		return ("OFFLINE");
+	case VDEV_STATE_REMOVED:
+		return ("REMOVED");
+	case VDEV_STATE_CANT_OPEN:
+		if (aux == VDEV_AUX_CORRUPT_DATA || aux == VDEV_AUX_BAD_LOG)
+			return ("FAULTED");
+		else if (aux == VDEV_AUX_SPLIT_POOL)
+			return ("SPLIT");
+		else
+			return ("UNAVAIL");
+	case VDEV_STATE_FAULTED:
+		return ("FAULTED");
+	case VDEV_STATE_DEGRADED:
+		return ("DEGRADED");
+	case VDEV_STATE_HEALTHY:
+		return ("ONLINE");
+	default:
+		break;
+	}
+
+	return ("UNKNOWN");
+}
+
 #if defined(_KERNEL)
 
 #include <linux/mod_compat.h>
@@ -2406,6 +2445,7 @@ EXPORT_SYMBOL(spa_namespace_lock);
 EXPORT_SYMBOL(spa_trust_config);
 EXPORT_SYMBOL(spa_missing_tvds_allowed);
 EXPORT_SYMBOL(spa_set_missing_tvds);
+EXPORT_SYMBOL(spa_state_to_name);
 
 /* BEGIN CSTYLED */
 module_param(zfs_flags, uint, 0644);

module/zfs/spa_stats.c

@@ -22,6 +22,8 @@
 #include <sys/zfs_context.h>
 #include <sys/spa_impl.h>
 #include <sys/vdev_impl.h>
+#include <sys/spa.h>
+#include <zfs_comutil.h>
 
 /*
  * Keeps stats on last N reads per spa_t, disabled by default.
@@ -997,6 +999,64 @@ spa_mmp_history_add(spa_t *spa, uint64_t txg, uint64_t timestamp,
 	return ((void *)smh);
 }
 
+static void *
+spa_state_addr(kstat_t *ksp, loff_t n)
+{
+	return (ksp->ks_private);	/* return the spa_t */
+}
+
+static int
+spa_state_data(char *buf, size_t size, void *data)
+{
+	spa_t *spa = (spa_t *)data;
+	(void) snprintf(buf, size, "%s\n", spa_state_to_name(spa));
+	return (0);
+}
+
+/*
+ * Return the state of the pool in /proc/spl/kstat/zfs/<pool>/state.
+ *
+ * This is a lock-less read of the pool's state (unlike using 'zpool', which
+ * can potentially block for seconds).  Because it doesn't block, it can useful
+ * as a pool heartbeat value.
+ */
+static void
+spa_state_init(spa_t *spa)
+{
+	spa_stats_history_t *ssh = &spa->spa_stats.state;
+	char *name;
+	kstat_t *ksp;
+
+	mutex_init(&ssh->lock, NULL, MUTEX_DEFAULT, NULL);
+
+	name = kmem_asprintf("zfs/%s", spa_name(spa));
+	ksp = kstat_create(name, 0, "state", "misc",
+	    KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
+
+	ssh->kstat = ksp;
+	if (ksp) {
+		ksp->ks_lock = &ssh->lock;
+		ksp->ks_data = NULL;
+		ksp->ks_private = spa;
+		ksp->ks_flags |= KSTAT_FLAG_NO_HEADERS;
+		kstat_set_raw_ops(ksp, NULL, spa_state_data, spa_state_addr);
+		kstat_install(ksp);
+	}
+
+	strfree(name);
+}
+
+static void
+spa_health_destroy(spa_t *spa)
+{
+	spa_stats_history_t *ssh = &spa->spa_stats.state;
+	kstat_t *ksp = ssh->kstat;
+	if (ksp)
+		kstat_delete(ksp);
+
+	mutex_destroy(&ssh->lock);
+}
+
 void
 spa_stats_init(spa_t *spa)
 {
@@ -1005,11 +1065,13 @@ spa_stats_init(spa_t *spa)
 	spa_tx_assign_init(spa);
 	spa_io_history_init(spa);
 	spa_mmp_history_init(spa);
+	spa_state_init(spa);
 }
 
 void
 spa_stats_destroy(spa_t *spa)
 {
+	spa_health_destroy(spa);
 	spa_tx_assign_destroy(spa);
 	spa_txg_history_destroy(spa);
 	spa_read_history_destroy(spa);