Illumos #2762: zpool command should have better support for feature flags

2762 zpool command should have better support for feature flags
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Eric Schrock <Eric.Schrock@delphix.com>

References:
  illumos/illumos-gate@57221772c3
  https://www.illumos.org/issues/2762

Ported-by: Brian Behlendorf <behlendorf1@llnl.gov>

cmd/zpool/zpool_main.c

@@ -391,6 +391,18 @@ print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
 	}
 }
 
+static boolean_t
+prop_list_contains_feature(nvlist_t *proplist)
+{
+	nvpair_t *nvp;
+	for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
+	    nvp = nvlist_next_nvpair(proplist, nvp)) {
+		if (zpool_prop_feature(nvpair_name(nvp)))
+			return (B_TRUE);
+	}
+	return (B_FALSE);
+}
+
 /*
  * Add a property pair (name, string-value) into a property nvlist.
  */
@@ -414,12 +426,30 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 	proplist = *props;
 
 	if (poolprop) {
+		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
+
 		if ((prop = zpool_name_to_prop(propname)) == ZPROP_INVAL &&
 		    !zpool_prop_feature(propname)) {
 			(void) fprintf(stderr, gettext("property '%s' is "
 			    "not a valid pool property\n"), propname);
 			return (2);
 		}
+
+		/*
+		 * feature@ properties and version should not be specified
+		 * at the same time.
+		 */
+		if ((prop == ZPROP_INVAL && zpool_prop_feature(propname) &&
+		    nvlist_exists(proplist, vname)) ||
+		    (prop == ZPOOL_PROP_VERSION &&
+		    prop_list_contains_feature(proplist))) {
+			(void) fprintf(stderr, gettext("'feature@' and "
+			    "'version' properties cannot be specified "
+			    "together\n"));
+			return (2);
+		}
+
+
 		if (zpool_prop_feature(propname))
 			normnm = propname;
 		else
@@ -1482,8 +1512,8 @@ show_import(nvlist_t *config)
 		break;
 
 	case ZPOOL_STATUS_VERSION_OLDER:
-		(void) printf(gettext(" status: The pool is formatted using an "
+		(void) printf(gettext(" status: The pool is formatted using a "
-		    "older on-disk version.\n"));
+		    "legacy on-disk version.\n"));
 		break;
 
 	case ZPOOL_STATUS_VERSION_NEWER:
@@ -1491,6 +1521,11 @@ show_import(nvlist_t *config)
 		    "incompatible version.\n"));
 		break;
 
+	case ZPOOL_STATUS_FEAT_DISABLED:
+		(void) printf(gettext(" status: Some supported features are "
+		    "not enabled on the pool.\n"));
+		break;
+
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
 		(void) printf(gettext("status: The pool uses the following "
 		    "feature(s) not supported on this sytem:\n"));
@@ -1537,19 +1572,21 @@ show_import(nvlist_t *config)
 	 * Print out an action according to the overall state of the pool.
 	 */
 	if (vs->vs_state == VDEV_STATE_HEALTHY) {
-		if (reason == ZPOOL_STATUS_VERSION_OLDER)
+		if (reason == ZPOOL_STATUS_VERSION_OLDER ||
+		    reason == ZPOOL_STATUS_FEAT_DISABLED) {
 			(void) printf(gettext(" action: The pool can be "
 			    "imported using its name or numeric identifier, "
 			    "though\n\tsome features will not be available "
 			    "without an explicit 'zpool upgrade'.\n"));
-		else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH)
+		} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
 			(void) printf(gettext(" action: The pool can be "
 			    "imported using its name or numeric "
 			    "identifier and\n\tthe '-f' flag.\n"));
-		else
+		} else {
 			(void) printf(gettext(" action: The pool can be "
 			    "imported using its name or numeric "
 			    "identifier.\n"));
+		}
 	} else if (vs->vs_state == VDEV_STATE_DEGRADED) {
 		(void) printf(gettext(" action: The pool can be imported "
 		    "despite missing or damaged devices.  The\n\tfault "
@@ -4042,12 +4079,13 @@ status_callback(zpool_handle_t *zhp, void *data)
 		break;
 
 	case ZPOOL_STATUS_VERSION_OLDER:
-		(void) printf(gettext("status: The pool is formatted using an "
+		(void) printf(gettext("status: The pool is formatted using a "
-		    "older on-disk format.  The pool can\n\tstill be used, but "
+		    "legacy on-disk format.  The pool can\n\tstill be used, "
-		    "some features are unavailable.\n"));
+		    "but some features are unavailable.\n"));
 		(void) printf(gettext("action: Upgrade the pool using 'zpool "
 		    "upgrade'.  Once this is done, the\n\tpool will no longer "
-		    "be accessible on older software versions.\n"));
+		    "be accessible on software that does not support feature\n"
+		    "\tflags.\n"));
 		break;
 
 	case ZPOOL_STATUS_VERSION_NEWER:
@@ -4059,6 +4097,16 @@ status_callback(zpool_handle_t *zhp, void *data)
 		    "backup.\n"));
 		break;
 
+	case ZPOOL_STATUS_FEAT_DISABLED:
+		(void) printf(gettext("status: Some supported features are not "
+		    "enabled on the pool. The pool can\n\tstill be used, but "
+		    "some features are unavailable.\n"));
+		(void) printf(gettext("action: Enable all features using "
+		    "'zpool upgrade'. Once this is done,\n\tthe pool may no "
+		    "longer be accessible by software that does not support\n\t"
+		    "the features. See zpool-features(5) for details.\n"));
+		break;
+
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
 		(void) printf(gettext("status: The pool cannot be accessed on "
 		    "this system because it uses the\n\tfollowing feature(s) "
@@ -4288,58 +4336,162 @@ zpool_do_status(int argc, char **argv)
 }
 
 typedef struct upgrade_cbdata {
-	int	cb_all;
 	int	cb_first;
-	int	cb_newer;
 	int	cb_argc;
 	uint64_t cb_version;
 	char	**cb_argv;
 } upgrade_cbdata_t;
 
+static int
+upgrade_version(zpool_handle_t *zhp, uint64_t version)
+{
+	int ret;
+	nvlist_t *config;
+	uint64_t oldversion;
+
+	config = zpool_get_config(zhp, NULL);
+	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+	    &oldversion) == 0);
+
+	assert(SPA_VERSION_IS_SUPPORTED(oldversion));
+	assert(oldversion < version);
+
+	ret = zpool_upgrade(zhp, version);
+	if (ret != 0)
+		return (ret);
+
+	if (version >= SPA_VERSION_FEATURES) {
+		(void) printf(gettext("Successfully upgraded "
+		    "'%s' from version %llu to feature flags.\n"),
+		    zpool_get_name(zhp), (u_longlong_t) oldversion);
+	} else {
+		(void) printf(gettext("Successfully upgraded "
+		    "'%s' from version %llu to version %llu.\n"),
+		    zpool_get_name(zhp), (u_longlong_t) oldversion,
+		    (u_longlong_t) version);
+	}
+
+	return (0);
+}
+
+static int
+upgrade_enable_all(zpool_handle_t *zhp, int *countp)
+{
+	int i, ret, count;
+	boolean_t firstff = B_TRUE;
+	nvlist_t *enabled = zpool_get_features(zhp);
+
+	count = 0;
+	for (i = 0; i < SPA_FEATURES; i++) {
+		const char *fname = spa_feature_table[i].fi_uname;
+		const char *fguid = spa_feature_table[i].fi_guid;
+		if (!nvlist_exists(enabled, fguid)) {
+			char *propname;
+			verify(-1 != asprintf(&propname, "feature@%s", fname));
+			ret = zpool_set_prop(zhp, propname,
+			    ZFS_FEATURE_ENABLED);
+			if (ret != 0) {
+				free(propname);
+				return (ret);
+			}
+			count++;
+
+			if (firstff) {
+				(void) printf(gettext("Enabled the "
+				    "following features on '%s':\n"),
+				    zpool_get_name(zhp));
+				firstff = B_FALSE;
+			}
+			(void) printf(gettext("  %s\n"), fname);
+			free(propname);
+		}
+	}
+
+	if (countp != NULL)
+		*countp = count;
+	return (0);
+}
+
 static int
 upgrade_cb(zpool_handle_t *zhp, void *arg)
 {
 	upgrade_cbdata_t *cbp = arg;
 	nvlist_t *config;
 	uint64_t version;
-	int ret = 0;
+	boolean_t printnl = B_FALSE;
+	int ret;
 
 	config = zpool_get_config(zhp, NULL);
 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
 	    &version) == 0);
 
-	if (!cbp->cb_newer && SPA_VERSION_IS_SUPPORTED(version) &&
+	assert(SPA_VERSION_IS_SUPPORTED(version));
-	    version != SPA_VERSION) {
-		if (!cbp->cb_all) {
-			if (cbp->cb_first) {
-				(void) printf(gettext("The following pools are "
-				    "out of date, and can be upgraded.  After "
-				    "being\nupgraded, these pools will no "
-				    "longer be accessible by older software "
-				    "versions.\n\n"));
-				(void) printf(gettext("VER  POOL\n"));
-				(void) printf(gettext("---  ------------\n"));
-				cbp->cb_first = B_FALSE;
-			}
 
-			(void) printf("%2llu   %s\n", (u_longlong_t)version,
+	if (version < cbp->cb_version) {
-			    zpool_get_name(zhp));
+		cbp->cb_first = B_FALSE;
-		} else {
+		ret = upgrade_version(zhp, cbp->cb_version);
+		if (ret != 0)
+			return (ret);
+		printnl = B_TRUE;
+
+#if 0
+		/*
+		 * XXX: This code can be enabled when Illumos commit
+		 * 4445fffbbb1ea25fd0e9ea68b9380dd7a6709025 is merged.
+		 * It reworks the history logging among other things.
+		 */
+
+		/*
+		 * If they did "zpool upgrade -a", then we could
+		 * be doing ioctls to different pools.  We need
+		 * to log this history once to each pool, and bypass
+		 * the normal history logging that happens in main().
+		 */
+		(void) zpool_log_history(g_zfs, history_str);
+		log_history = B_FALSE;
+#endif
+	}
+
+	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
+		int count;
+		ret = upgrade_enable_all(zhp, &count);
+		if (ret != 0)
+			return (ret);
+
+		if (count > 0) {
 			cbp->cb_first = B_FALSE;
-			ret = zpool_upgrade(zhp, cbp->cb_version);
+			printnl = B_TRUE;
-			if (!ret) {
-				(void) printf(gettext("Successfully upgraded "
-				    "'%s'\n\n"), zpool_get_name(zhp));
-			}
 		}
-	} else if (cbp->cb_newer && !SPA_VERSION_IS_SUPPORTED(version)) {
+	}
-		assert(!cbp->cb_all);
 
+	if (printnl) {
+		(void) printf(gettext("\n"));
+	}
+
+	return (0);
+}
+
+static int
+upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
+{
+	upgrade_cbdata_t *cbp = arg;
+	nvlist_t *config;
+	uint64_t version;
+
+	config = zpool_get_config(zhp, NULL);
+	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+	    &version) == 0);
+
+	assert(SPA_VERSION_IS_SUPPORTED(version));
+
+	if (version < SPA_VERSION_FEATURES) {
 		if (cbp->cb_first) {
 			(void) printf(gettext("The following pools are "
-			    "formatted using an unsupported software version "
+			    "formatted with legacy version numbers and can\n"
-			    "and\ncannot be accessed on the current "
+			    "be upgraded to use feature flags.  After "
-			    "system.\n\n"));
+			    "being upgraded, these pools\nwill no "
+			    "longer be accessible by software that does not "
+			    "support feature\nflags.\n\n"));
 			(void) printf(gettext("VER  POOL\n"));
 			(void) printf(gettext("---  ------------\n"));
 			cbp->cb_first = B_FALSE;
@@ -4349,14 +4501,65 @@ upgrade_cb(zpool_handle_t *zhp, void *arg)
 		    zpool_get_name(zhp));
 	}
 
-	zpool_close(zhp);
+	return (0);
-	return (ret);
+}
+
+static int
+upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
+{
+	upgrade_cbdata_t *cbp = arg;
+	nvlist_t *config;
+	uint64_t version;
+
+	config = zpool_get_config(zhp, NULL);
+	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+	    &version) == 0);
+
+	if (version >= SPA_VERSION_FEATURES) {
+		int i;
+		boolean_t poolfirst = B_TRUE;
+		nvlist_t *enabled = zpool_get_features(zhp);
+
+		for (i = 0; i < SPA_FEATURES; i++) {
+			const char *fguid = spa_feature_table[i].fi_guid;
+			const char *fname = spa_feature_table[i].fi_uname;
+			if (!nvlist_exists(enabled, fguid)) {
+				if (cbp->cb_first) {
+					(void) printf(gettext("\nSome "
+					    "supported features are not "
+					    "enabled on the following pools. "
+					    "Once a\nfeature is enabled the "
+					    "pool may become incompatible with "
+					    "software\nthat does not support "
+					    "the feature. See "
+					    "zpool-features(5) for "
+					    "details.\n\n"));
+					(void) printf(gettext("POOL  "
+					    "FEATURE\n"));
+					(void) printf(gettext("------"
+					    "---------\n"));
+					cbp->cb_first = B_FALSE;
+				}
+
+				if (poolfirst) {
+					(void) printf(gettext("%s\n"),
+					    zpool_get_name(zhp));
+					poolfirst = B_FALSE;
+				}
+
+				(void) printf(gettext("      %s\n"), fname);
+			}
+		}
+	}
+
+	return (0);
 }
 
 /* ARGSUSED */
 static int
 upgrade_one(zpool_handle_t *zhp, void *data)
 {
+	boolean_t printnl = B_FALSE;
 	upgrade_cbdata_t *cbp = data;
 	uint64_t cur_version;
 	int ret;
@@ -4371,26 +4574,45 @@ upgrade_one(zpool_handle_t *zhp, void *data)
 	cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
 	if (cur_version > cbp->cb_version) {
 		(void) printf(gettext("Pool '%s' is already formatted "
-		    "using more current version '%llu'.\n"),
+		    "using more current version '%llu'.\n\n"),
 		    zpool_get_name(zhp), (u_longlong_t) cur_version);
 		return (0);
 	}
-	if (cur_version == cbp->cb_version) {
+
+	if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
 		(void) printf(gettext("Pool '%s' is already formatted "
-		    "using the current version.\n"), zpool_get_name(zhp));
+		    "using version %llu.\n\n"), zpool_get_name(zhp),
+		    (u_longlong_t) cbp->cb_version);
 		return (0);
 	}
 
-	ret = zpool_upgrade(zhp, cbp->cb_version);
+	if (cur_version != cbp->cb_version) {
-
+		printnl = B_TRUE;
-	if (!ret) {
+		ret = upgrade_version(zhp, cbp->cb_version);
-		(void) printf(gettext("Successfully upgraded '%s' "
+		if (ret != 0)
-		    "from version %llu to version %llu\n\n"),
+			return (ret);
-		    zpool_get_name(zhp), (u_longlong_t)cur_version,
-		    (u_longlong_t)cbp->cb_version);
 	}
 
-	return (ret != 0);
+	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
+		int count = 0;
+		ret = upgrade_enable_all(zhp, &count);
+		if (ret != 0)
+			return (ret);
+
+		if (count != 0) {
+			printnl = B_TRUE;
+		} else if (cur_version == SPA_VERSION) {
+			(void) printf(gettext("Pool '%s' already has all "
+			    "supported features enabled.\n"),
+			    zpool_get_name(zhp));
+		}
+	}
+
+	if (printnl) {
+		(void) printf(gettext("\n"));
+	}
+
+	return (0);
 }
 
 /*
@@ -4409,6 +4631,7 @@ zpool_do_upgrade(int argc, char **argv)
 	upgrade_cbdata_t cb = { 0 };
 	int ret = 0;
 	boolean_t showversions = B_FALSE;
+	boolean_t upgradeall = B_FALSE;
 	char *end;
 
 
@@ -4416,7 +4639,7 @@ zpool_do_upgrade(int argc, char **argv)
 	while ((c = getopt(argc, argv, ":avV:")) != -1) {
 		switch (c) {
 		case 'a':
-			cb.cb_all = B_TRUE;
+			upgradeall = B_TRUE;
 			break;
 		case 'v':
 			showversions = B_TRUE;
@@ -4449,19 +4672,19 @@ zpool_do_upgrade(int argc, char **argv)
 
 	if (cb.cb_version == 0) {
 		cb.cb_version = SPA_VERSION;
-	} else if (!cb.cb_all && argc == 0) {
+	} else if (!upgradeall && argc == 0) {
 		(void) fprintf(stderr, gettext("-V option is "
 		    "incompatible with other arguments\n"));
 		usage(B_FALSE);
 	}
 
 	if (showversions) {
-		if (cb.cb_all || argc != 0) {
+		if (upgradeall || argc != 0) {
 			(void) fprintf(stderr, gettext("-v option is "
 			    "incompatible with other arguments\n"));
 			usage(B_FALSE);
 		}
-	} else if (cb.cb_all) {
+	} else if (upgradeall) {
 		if (argc != 0) {
 			(void) fprintf(stderr, gettext("-a option should not "
 			    "be used along with a pool name\n"));
@@ -4471,9 +4694,25 @@ zpool_do_upgrade(int argc, char **argv)
 
 	(void) printf(gettext("This system supports ZFS pool feature "
 	    "flags.\n\n"));
-	cb.cb_first = B_TRUE;
 	if (showversions) {
-		(void) printf(gettext("The following versions are "
+		int i;
+
+		(void) printf(gettext("The following features are "
+		    "supported:\n\n"));
+		(void) printf(gettext("FEAT DESCRIPTION\n"));
+		(void) printf("----------------------------------------------"
+		    "---------------\n");
+		for (i = 0; i < SPA_FEATURES; i++) {
+			zfeature_info_t *fi = &spa_feature_table[i];
+			const char *ro = fi->fi_can_readonly ?
+			    " (read-only compatible)" : "";
+
+			(void) printf("%-37s%s\n", fi->fi_uname, ro);
+			(void) printf("     %s\n", fi->fi_desc);
+		}
+		(void) printf("\n");
+
+		(void) printf(gettext("The following legacy versions are also "
 		    "supported:\n\n"));
 		(void) printf(gettext("VER  DESCRIPTION\n"));
 		(void) printf("---  -----------------------------------------"
@@ -4516,32 +4755,44 @@ zpool_do_upgrade(int argc, char **argv)
 		(void) printf(gettext("\nFor more information on a particular "
 		    "version, including supported releases,\n"));
 		(void) printf(gettext("see the ZFS Administration Guide.\n\n"));
-	} else if (argc == 0) {
+	} else if (argc == 0 && upgradeall) {
-		int notfound;
+		cb.cb_first = B_TRUE;
-
 		ret = zpool_iter(g_zfs, upgrade_cb, &cb);
-		notfound = cb.cb_first;
+		if (ret == 0 && cb.cb_first) {
-
+			if (cb.cb_version == SPA_VERSION) {
-		if (!cb.cb_all && ret == 0) {
+				(void) printf(gettext("All pools are already "
-			if (!cb.cb_first)
+				    "formatted using feature flags.\n\n"));
-				(void) printf("\n");
+				(void) printf(gettext("Every feature flags "
-			cb.cb_first = B_TRUE;
+				    "pool already has all supported features "
-			cb.cb_newer = B_TRUE;
+				    "enabled.\n"));
-			ret = zpool_iter(g_zfs, upgrade_cb, &cb);
+			} else {
-			if (!cb.cb_first) {
+				(void) printf(gettext("All pools are already "
-				notfound = B_FALSE;
+				    "formatted with version %llu or higher.\n"),
-				(void) printf("\n");
+				    (u_longlong_t) cb.cb_version);
 			}
 		}
+	} else if (argc == 0) {
+		cb.cb_first = B_TRUE;
+		ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
+		assert(ret == 0);
 
-		if (ret == 0) {
+		if (cb.cb_first) {
-			if (notfound)
+			(void) printf(gettext("All pools are formatted "
-				(void) printf(gettext("All pools are formatted "
+			    "using feature flags.\n\n"));
-				    "using this version.\n"));
+		} else {
-			else if (!cb.cb_all)
+			(void) printf(gettext("\nUse 'zpool upgrade -v' "
-				(void) printf(gettext("Use 'zpool upgrade -v' "
+			    "for a list of available legacy versions.\n"));
-				    "for a list of available versions and "
+		}
-				    "their associated\nfeatures.\n"));
+
+		cb.cb_first = B_TRUE;
+		ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
+		assert(ret == 0);
+
+		if (cb.cb_first) {
+			(void) printf(gettext("Every feature flags pool has "
+			    "all supported features enabled.\n"));
+		} else {
+			(void) printf(gettext("\n"));
 		}
 	} else {
 		ret = for_each_pool(argc, argv, B_FALSE, NULL,

include/libzfs.h

@@ -322,7 +322,8 @@ typedef enum {
 	 * requiring administrative attention.  There is no corresponding
 	 * message ID.
 	 */
-	ZPOOL_STATUS_VERSION_OLDER,	/* older on-disk version */
+	ZPOOL_STATUS_VERSION_OLDER,	/* older legacy on-disk version */
+	ZPOOL_STATUS_FEAT_DISABLED,	/* supported features are disabled */
 	ZPOOL_STATUS_RESILVERING,	/* device being resilvered */
 	ZPOOL_STATUS_OFFLINE_DEV,	/* device online */
 	ZPOOL_STATUS_REMOVED_DEV,	/* removed device */

include/sys/fs/zfs.h

@@ -524,6 +524,7 @@ typedef struct zpool_rewind_policy {
 #define	ZPOOL_CONFIG_LOAD_INFO		"load_info"	/* not stored on disk */
 #define	ZPOOL_CONFIG_REWIND_INFO	"rewind_info"	/* not stored on disk */
 #define	ZPOOL_CONFIG_UNSUP_FEAT		"unsup_feat"	/* not stored on disk */
+#define	ZPOOL_CONFIG_ENABLED_FEAT	"enabled_feat"	/* not stored on disk */
 #define	ZPOOL_CONFIG_CAN_RDONLY		"can_rdonly"	/* not stored on disk */
 #define	ZPOOL_CONFIG_FEATURES_FOR_READ	"features_for_read"
 #define	ZPOOL_CONFIG_FEATURE_STATS	"feature_stats"	/* not stored on disk */

include/sys/zfeature.h

@@ -35,7 +35,7 @@ extern "C" {
 #endif
 
 extern boolean_t feature_is_supported(objset_t *os, uint64_t obj,
-    uint64_t desc_obj, nvlist_t *unsup_feat);
+    uint64_t desc_obj, nvlist_t *unsup_feat, nvlist_t *enabled_feat);
 
 struct spa;
 extern void spa_feature_create_zap_objects(struct spa *, dmu_tx_t *);

lib/libzfs/libzfs_status.c

@@ -44,6 +44,7 @@
 #include <string.h>
 #include <unistd.h>
 #include "libzfs_impl.h"
+#include "zfeature_common.h"
 
 /*
  * Message ID table.  This must be kept in sync with the ZPOOL_STATUS_* defines
@@ -320,6 +321,30 @@ check_status(nvlist_t *config, boolean_t isimport)
 	if (SPA_VERSION_IS_SUPPORTED(version) && version != SPA_VERSION)
 		return (ZPOOL_STATUS_VERSION_OLDER);
 
+	/*
+	 * Usable pool with disabled features
+	 */
+	if (version >= SPA_VERSION_FEATURES) {
+		int i;
+		nvlist_t *feat;
+
+		if (isimport) {
+			feat = fnvlist_lookup_nvlist(config,
+			    ZPOOL_CONFIG_LOAD_INFO);
+			feat = fnvlist_lookup_nvlist(feat,
+			    ZPOOL_CONFIG_ENABLED_FEAT);
+		} else {
+			feat = fnvlist_lookup_nvlist(config,
+			    ZPOOL_CONFIG_FEATURE_STATS);
+		}
+
+		for (i = 0; i < SPA_FEATURES; i++) {
+			zfeature_info_t *fi = &spa_feature_table[i];
+			if (!nvlist_exists(feat, fi->fi_guid))
+				return (ZPOOL_STATUS_FEAT_DISABLED);
+		}
+	}
+
 	return (ZPOOL_STATUS_OK);
 }
 

man/man5/zpool-features.5

@@ -20,8 +20,9 @@ zpool\-features \- ZFS pool feature descriptions
 .LP
 ZFS pool on\-disk format versions are specified via "features" which replace
 the old on\-disk format numbers (the last supported on\-disk format number is
-28). To enable a feature on a pool use the \fBzpool\fR(1M) command to set
+28). To enable a feature on a pool use the \fBupgrade\fR subcommand of the
-the \fBfeature@\fR\fIfeature_name\fR property to \fBenabled\fR.
+\fBzpool\fR(1M) command, or set the \fBfeature@\fR\fIfeature_name\fR property
+to \fBenabled\fR.
 .sp
 .LP
 The pool format does not affect file system version compatibility or the ability

man/man8/zpool.8

@@ -1628,7 +1628,7 @@ Displays verbose data error information, printing out a complete list of all dat
 .ad
 .sp .6
 .RS 4n
-Displays all pools formatted using a different \fBZFS\fR on-disk version. Older versions can continue to be used, but some features may not be available. These pools can be upgraded using "\fBzpool upgrade -a\fR". Pools that are formatted with a more recent version are also displayed, although these pools will be inaccessible on the system.
+Displays pools which do not have all supported features enabled and pools formatted using a legacy ZFS version number. These pools can continue to be used, but some features may not be available. Use "\fBzpool upgrade -a\fR" to enable all features on all pools.
 .RE
 
 .sp
@@ -1639,7 +1639,7 @@ Displays all pools formatted using a different \fBZFS\fR on-disk version. Older
 .ad
 .sp .6
 .RS 4n
-Displays \fBZFS\fR versions supported by the current software. The current \fBZFS\fR versions and all previous supported versions are displayed, along with an explanation of the features provided with each version.
+Displays legacy \fBZFS\fR versions supported by the current software. See \fBzfs-features\fR(5) for a description of feature flags features supported by the current software.
 .RE
 
 .sp
@@ -1650,7 +1650,7 @@ Displays \fBZFS\fR versions supported by the current software. The current \fBZF
 .ad
 .sp .6
 .RS 4n
-Upgrades the given pool to the latest on-disk version. Once this is done, the pool will no longer be accessible on systems running older versions of the software.
+Enables all supported features on the given pool. Once this is done, the pool will no longer be accessible on systems that do not support feature flags. See \fBzfs-features\fR(5) for details on compatability with systems that support feature flags, but do not support all features enabled on the pool.
 .sp
 .ne 2
 .mk
@@ -1659,7 +1659,7 @@ Upgrades the given pool to the latest on-disk version. Once this is done, the po
 .ad
 .RS 14n
 .rt
-Upgrades all pools.
+Enables all supported features on all pools.
 .RE
 
 .sp
@@ -1670,7 +1670,7 @@ Upgrades all pools.
 .ad
 .RS 14n
 .rt
-Upgrade to the specified version. If the \fB-V\fR flag is not specified, the pool is upgraded to the most recent version. This option can only be used to increase the version number, and only up to the most recent version supported by this software.
+Upgrade to the specified legacy version. If the \fB-V\fR flag is specified, no features will be enabled on the pool. This option can only be used to increase the version number up to the last supported legacy version number.
 .RE
 
 .RE

module/zfs/spa.c

@@ -2209,7 +2209,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 
 	if (spa_version(spa) >= SPA_VERSION_FEATURES) {
 		boolean_t missing_feat_read = B_FALSE;
-		nvlist_t *unsup_feat;
+		nvlist_t *unsup_feat, *enabled_feat;
 
 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ,
 		    &spa->spa_feat_for_read_obj) != 0) {
@@ -2226,27 +2226,32 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
 		}
 
-		VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) ==
+		enabled_feat = fnvlist_alloc();
-		    0);
+		unsup_feat = fnvlist_alloc();
 
 		if (!feature_is_supported(spa->spa_meta_objset,
 		    spa->spa_feat_for_read_obj, spa->spa_feat_desc_obj,
-		    unsup_feat))
+		    unsup_feat, enabled_feat))
 			missing_feat_read = B_TRUE;
 
 		if (spa_writeable(spa) || state == SPA_LOAD_TRYIMPORT) {
 			if (!feature_is_supported(spa->spa_meta_objset,
 			    spa->spa_feat_for_write_obj, spa->spa_feat_desc_obj,
-			    unsup_feat))
+			    unsup_feat, enabled_feat)) {
 				missing_feat_write = B_TRUE;
+			}
 		}
 
+		fnvlist_add_nvlist(spa->spa_load_info,
+		    ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat);
+
 		if (!nvlist_empty(unsup_feat)) {
-			VERIFY(nvlist_add_nvlist(spa->spa_load_info,
+			fnvlist_add_nvlist(spa->spa_load_info,
-			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0);
+			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
 		}
 
-		nvlist_free(unsup_feat);
+		fnvlist_free(enabled_feat);
+		fnvlist_free(unsup_feat);
 
 		if (!missing_feat_read) {
 			fnvlist_add_boolean(spa->spa_load_info,

module/zfs/zfeature.c

@@ -173,7 +173,7 @@ typedef enum {
  */
 boolean_t
 feature_is_supported(objset_t *os, uint64_t obj, uint64_t desc_obj,
-    nvlist_t *unsup_feat)
+    nvlist_t *unsup_feat, nvlist_t *enabled_feat)
 {
 	boolean_t supported;
 	zap_cursor_t *zc;
@@ -191,11 +191,16 @@ feature_is_supported(objset_t *os, uint64_t obj, uint64_t desc_obj,
 		ASSERT(za->za_integer_length == sizeof (uint64_t) &&
 		    za->za_num_integers == 1);
 
+		if (NULL != enabled_feat) {
+			fnvlist_add_uint64(enabled_feat, za->za_name,
+			    za->za_first_integer);
+		}
+
 		if (za->za_first_integer != 0 &&
 		    !zfeature_is_supported(za->za_name)) {
 			supported = B_FALSE;
 
-			if (unsup_feat != NULL) {
+			if (NULL != unsup_feat) {
 				char *desc = "";
 
 				if (zap_lookup(os, desc_obj, za->za_name,