Illumos #1948: zpool list should show more detailed pool info

Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: Matt Ahrens <matt@delphix.com>
Reviewed by: Eric Schrock <eric.schrock@delphix.com>
Reviewed by: Richard Lowe <richlowe@richlowe.net>
Reviewed by: Albert Lee <trisk@nexenta.com>
Reviewed by: Dan McDonald <danmcd@nexenta.com>
Reviewed by: Garrett D'Amore <garrett@damore.org>
Approved by: Eric Schrock <eric.schrock@delphix.com>

References:
  https://www.illumos.org/issues/1948

Ported by:	Martin Matuska <martin@matuska.org>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #685

cmd/zpool/zpool_main.c

@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  */
 
@@ -43,6 +43,7 @@
 #include <priv.h>
 #include <pwd.h>
 #include <zone.h>
+#include <zfs_prop.h>
 #include <sys/fs/zfs.h>
 #include <sys/stat.h>
 #include <sys/fm/util.h>
@@ -68,6 +69,7 @@ static int zpool_do_status(int, char **);
 static int zpool_do_online(int, char **);
 static int zpool_do_offline(int, char **);
 static int zpool_do_clear(int, char **);
+static int zpool_do_reopen(int, char **);
 
 static int zpool_do_reguid(int, char **);
 
@@ -131,7 +133,8 @@ typedef enum {
 	HELP_GET,
 	HELP_SET,
 	HELP_SPLIT,
-	HELP_REGUID
+	HELP_REGUID,
+	HELP_REOPEN
 } zpool_help_t;
 
 
@@ -164,6 +167,7 @@ static zpool_command_t command_table[] = {
 	{ "online",	zpool_do_online,	HELP_ONLINE		},
 	{ "offline",	zpool_do_offline,	HELP_OFFLINE		},
 	{ "clear",	zpool_do_clear,		HELP_CLEAR		},
+	{ "reopen",	zpool_do_reopen,	HELP_REOPEN		},
 	{ NULL },
 	{ "attach",	zpool_do_attach,	HELP_ATTACH		},
 	{ "detach",	zpool_do_detach,	HELP_DETACH		},
@@ -238,6 +242,8 @@ get_usage(zpool_help_t idx) {
 		    "[new-device]\n"));
 	case HELP_REMOVE:
 		return (gettext("\tremove <pool> <device> ...\n"));
+	case HELP_REOPEN:
+		return (""); /* Undocumented command */
 	case HELP_SCRUB:
 		return (gettext("\tscrub [-s] <pool> ...\n"));
 	case HELP_STATUS:
@@ -2018,10 +2024,10 @@ error:
 }
 
 typedef struct iostat_cbdata {
-	zpool_list_t *cb_list;
+	boolean_t cb_verbose;
-	int cb_verbose;
-	int cb_iteration;
 	int cb_namewidth;
+	int cb_iteration;
+	zpool_list_t *cb_list;
 } iostat_cbdata_t;
 
 static void
@@ -2518,8 +2524,9 @@ zpool_do_iostat(int argc, char **argv)
 }
 
 typedef struct list_cbdata {
+	boolean_t	cb_verbose;
+	int		cb_namewidth;
 	boolean_t	cb_scripted;
-	boolean_t	cb_first;
 	zprop_list_t	*cb_proplist;
 } list_cbdata_t;
 
@@ -2527,16 +2534,27 @@ typedef struct list_cbdata {
  * Given a list of columns to display, output appropriate headers for each one.
  */
 static void
-print_header(zprop_list_t *pl)
+print_header(list_cbdata_t *cb)
 {
+	zprop_list_t *pl = cb->cb_proplist;
 	const char *header;
 	boolean_t first = B_TRUE;
 	boolean_t right_justify;
+	size_t width = 0;
 
 	for (; pl != NULL; pl = pl->pl_next) {
 		if (pl->pl_prop == ZPROP_INVAL)
 			continue;
 
+		width = pl->pl_width;
+		if (first && cb->cb_verbose) {
+			/*
+			 * Reset the width to accommodate the verbose listing
+			 * of devices.
+			 */
+			width = cb->cb_namewidth;
+		}
+
 		if (!first)
 			(void) printf("  ");
 		else
@@ -2548,9 +2566,9 @@ print_header(zprop_list_t *pl)
 		if (pl->pl_next == NULL && !right_justify)
 			(void) printf("%s", header);
 		else if (right_justify)
-			(void) printf("%*s", (int)pl->pl_width, header);
+			(void) printf("%*s", (int)width, header);
 		else
-			(void) printf("%-*s", (int)pl->pl_width, header);
+			(void) printf("%-*s", (int)width, header);
 	}
 
 	(void) printf("\n");
@@ -2561,17 +2579,28 @@ print_header(zprop_list_t *pl)
  * to the described layout.
  */
 static void
-print_pool(zpool_handle_t *zhp, zprop_list_t *pl, int scripted)
+print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
 {
+	zprop_list_t *pl = cb->cb_proplist;
 	boolean_t first = B_TRUE;
 	char property[ZPOOL_MAXPROPLEN];
 	char *propstr;
 	boolean_t right_justify;
-	int width;
+	size_t width;
 
 	for (; pl != NULL; pl = pl->pl_next) {
+
+		width = pl->pl_width;
+		if (first && cb->cb_verbose) {
+			/*
+			 * Reset the width to accommodate the verbose listing
+			 * of devices.
+			 */
+			width = cb->cb_namewidth;
+		}
+
 		if (!first) {
-			if (scripted)
+			if (cb->cb_scripted)
 				(void) printf("\t");
 			else
 				(void) printf("  ");
@@ -2581,7 +2610,10 @@ print_pool(zpool_handle_t *zhp, zprop_list_t *pl, int scripted)
 
 		right_justify = B_FALSE;
 		if (pl->pl_prop != ZPROP_INVAL) {
-			if (zpool_get_prop(zhp, pl->pl_prop, property,
+			if (pl->pl_prop == ZPOOL_PROP_EXPANDSZ &&
+			    zpool_get_prop_int(zhp, pl->pl_prop, NULL) == 0)
+				propstr = "-";
+			else if (zpool_get_prop(zhp, pl->pl_prop, property,
 			    sizeof (property), NULL) != 0)
 				propstr = "-";
 			else
@@ -2592,24 +2624,118 @@ print_pool(zpool_handle_t *zhp, zprop_list_t *pl, int scripted)
 			propstr = "-";
 		}
 
-		width = pl->pl_width;
 
 		/*
 		 * If this is being called in scripted mode, or if this is the
 		 * last column and it is left-justified, don't include a width
 		 * format specifier.
 		 */
-		if (scripted || (pl->pl_next == NULL && !right_justify))
+		if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
 			(void) printf("%s", propstr);
 		else if (right_justify)
-			(void) printf("%*s", width, propstr);
+			(void) printf("%*s", (int)width, propstr);
 		else
-			(void) printf("%-*s", width, propstr);
+			(void) printf("%-*s", (int)width, propstr);
 	}
 
 	(void) printf("\n");
 }
 
+static void
+print_one_column(zpool_prop_t prop, uint64_t value, boolean_t scripted)
+{
+	char propval[64];
+	boolean_t fixed;
+	size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
+
+	zfs_nicenum(value, propval, sizeof (propval));
+
+	if (prop == ZPOOL_PROP_EXPANDSZ && value == 0)
+		(void) strlcpy(propval, "-", sizeof (propval));
+
+	if (scripted)
+		(void) printf("\t%s", propval);
+	else
+		(void) printf("  %*s", (int)width, propval);
+}
+
+void
+print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
+    list_cbdata_t *cb, int depth)
+{
+	nvlist_t **child;
+	vdev_stat_t *vs;
+	uint_t c, children;
+	char *vname;
+	boolean_t scripted = cb->cb_scripted;
+
+	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
+	    (uint64_t **)&vs, &c) == 0);
+
+	if (name != NULL) {
+		if (scripted)
+			(void) printf("\t%s", name);
+		else if (strlen(name) + depth > cb->cb_namewidth)
+			(void) printf("%*s%s", depth, "", name);
+		else
+			(void) printf("%*s%s%*s", depth, "", name,
+			    (int)(cb->cb_namewidth - strlen(name) - depth), "");
+
+		/* only toplevel vdevs have capacity stats */
+		if (vs->vs_space == 0) {
+			if (scripted)
+				(void) printf("\t-\t-\t-");
+			else
+				(void) printf("      -      -      -");
+		} else {
+			print_one_column(ZPOOL_PROP_SIZE, vs->vs_space,
+			    scripted);
+			print_one_column(ZPOOL_PROP_CAPACITY, vs->vs_alloc,
+			    scripted);
+			print_one_column(ZPOOL_PROP_FREE,
+			    vs->vs_space - vs->vs_alloc, scripted);
+		}
+		print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize,
+		    scripted);
+		(void) printf("\n");
+	}
+
+	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
+	    &child, &children) != 0)
+		return;
+
+	for (c = 0; c < children; c++) {
+		uint64_t ishole = B_FALSE;
+
+		if (nvlist_lookup_uint64(child[c],
+		    ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
+			continue;
+
+		vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
+		print_list_stats(zhp, vname, child[c], cb, depth + 2);
+		free(vname);
+	}
+
+	/*
+	 * Include level 2 ARC devices in iostat output
+	 */
+	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
+	    &child, &children) != 0)
+		return;
+
+	if (children > 0) {
+		(void) printf("%-*s      -      -      -      -      -      "
+		    "-\n", cb->cb_namewidth, "cache");
+		for (c = 0; c < children; c++) {
+			vname = zpool_vdev_name(g_zfs, zhp, child[c],
+			    B_FALSE);
+			print_list_stats(zhp, vname, child[c], cb, depth + 2);
+			free(vname);
+		}
+	}
+}
+
+
 /*
  * Generic callback function to list a pool.
  */
@@ -2617,14 +2743,18 @@ int
 list_callback(zpool_handle_t *zhp, void *data)
 {
 	list_cbdata_t *cbp = data;
+	nvlist_t *config;
+	nvlist_t *nvroot;
 
-	if (cbp->cb_first) {
+	config = zpool_get_config(zhp, NULL);
-		if (!cbp->cb_scripted)
-			print_header(cbp->cb_proplist);
-		cbp->cb_first = B_FALSE;
-	}
 
-	print_pool(zhp, cbp->cb_proplist, cbp->cb_scripted);
+	print_pool(zhp, cbp);
+	if (!cbp->cb_verbose)
+		return (0);
+
+	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
+	    &nvroot) == 0);
+	print_list_stats(zhp, NULL, nvroot, cbp, 0);
 
 	return (0);
 }
@@ -2648,12 +2778,15 @@ zpool_do_list(int argc, char **argv)
 	int ret;
 	list_cbdata_t cb = { 0 };
 	static char default_props[] =
-	    "name,size,allocated,free,capacity,dedupratio,health,altroot";
+	    "name,size,allocated,free,capacity,dedupratio,"
+	    "health,altroot";
 	char *props = default_props;
 	unsigned long interval = 0, count = 0;
+	zpool_list_t *list;
+	boolean_t first = B_TRUE;
 
 	/* check options */
-	while ((c = getopt(argc, argv, ":Ho:T:")) != -1) {
+	while ((c = getopt(argc, argv, ":Ho:T:v")) != -1) {
 		switch (c) {
 		case 'H':
 			cb.cb_scripted = B_TRUE;
@@ -2664,6 +2797,9 @@ zpool_do_list(int argc, char **argv)
 		case 'T':
 			get_timestamp_arg(*optarg);
 			break;
+		case 'v':
+			cb.cb_verbose = B_TRUE;
+			break;
 		case ':':
 			(void) fprintf(stderr, gettext("missing argument for "
 			    "'%c' option\n"), optopt);
@@ -2684,23 +2820,29 @@ zpool_do_list(int argc, char **argv)
 	if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
 		usage(B_FALSE);
 
-	cb.cb_first = B_TRUE;
+	if ((list = pool_list_get(argc, argv, &cb.cb_proplist, &ret)) == NULL)
+		return (1);
+
+	if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
+		(void) printf(gettext("no pools available\n"));
+		zprop_free_list(cb.cb_proplist);
+		return (0);
+	}
 
 	for (;;) {
+		pool_list_update(list);
+
+		if (pool_list_count(list) == 0)
+			break;
 
 		if (timestamp_fmt != NODATE)
 			print_timestamp(timestamp_fmt);
 
-		ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist,
+		if (!cb.cb_scripted && (first || cb.cb_verbose)) {
-		    list_callback, &cb);
+			print_header(&cb);
-
+			first = B_FALSE;
-		if (argc == 0 && cb.cb_first)
-			(void) fprintf(stderr, gettext("no pools available\n"));
-		else if (argc && cb.cb_first) {
-			/* cannot open the given pool */
-			zprop_free_list(cb.cb_proplist);
-			return (1);
 		}
+		ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
 
 		if (interval == 0)
 			break;
@@ -3293,6 +3435,36 @@ zpool_do_reguid(int argc, char **argv)
 }
 
 
+/*
+ * zpool reopen <pool>
+ *
+ * Reopen the pool so that the kernel can update the sizes of all vdevs.
+ *
+ * NOTE: This command is currently undocumented.  If the command is ever
+ * exposed then the appropriate usage() messages will need to be made.
+ */
+int
+zpool_do_reopen(int argc, char **argv)
+{
+	int ret = 0;
+	zpool_handle_t *zhp;
+	char *pool;
+
+	argc--;
+	argv++;
+
+	if (argc != 1)
+		return (2);
+
+	pool = argv[0];
+	if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL)
+		return (1);
+
+	ret = zpool_reopen(zhp);
+	zpool_close(zhp);
+	return (ret);
+}
+
 typedef struct scrub_cbdata {
 	int	cb_type;
 	int	cb_argc;

include/libzfs.h

@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #ifndef	_LIBZFS_H
@@ -252,6 +252,7 @@ typedef struct splitflags {
 extern int zpool_scan(zpool_handle_t *, pool_scan_func_t);
 extern int zpool_clear(zpool_handle_t *, const char *, nvlist_t *);
 extern int zpool_reguid(zpool_handle_t *);
+extern int zpool_reopen(zpool_handle_t *);
 
 extern int zpool_vdev_online(zpool_handle_t *, const char *, int,
     vdev_state_t *);

include/sys/fs/zfs.h

@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  */
 
@@ -168,6 +168,7 @@ typedef enum {
 	ZPOOL_PROP_READONLY,
 	ZPOOL_PROP_ASHIFT,
 	ZPOOL_PROP_COMMENT,
+	ZPOOL_PROP_EXPANDSZ,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;
 
@@ -678,6 +679,7 @@ typedef struct vdev_stat {
 	uint64_t	vs_space;		/* total capacity	*/
 	uint64_t	vs_dspace;		/* deflated capacity	*/
 	uint64_t	vs_rsize;		/* replaceable dev size */
+	uint64_t	vs_esize;		/* expandable dev size */
 	uint64_t	vs_ops[ZIO_TYPES];	/* operation count	*/
 	uint64_t	vs_bytes[ZIO_TYPES];	/* bytes read/written	*/
 	uint64_t	vs_read_errors;		/* read errors		*/
@@ -800,6 +802,7 @@ typedef enum zfs_ioc {
 	ZFS_IOC_POOL_REGUID,
 	ZFS_IOC_SPACE_WRITTEN,
 	ZFS_IOC_SPACE_SNAPS,
+	ZFS_IOC_POOL_REOPEN,
 } zfs_ioc_t;
 
 /*

include/sys/vdev_impl.h

@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_VDEV_IMPL_H
@@ -56,7 +57,8 @@ typedef struct vdev_cache_entry vdev_cache_entry_t;
 /*
  * Virtual device operations
  */
-typedef int	vdev_open_func_t(vdev_t *vd, uint64_t *size, uint64_t *ashift);
+typedef int	vdev_open_func_t(vdev_t *vd, uint64_t *size, uint64_t *max_size,
+    uint64_t *ashift);
 typedef void	vdev_close_func_t(vdev_t *vd);
 typedef uint64_t vdev_asize_func_t(vdev_t *vd, uint64_t psize);
 typedef int	vdev_io_start_func_t(zio_t *zio);
@@ -125,6 +127,7 @@ struct vdev {
 	uint64_t	vdev_orig_guid;	/* orig. guid prior to remove	*/
 	uint64_t	vdev_asize;	/* allocatable device capacity	*/
 	uint64_t	vdev_min_asize;	/* min acceptable asize		*/
+	uint64_t	vdev_max_asize;	/* max acceptable asize		*/
 	uint64_t	vdev_ashift;	/* block alignment shift	*/
 	uint64_t	vdev_state;	/* see VDEV_STATE_* #defines	*/
 	uint64_t	vdev_prevstate;	/* used when reopening a vdev	*/

lib/libzfs/libzfs_pool.c

@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #include <ctype.h>
@@ -273,6 +273,7 @@ zpool_get_prop(zpool_handle_t *zhp, zpool_prop_t prop, char *buf, size_t len,
 		case ZPOOL_PROP_SIZE:
 		case ZPOOL_PROP_ALLOCATED:
 		case ZPOOL_PROP_FREE:
+		case ZPOOL_PROP_EXPANDSZ:
 		case ZPOOL_PROP_ASHIFT:
 			(void) zfs_nicenum(intval, buf, len);
 			break;
@@ -361,8 +362,8 @@ pool_uses_efi(nvlist_t *config)
 	return (B_FALSE);
 }
 
-static boolean_t
+boolean_t
-pool_is_bootable(zpool_handle_t *zhp)
+zpool_is_bootable(zpool_handle_t *zhp)
 {
 	char bootfs[ZPOOL_MAXNAMELEN];
 
@@ -1127,7 +1128,7 @@ zpool_add(zpool_handle_t *zhp, nvlist_t *nvroot)
 		return (zfs_error(hdl, EZFS_BADVERSION, msg));
 	}
 
-	if (pool_is_bootable(zhp) && nvlist_lookup_nvlist_array(nvroot,
+	if (zpool_is_bootable(zhp) && nvlist_lookup_nvlist_array(nvroot,
 	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0) {
 		uint64_t s;
 
@@ -2374,7 +2375,7 @@ zpool_vdev_attach(zpool_handle_t *zhp,
 	uint_t children;
 	nvlist_t *config_root;
 	libzfs_handle_t *hdl = zhp->zpool_hdl;
-	boolean_t rootpool = pool_is_bootable(zhp);
+	boolean_t rootpool = zpool_is_bootable(zhp);
 
 	if (replacing)
 		(void) snprintf(msg, sizeof (msg), dgettext(TEXT_DOMAIN,
@@ -3005,6 +3006,26 @@ zpool_reguid(zpool_handle_t *zhp)
 	return (zpool_standard_error(hdl, errno, msg));
 }
 
+/*
+ * Reopen the pool.
+ */
+int
+zpool_reopen(zpool_handle_t *zhp)
+{
+	zfs_cmd_t zc = { "\0", "\0", "\0", "\0", 0 };
+	char msg[1024];
+	libzfs_handle_t *hdl = zhp->zpool_hdl;
+
+	(void) snprintf(msg, sizeof (msg),
+	    dgettext(TEXT_DOMAIN, "cannot reopen '%s'"),
+	    zhp->zpool_name);
+
+	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
+	if (zfs_ioctl(hdl, ZFS_IOC_POOL_REOPEN, &zc) == 0)
+		return (0);
+	return (zpool_standard_error(hdl, errno, msg));
+}
+
 /*
  * Convert from a devid string to a path.
  */
@@ -3798,7 +3819,7 @@ zpool_label_disk(libzfs_handle_t *hdl, zpool_handle_t *zhp, char *name)
 	if (zhp) {
 		nvlist_t *nvroot;
 
-		if (pool_is_bootable(zhp)) {
+		if (zpool_is_bootable(zhp)) {
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 			    "EFI labeled devices are not supported on root "
 			    "pools."));

man/man8/zpool.8

@@ -1,6 +1,7 @@
 '\" te
 .\" Copyright (c) 2007, Sun Microsystems, Inc. All Rights Reserved.
 .\" Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2012 by Delphix. All Rights Reserved.
 .\" The contents of this file are subject to the terms of the Common Development and Distribution License (the "License"). You may not use this file except in compliance with the License. You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing.
 .\" See the License for the specific language governing permissions and limitations under the License. When distributing Covered Code, include this CDDL HEADER in each file and include the License file at usr/src/OPENSOLARIS.LICENSE. If applicable, add the following below this CDDL HEADER, with the
 .\" fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner]
@@ -83,7 +84,7 @@ zpool \- configures ZFS storage pools
 
 .LP
 .nf
-\fBzpool list\fR [\fB-H\fR] [\fB-o\fR \fIproperty\fR[,...]] [\fIpool\fR] ...
+\fBzpool list\fR [\fB-Hv\fR] [\fB-o\fR \fIproperty\fR[,...]] [\fIpool\fR] ...
 .fi
 
 .LP
@@ -476,6 +477,18 @@ A text string consisting of printable ASCII characters that will be stored such
 .ne 2
 .mk
 .na
+\fB\fBexpandsize\fR\fR
+.ad
+.RS 20n
+Amount of uninitialized space within the pool or device that can be used to
+increase the total capacity of the pool.  Uninitialized space consists of 
+any space on an EFI labeled vdev which has not been brought online 
+(i.e. zpool online -e).  This space occurs when a LUN is dynamically expanded.
+.RE
+
+.sp
+.ne 2
+.na
 \fB\fBhealth\fR\fR
 .ad
 .RS 20n
@@ -1312,7 +1325,7 @@ Verbose statistics. Reports usage statistics for individual \fIvdevs\fR within t
 .ne 2
 .mk
 .na
-\fB\fBzpool list\fR [\fB-H\fR] [\fB-o\fR \fIprops\fR[,...]] [\fIpool\fR] ...\fR
+\fB\fBzpool list\fR [\fB-Hv\fR] [\fB-o\fR \fIprops\fR[,...]] [\fIpool\fR] ...\fR
 .ad
 .sp .6
 .RS 4n
@@ -1336,7 +1349,18 @@ Scripted mode. Do not display headers, and separate fields by a single tab inste
 .ad
 .RS 12n
 .rt  
-Comma-separated list of properties to display. See the "Properties" section for a list of valid properties. The default list is "name, size, used, available, capacity, health, altroot"
+Comma-separated list of properties to display. See the "Properties" section for a list of valid properties. The default list is "name, size, used, available, expandsize, capacity, dedupratio, health, altroot"
+.RE
+
+.sp
+.ne 2
+.mk
+.na
+\fB\fB-v\fR\fR
+.ad
+.RS 12n
+.rt
+Verbose statistics. Reports usage statistics for individual \fIvdevs\fR within the pool, in addition to the pool-wise statistics.
 .RE
 
 .RE
@@ -1649,10 +1673,10 @@ The results from this command are similar to the following:
 .in +2
 .nf
 # \fBzpool list\fR
-     NAME              SIZE    USED   AVAIL    CAP  HEALTH     ALTROOT
+     NAME    SIZE  ALLOC   FREE  EXPANDSZ    CAP  DEDUP  HEALTH  ALTROOT
-     pool             67.5G   2.92M   67.5G     0%  ONLINE     -
+     rpool  19.9G  8.43G  11.4G         -    42%  1.00x  ONLINE  -
-     tank             67.5G   2.92M   67.5G     0%  ONLINE     -
+     tank   61.5G  20.0G  41.5G         -    32%  1.00x  ONLINE  -
-     zion                 -       -       -     0%  FAULTED    -
+     zion       -      -      -         -      -      -  FAULTED -
 .fi
 .in -2
 .sp
@@ -1855,6 +1879,29 @@ The command to remove the mirrored log \fBmirror-2\fR is:
 .in -2
 .sp
 
+.LP
+\fBExample 15 \fRDisplaying expanded space on a device
+.sp
+.LP
+The following command dipslays the detailed information for the \fIdata\fR
+pool. This pool is comprised of a single \fIraidz\fR vdev where one of its
+devices increased its capacity by 1GB. In this example, the pool will not 
+be able to utilized this extra capacity until all the devices under the
+\fIraidz\fR vdev have been expanded.
+
+.sp
+.in +2
+.nf
+# \fBzpool list -v data\fR
+   NAME         SIZE  ALLOC   FREE  EXPANDSZ    CAP  DEDUP  HEALTH  ALTROOT
+   data        17.9G   174K  17.9G         -     0%  1.00x  ONLINE  -
+     raidz1    17.9G   174K  17.9G         -
+       c4t2d0      -      -      -        1G
+       c4t3d0      -      -      -         -
+       c4t4d0      -      -      -         - 
+.fi
+.in -2
+
 .SH EXIT STATUS
 .sp
 .LP

module/zcommon/zpool_prop.c

@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #include <sys/zio.h>
@@ -81,6 +81,8 @@ zpool_prop_init(void)
 	    ZFS_TYPE_POOL, "<size>", "FREE");
 	zprop_register_number(ZPOOL_PROP_ALLOCATED, "allocated", 0,
 	    PROP_READONLY, ZFS_TYPE_POOL, "<size>", "ALLOC");
+	zprop_register_number(ZPOOL_PROP_EXPANDSZ, "expandsize", 0,
+	    PROP_READONLY, ZFS_TYPE_POOL, "<size>", "EXPANDSZ");
 	zprop_register_number(ZPOOL_PROP_CAPACITY, "capacity", 0, PROP_READONLY,
 	    ZFS_TYPE_POOL, "<size>", "CAP");
 	zprop_register_number(ZPOOL_PROP_GUID, "guid", 0, PROP_READONLY,

module/zfs/spa.c

@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 /*
@@ -168,15 +168,18 @@ spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval,
 static void
 spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 {
+	vdev_t *rvd = spa->spa_root_vdev;
 	uint64_t size;
 	uint64_t alloc;
+	uint64_t space;
 	uint64_t cap, version;
 	zprop_source_t src = ZPROP_SRC_NONE;
 	spa_config_dirent_t *dp;
+	int c;
 
 	ASSERT(MUTEX_HELD(&spa->spa_props_lock));
 
-	if (spa->spa_root_vdev != NULL) {
+	if (rvd != NULL) {
 		alloc = metaslab_class_get_alloc(spa_normal_class(spa));
 		size = metaslab_class_get_space(spa_normal_class(spa));
 		spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src);
@@ -184,6 +187,15 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src);
 		spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL,
 		    size - alloc, src);
+
+		space = 0;
+		for (c = 0; c < rvd->vdev_children; c++) {
+			vdev_t *tvd = rvd->vdev_child[c];
+			space += tvd->vdev_max_asize - tvd->vdev_asize;
+		}
+		spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, space,
+		    src);
+
 		spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL,
 		    (spa_mode(spa) == FREAD), src);
 
@@ -194,7 +206,7 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 		    ddt_get_pool_dedup_ratio(spa), src);
 
 		spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL,
-		    spa->spa_root_vdev->vdev_state, src);
+		    rvd->vdev_state, src);
 
 		version = spa_version(spa);
 		if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION))

module/zfs/vdev.c

@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -109,7 +109,7 @@ vdev_get_min_asize(vdev_t *vd)
 	vdev_t *pvd = vd->vdev_parent;
 
 	/*
-	 * The our parent is NULL (inactive spare or cache) or is the root,
+	 * If our parent is NULL (inactive spare or cache) or is the root,
 	 * just return our own asize.
 	 */
 	if (pvd == NULL)
@@ -747,6 +747,7 @@ vdev_add_parent(vdev_t *cvd, vdev_ops_t *ops)
 
 	mvd->vdev_asize = cvd->vdev_asize;
 	mvd->vdev_min_asize = cvd->vdev_min_asize;
+	mvd->vdev_max_asize = cvd->vdev_max_asize;
 	mvd->vdev_ashift = cvd->vdev_ashift;
 	mvd->vdev_state = cvd->vdev_state;
 	mvd->vdev_crtxg = cvd->vdev_crtxg;
@@ -1132,7 +1133,8 @@ vdev_open(vdev_t *vd)
 	spa_t *spa = vd->vdev_spa;
 	int error;
 	uint64_t osize = 0;
-	uint64_t asize, psize;
+	uint64_t max_osize = 0;
+	uint64_t asize, max_asize, psize;
 	uint64_t ashift = 0;
 	int c;
 
@@ -1164,7 +1166,7 @@ vdev_open(vdev_t *vd)
 		return (ENXIO);
 	}
 
-	error = vd->vdev_ops->vdev_op_open(vd, &osize, &ashift);
+	error = vd->vdev_ops->vdev_op_open(vd, &osize, &max_osize, &ashift);
 
 	/*
 	 * Reset the vdev_reopening flag so that we actually close
@@ -1222,6 +1224,7 @@ vdev_open(vdev_t *vd)
 	}
 
 	osize = P2ALIGN(osize, (uint64_t)sizeof (vdev_label_t));
+	max_osize = P2ALIGN(max_osize, (uint64_t)sizeof (vdev_label_t));
 
 	if (vd->vdev_children == 0) {
 		if (osize < SPA_MINDEVSIZE) {
@@ -1231,6 +1234,8 @@ vdev_open(vdev_t *vd)
 		}
 		psize = osize;
 		asize = osize - (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE);
+		max_asize = max_osize - (VDEV_LABEL_START_SIZE +
+		    VDEV_LABEL_END_SIZE);
 	} else {
 		if (vd->vdev_parent != NULL && osize < SPA_MINDEVSIZE -
 		    (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE)) {
@@ -1240,6 +1245,7 @@ vdev_open(vdev_t *vd)
 		}
 		psize = 0;
 		asize = osize;
+		max_asize = max_osize;
 	}
 
 	vd->vdev_psize = psize;
@@ -1259,6 +1265,7 @@ vdev_open(vdev_t *vd)
 		 * For testing purposes, a higher ashift can be requested.
 		 */
 		vd->vdev_asize = asize;
+		vd->vdev_max_asize = max_asize;
 		vd->vdev_ashift = MAX(ashift, vd->vdev_ashift);
 	} else {
 		/*
@@ -1269,6 +1276,7 @@ vdev_open(vdev_t *vd)
 			    VDEV_AUX_BAD_LABEL);
 			return (EINVAL);
 		}
+		vd->vdev_max_asize = max_asize;
 	}
 
 	/*
@@ -2499,6 +2507,7 @@ vdev_get_stats(vdev_t *vd, vdev_stat_t *vs)
 	vs->vs_rsize = vdev_get_min_asize(vd);
 	if (vd->vdev_ops->vdev_op_leaf)
 		vs->vs_rsize += VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
+	vs->vs_esize = vd->vdev_max_asize - vd->vdev_asize;
 	mutex_exit(&vd->vdev_stat_lock);
 
 	/*

module/zfs/vdev_disk.c

@@ -224,7 +224,8 @@ vdev_disk_rrpart(const char *path, int mode, vdev_disk_t *vd)
 }
 
 static int
-vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *ashift)
+vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
+    uint64_t *ashift)
 {
 	struct block_device *bdev = ERR_PTR(-ENXIO);
 	vdev_disk_t *vd;
@@ -288,6 +289,9 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *ashift)
 	/* Physical volume size in bytes */
 	*psize = bdev_capacity(bdev);
 
+	/* TODO: report possible expansion size */
+	*max_psize = *psize;
+
 	/* Based on the minimum sector size set the block size */
 	*ashift = highbit(MAX(block_size, SPA_MINBLOCKSIZE)) - 1;
 

module/zfs/vdev_file.c

@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -47,7 +48,8 @@ vdev_file_rele(vdev_t *vd)
 }
 
 static int
-vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *ashift)
+vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize,
+    uint64_t *ashift)
 {
 	vdev_file_t *vf;
 	vnode_t *vp;
@@ -112,7 +114,7 @@ skip_open:
 		return (error);
 	}
 
-	*psize = vattr.va_size;
+	*max_psize = *psize = vattr.va_size;
 	*ashift = SPA_MINBLOCKSHIFT;
 
 	return (0);

module/zfs/vdev_mirror.c

@@ -23,6 +23,10 @@
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright (c) 2012 by Delphix. All rights reserved.
+ */
+
 #include <sys/zfs_context.h>
 #include <sys/spa.h>
 #include <sys/vdev_impl.h>
@@ -127,7 +131,8 @@ vdev_mirror_map_alloc(zio_t *zio)
 }
 
 static int
-vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *ashift)
+vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
+    uint64_t *ashift)
 {
 	int numerrors = 0;
 	int lasterror = 0;
@@ -150,6 +155,7 @@ vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *ashift)
 		}
 
 		*asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1;
+		*max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1;
 		*ashift = MAX(*ashift, cvd->vdev_ashift);
 	}
 

module/zfs/vdev_missing.c

@@ -23,6 +23,10 @@
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright (c) 2012 by Delphix. All rights reserved.
+ */
+
 /*
  * The 'missing' vdev is a special vdev type used only during import.  It
  * signifies a placeholder in the root vdev for some vdev that we know is
@@ -40,7 +44,8 @@
 
 /* ARGSUSED */
 static int
-vdev_missing_open(vdev_t *vd, uint64_t *psize, uint64_t *ashift)
+vdev_missing_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize,
+    uint64_t *ashift)
 {
 	/*
 	 * Really this should just fail.  But then the root vdev will be in the
@@ -49,6 +54,7 @@ vdev_missing_open(vdev_t *vd, uint64_t *psize, uint64_t *ashift)
 	 * will fail the GUID sum check before ever trying to open the pool.
 	 */
 	*psize = 0;
+	*max_psize = 0;
 	*ashift = 0;
 	return (0);
 }

module/zfs/vdev_raidz.c

@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -1441,7 +1442,8 @@ vdev_raidz_reconstruct(raidz_map_t *rm, int *t, int nt)
 }
 
 static int
-vdev_raidz_open(vdev_t *vd, uint64_t *asize, uint64_t *ashift)
+vdev_raidz_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
+    uint64_t *ashift)
 {
 	vdev_t *cvd;
 	uint64_t nparity = vd->vdev_nparity;
@@ -1469,10 +1471,12 @@ vdev_raidz_open(vdev_t *vd, uint64_t *asize, uint64_t *ashift)
 		}
 
 		*asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1;
+		*max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1;
 		*ashift = MAX(*ashift, cvd->vdev_ashift);
 	}
 
 	*asize *= vd->vdev_children;
+	*max_asize *= vd->vdev_children;
 
 	if (numerrors > nparity) {
 		vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;

module/zfs/vdev_root.c

@@ -23,6 +23,10 @@
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright (c) 2012 by Delphix. All rights reserved.
+ */
+
 #include <sys/zfs_context.h>
 #include <sys/spa.h>
 #include <sys/vdev_impl.h>
@@ -50,7 +54,8 @@ too_many_errors(vdev_t *vd, int numerrors)
 }
 
 static int
-vdev_root_open(vdev_t *vd, uint64_t *asize, uint64_t *ashift)
+vdev_root_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
+    uint64_t *ashift)
 {
 	int lasterror = 0;
 	int numerrors = 0;
@@ -78,6 +83,7 @@ vdev_root_open(vdev_t *vd, uint64_t *asize, uint64_t *ashift)
 	}
 
 	*asize = 0;
+	*max_asize = 0;
 	*ashift = 0;
 
 	return (0);

module/zfs/zfs_ioctl.c

@@ -24,7 +24,7 @@
  * Portions Copyright 2012 Pawel Jakub Dawidek <pawel@dawidek.net>
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2011 by Delphix. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -4037,6 +4037,22 @@ zfs_ioc_clear(zfs_cmd_t *zc)
 	return (error);
 }
 
+static int
+zfs_ioc_pool_reopen(zfs_cmd_t *zc)
+{
+	spa_t *spa;
+	int error;
+
+	error = spa_open(zc->zc_name, &spa, FTAG);
+	if (error)
+		return (error);
+
+	spa_vdev_state_enter(spa, SCL_NONE);
+	vdev_reopen(spa->spa_root_vdev);
+	(void) spa_vdev_state_exit(spa, NULL, 0);
+	spa_close(spa, FTAG);
+	return (0);
+}
 /*
  * inputs:
  * zc_name	name of filesystem
@@ -4831,6 +4847,8 @@ static zfs_ioc_vec_t zfs_ioc_vec[] = {
 	    POOL_CHECK_SUSPENDED },
 	{ zfs_ioc_space_snaps, zfs_secpolicy_read, DATASET_NAME, B_FALSE,
 	    POOL_CHECK_SUSPENDED },
+	{ zfs_ioc_pool_reopen, zfs_secpolicy_config, POOL_NAME, B_TRUE,
+	    POOL_CHECK_SUSPENDED },
 };
 
 int