zpool iostat: refresh pool list every interval

When running zpool iostat in interval mode, it would not notice any new
pools created or imported, and would forget any destroyed or exported,
so would not notice if they came back. This leads to outputting "no
pools available" every interval until killed.

It looks like this was at least intended to work; the comment above
zpool_do_iostat() indicates that it is expected to "deal with pool
creation/destruction" and that pool_list_update() would detect new
pools. That call however was removed in 3e43edd2c5, though its unclear
if that broke this behaviour and it wasn't noticed, or if it never
worked, or if something later broke it. That said, the lack of
pool_list_update() is only part of the reason it doesn't work properly.

The fundamental problem is that the various things involved in
refreshing or updating the list of pools would aggressively ignore,
remove, skip or fail on pools that stop existing, or that already exist.
Mostly this meant that once a pool is removed from the list, it will
never be seen again. Restoring pool_list_update() to the
zpool_do_iostat() loop only partially fixes this - it would find "new"
pools again, but only in the "all pools" (no args) mode, and because its
iterator callback add_pool() would abort the iterator if it already has
a pool listed, it would only add pools if there weren't any already.

So, this commit reworks the structure somewhat. pool_list_update()
becomes pool_list_refresh(), and will ensure the state of all pools in
the list are updated. In the "all pools" mode, it will also add new
pools and remove pools that disappear, but when a fixed list of pools is
used, the list doesn't change, only the state of the pools within it.

The rest of the commit is adjusting things for this much simpler
structure. Regardless of the mode in use, pool_list_refresh() will
always do the right thing, so the driver code can just get on with the
display.

Now that pools can appear and disappear, I've made it so the header (if
enabled) is re-printed when the list changes, so that its easier to see
what's happening if the column widths change.

Since this is all rather complicated, I've included tests for the "all
pools" and "set of pools" modes.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Igor Kozhukhov <igor@dilos.org>
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
Closes #17786

cmd/zpool/zpool_iter.c

@@ -26,6 +26,7 @@
 
 /*
  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
+ * Copyright (c) 2025, Klara, Inc.
  */
 
 #include <libintl.h>
@@ -52,7 +53,7 @@
 typedef struct zpool_node {
 	zpool_handle_t	*zn_handle;
 	uu_avl_node_t	zn_avlnode;
-	int		zn_mark;
+	hrtime_t	zn_last_refresh;
 } zpool_node_t;
 
 struct zpool_list {
@@ -62,6 +63,7 @@ struct zpool_list {
 	uu_avl_pool_t	*zl_pool;
 	zprop_list_t	**zl_proplist;
 	zfs_type_t	zl_type;
+	hrtime_t	zl_last_refresh;
 };
 
 static int
@@ -81,32 +83,47 @@ zpool_compare(const void *larg, const void *rarg, void *unused)
  * of known pools.
  */
 static int
-add_pool(zpool_handle_t *zhp, void *data)
+add_pool(zpool_handle_t *zhp, zpool_list_t *zlp)
 {
-	zpool_list_t *zlp = data;
-	zpool_node_t *node = safe_malloc(sizeof (zpool_node_t));
+	zpool_node_t *node, *new = safe_malloc(sizeof (zpool_node_t));
 	uu_avl_index_t idx;
 
-	node->zn_handle = zhp;
-	uu_avl_node_init(node, &node->zn_avlnode, zlp->zl_pool);
-	if (uu_avl_find(zlp->zl_avl, node, NULL, &idx) == NULL) {
+	new->zn_handle = zhp;
+	uu_avl_node_init(new, &new->zn_avlnode, zlp->zl_pool);
+
+	node = uu_avl_find(zlp->zl_avl, new, NULL, &idx);
+	if (node == NULL) {
 		if (zlp->zl_proplist &&
 		    zpool_expand_proplist(zhp, zlp->zl_proplist,
 		    zlp->zl_type, zlp->zl_literal) != 0) {
 			zpool_close(zhp);
-			free(node);
+			free(new);
 			return (-1);
 		}
-		uu_avl_insert(zlp->zl_avl, node, idx);
+		new->zn_last_refresh = zlp->zl_last_refresh;
+		uu_avl_insert(zlp->zl_avl, new, idx);
 	} else {
+		node->zn_last_refresh = zlp->zl_last_refresh;
 		zpool_close(zhp);
-		free(node);
+		free(new);
 		return (-1);
 	}
 
 	return (0);
 }
 
+/*
+ * add_pool(), but always returns 0. This allows zpool_iter() to continue
+ * even if a pool exists in the tree, or we fail to get the properties for
+ * a new one.
+ */
+static int
+add_pool_cb(zpool_handle_t *zhp, void *data)
+{
+	(void) add_pool(zhp, data);
+	return (0);
+}
+
 /*
  * Create a list of pools based on the given arguments.  If we're given no
  * arguments, then iterate over all pools in the system and add them to the AVL
@@ -135,9 +152,10 @@ pool_list_get(int argc, char **argv, zprop_list_t **proplist, zfs_type_t type,
 	zlp->zl_type = type;
 
 	zlp->zl_literal = literal;
+	zlp->zl_last_refresh = gethrtime();
 
 	if (argc == 0) {
-		(void) zpool_iter(g_zfs, add_pool, zlp);
+		(void) zpool_iter(g_zfs, add_pool_cb, zlp);
 		zlp->zl_findall = B_TRUE;
 	} else {
 		int i;
@@ -159,15 +177,69 @@ pool_list_get(int argc, char **argv, zprop_list_t **proplist, zfs_type_t type,
 }
 
 /*
- * Search for any new pools, adding them to the list.  We only add pools when no
- * options were given on the command line.  Otherwise, we keep the list fixed as
- * those that were explicitly specified.
+ * Refresh the state of all pools on the list. Additionally, if no options were
+ * given on the command line, add any new pools and remove any that are no
+ * longer available.
  */
-void
-pool_list_update(zpool_list_t *zlp)
+int
+pool_list_refresh(zpool_list_t *zlp)
 {
-	if (zlp->zl_findall)
-		(void) zpool_iter(g_zfs, add_pool, zlp);
+	zlp->zl_last_refresh = gethrtime();
+
+	if (!zlp->zl_findall) {
+		/*
+		 * This list is a fixed list of pools, so we must not add
+		 * or remove any. Just walk over them and refresh their
+		 * state.
+		 */
+		int navail = 0;
+		for (zpool_node_t *node = uu_avl_first(zlp->zl_avl);
+		    node != NULL; node = uu_avl_next(zlp->zl_avl, node)) {
+			boolean_t missing;
+			zpool_refresh_stats(node->zn_handle, &missing);
+			navail += !missing;
+			node->zn_last_refresh = zlp->zl_last_refresh;
+		}
+		return (navail);
+	}
+
+	/*
+	 * Search for any new pools and add them to the list. zpool_iter()
+	 * will call zpool_refresh_stats() as part of its work, so this has
+	 * the side effect of updating all active handles.
+	 */
+	(void) zpool_iter(g_zfs, add_pool_cb, zlp);
+
+	/*
+	 * Walk the list for any that weren't refreshed, and update and remove
+	 * them. It's not enough to just skip available ones, as zpool_iter()
+	 * won't update them, so they'll still appear active in our list.
+	 */
+	zpool_node_t *node, *next;
+	for (node = uu_avl_first(zlp->zl_avl); node != NULL; node = next) {
+		next = uu_avl_next(zlp->zl_avl, node);
+
+		/*
+		 * Skip any that were refreshed and are online; they're already
+		 * handled.
+		 */
+		if (node->zn_last_refresh == zlp->zl_last_refresh &&
+		    zpool_get_state(node->zn_handle) != POOL_STATE_UNAVAIL)
+			continue;
+
+		/* Do the refresh ourselves, just in case. */
+		boolean_t missing;
+		zpool_refresh_stats(node->zn_handle, &missing);
+		if (missing) {
+			uu_avl_remove(zlp->zl_avl, node);
+			zpool_close(node->zn_handle);
+			free(node);
+		} else {
+			node->zn_last_refresh = zlp->zl_last_refresh;
+		}
+	}
+
+	return (uu_avl_numnodes(zlp->zl_avl));
 }
 
 /*
@@ -190,23 +262,6 @@ pool_list_iter(zpool_list_t *zlp, int unavail, zpool_iter_f func,
 	return (ret);
 }
 
-/*
- * Remove the given pool from the list.  When running iostat, we want to remove
- * those pools that no longer exist.
- */
-void
-pool_list_remove(zpool_list_t *zlp, zpool_handle_t *zhp)
-{
-	zpool_node_t search, *node;
-
-	search.zn_handle = zhp;
-	if ((node = uu_avl_find(zlp->zl_avl, &search, NULL, NULL)) != NULL) {
-		uu_avl_remove(zlp->zl_avl, node);
-		zpool_close(node->zn_handle);
-		free(node);
-	}
-}
-
 /*
  * Free all the handles associated with this list.
  */

cmd/zpool/zpool_main.c

@@ -33,7 +33,7 @@
  * Copyright (c) 2017, Intel Corporation.
  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
- * Copyright (c) 2021, 2023, Klara Inc.
+ * Copyright (c) 2021, 2023, 2025, Klara, Inc.
  * Copyright (c) 2021, 2025 Hewlett Packard Enterprise Development LP.
  */
 
@@ -5761,24 +5761,6 @@ children:
 	return (ret);
 }
 
-static int
-refresh_iostat(zpool_handle_t *zhp, void *data)
-{
-	iostat_cbdata_t *cb = data;
-	boolean_t missing;
-
-	/*
-	 * If the pool has disappeared, remove it from the list and continue.
-	 */
-	if (zpool_refresh_stats(zhp, &missing) != 0)
-		return (-1);
-
-	if (missing)
-		pool_list_remove(cb->cb_list, zhp);
-
-	return (0);
-}
-
 /*
  * Callback to print out the iostats for the given pool.
  */
@@ -6359,15 +6341,14 @@ get_namewidth_iostat(zpool_handle_t *zhp, void *data)
  * This command can be tricky because we want to be able to deal with pool
  * creation/destruction as well as vdev configuration changes.  The bulk of this
  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
- * on pool_list_update() to detect the addition of new pools.  Configuration
- * changes are all handled within libzfs.
+ * on pool_list_refresh() to detect the addition and removal of pools.
+ * Configuration changes are all handled within libzfs.
  */
 int
 zpool_do_iostat(int argc, char **argv)
 {
 	int c;
 	int ret;
-	int npools;
 	float interval = 0;
 	unsigned long count = 0;
 	zpool_list_t *list;
@@ -6618,10 +6599,24 @@ zpool_do_iostat(int argc, char **argv)
 		return (1);
 	}
 
+	int last_npools = 0;
 	for (;;) {
-		if ((npools = pool_list_count(list)) == 0)
+		/*
+		 * Refresh all pools in list, adding or removing pools as
+		 * necessary.
+		 */
+		int npools = pool_list_refresh(list);
+		if (npools == 0) {
 			(void) fprintf(stderr, gettext("no pools available\n"));
-		else {
+		} else {
+			/*
+			 * If the list of pools has changed since last time
+			 * around, reset the iteration count to force the
+			 * header to be redisplayed.
+			 */
+			if (last_npools != npools)
+				cb.cb_iteration = 0;
+
 			/*
 			 * If this is the first iteration and -y was supplied
 			 * we skip any printing.
@@ -6629,15 +6624,6 @@ zpool_do_iostat(int argc, char **argv)
 			boolean_t skip = (omit_since_boot &&
 			    cb.cb_iteration == 0);
 
-			/*
-			 * Refresh all statistics.  This is done as an
-			 * explicit step before calculating the maximum name
-			 * width, so that any * configuration changes are
-			 * properly accounted for.
-			 */
-			(void) pool_list_iter(list, B_FALSE, refresh_iostat,
-			    &cb);
-
 			/*
 			 * Iterate over all pools to determine the maximum width
 			 * for the pool / device name column across all pools.
@@ -6728,6 +6714,8 @@ zpool_do_iostat(int argc, char **argv)
 
 		(void) fflush(stdout);
 		(void) fsleep(interval);
+
+		last_npools = npools;
 	}
 
 	pool_list_free(list);

cmd/zpool/zpool_util.h

@@ -76,11 +76,10 @@ typedef struct zpool_list zpool_list_t;
 
 zpool_list_t *pool_list_get(int, char **, zprop_list_t **, zfs_type_t,
     boolean_t, int *);
-void pool_list_update(zpool_list_t *);
+int pool_list_refresh(zpool_list_t *);
 int pool_list_iter(zpool_list_t *, int unavail, zpool_iter_f, void *);
 void pool_list_free(zpool_list_t *);
 int pool_list_count(zpool_list_t *);
-void pool_list_remove(zpool_list_t *, zpool_handle_t *);
 
 extern libzfs_handle_t *g_zfs;
 

tests/runfiles/common.run

@@ -491,6 +491,10 @@ tests = ['zpool_import_001_pos', 'zpool_import_002_pos',
 tags = ['functional', 'cli_root', 'zpool_import']
 timeout = 1200
 
+[tests/functional/cli_root/zpool_iostat]
+tests = ['zpool_iostat_interval_all', 'zpool_iostat_interval_some']
+tags = ['functional', 'cli_root', 'zpool_iostat']
+
 [tests/functional/cli_root/zpool_labelclear]
 tests = ['zpool_labelclear_active', 'zpool_labelclear_exported',
     'zpool_labelclear_removed', 'zpool_labelclear_valid']

tests/zfs-tests/tests/Makefile.am

@@ -197,6 +197,7 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \
 	functional/cli_root/zpool_import/blockfiles/unclean_export.dat.bz2 \
 	functional/cli_root/zpool_import/zpool_import.cfg \
 	functional/cli_root/zpool_import/zpool_import.kshlib \
+	functional/cli_root/zpool_iostat/zpool_iostat.kshlib \
 	functional/cli_root/zpool_initialize/zpool_initialize.kshlib \
 	functional/cli_root/zpool_labelclear/labelclear.cfg \
 	functional/cli_root/zpool_remove/zpool_remove.cfg \
@@ -1181,6 +1182,10 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zpool_import/zpool_import_parallel_admin.ksh \
 	functional/cli_root/zpool_import/zpool_import_parallel_neg.ksh \
 	functional/cli_root/zpool_import/zpool_import_parallel_pos.ksh \
+	functional/cli_root/zpool_iostat/setup.ksh \
+	functional/cli_root/zpool_iostat/cleanup.ksh \
+	functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh \
+	functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh \
 	functional/cli_root/zpool_initialize/cleanup.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_attach_detach_add_remove.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_fault_export_import_online.ksh \

tests/zfs-tests/tests/functional/cli_root/zpool_iostat/cleanup.ksh

@@ -0,0 +1,30 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+#
+. $STF_SUITE/include/libtest.shlib
+
+log_pass

tests/zfs-tests/tests/functional/cli_root/zpool_iostat/setup.ksh

@@ -0,0 +1,32 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+#
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_pass

tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib

@@ -0,0 +1,235 @@
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+# Since we want to make sure that iostat responds correctly as pools appear and
+# disappear, we run it in the background and capture its output to a file.
+# Once we're done, we parse the output and ensure it matches what we'd expect
+# from the operations we performed.
+#
+# Because iostat is producing output every interval, it may produce the "same"
+# output for each step of the change; in fact, we want that to make sure we
+# don't miss anything. So, we describe what we expect as a series of "chunks".
+# Each chunk is a particular kind of output, which may repeat. Current known
+# chunk types are:
+#
+# NOPOOL: the text "no pools available"
+# HEADER: three lines, starting with "capacity", "pool" and "----" respectively.
+#         (the rough shape of the normal iostat header).
+# POOL1: a line starting with "pool1" (stats line for a pool of that name)
+# POOL2: a line starting with "pool2"
+# POOLBOTH: three lines, starting with "pool1", "pool2" (either order) and
+#           "-----" respectively. (the pool stat output for multiple pools)
+#
+# (the parser may produce other chunks in a failed parse to assist with
+#  debugging, but they should never be part of the "wanted" output See the
+#  parser commentary below).
+#
+# To help recognise the start of a new interval output, we run iostat with the
+# -T u option, which will output a numeric timestamp before each header or
+# second-or-later pool stat after the header.
+#
+# To keep the test run shorter, we use a subsecond interval, but to make sure
+# nothing is missed, we sleep for three intervals after each change.
+
+typeset _iostat_out=$(mktemp)
+typeset _iostat_pid=""
+
+function cleanup_iostat {
+	if [[ -n $_iostat_pid ]] ; then
+		kill -KILL $_iostat_pid || true
+	fi
+	rm -f $_iostat_out
+}
+
+function start_iostat {
+	zpool iostat -T u $@ 0.1 > $_iostat_out 2>&1 &
+	_iostat_pid=$!
+}
+
+function stop_iostat {
+	kill -TERM $_iostat_pid
+	wait $_iostat_pid
+	_iostat_pid=""
+}
+
+function delay_iostat {
+	sleep 0.3
+}
+
+typeset -a _iostat_expect
+function expect_iostat {
+	typeset chunk=$1
+	_iostat_expect+=($chunk)
+}
+
+# Parse the output The `state` var is used to track state across
+# multiple lines. The `last` var and the `_got_iostat` function are used
+# to record the completed chunks, and to collapse repetitions.
+typeset -a _iostat_got
+typeset _iostat_last=""
+typeset _iostat_state=""
+
+function _got_iostat {
+	typeset chunk=$1
+	if [[ -n $chunk && $_iostat_last != $chunk ]] ; then
+		_iostat_last=$chunk
+		_iostat_got+=($chunk)
+	fi
+	_iostat_state=""
+}
+
+function verify_iostat {
+
+	cat $_iostat_out | while read line ; do
+
+		# The "no pools available" text has no timestamp or other
+		# header, and should never appear in the middle of multiline
+		# chunk, so we can close any in-flight state.
+		if [[ $line = "no pools available" ]] ; then
+			_got_iostat $_iostat_state
+			_got_iostat "NOPOOL"
+			continue
+		fi
+
+		# A run of digits alone on the line is a timestamp (the `-T u`
+		# switch to `iostat`). It closes any in-flight state as a
+		# complete chunk, and indicates the start of a new chunk.
+		if [[ -z ${line/#+([0-9])/} ]] ; then
+			_got_iostat $_iostat_state
+			_iostat_state="TIMESTAMP"
+			continue
+		fi
+
+		# For this test, the first word of each line should be unique,
+		# so we extract it and use it for simplicity.
+		typeset first=${line%% *}
+
+		# Header is emitted whenever the pool list changes. It has
+		# three lines:
+		#
+		#                   capacity     operations     bandwidth
+		#     pool        alloc   free   read  write   read  write
+		#     ----------  -----  -----  -----  -----  -----  -----
+		#
+		# Each line moves the state; when we get to a run of dashes, we
+		# commit.  Note that we check for one-or-more dashes, because
+		# the width can vary depending on the length of pool name.
+		#
+		if [[ $_iostat_state = "TIMESTAMP" &&
+		    $first = "capacity" ]] ; then
+			_iostat_state="INHEADER1"
+			continue
+		fi
+		if [[ $_iostat_state = "INHEADER1" &&
+		    $first = "pool" ]] ; then
+			_iostat_state="INHEADER2"
+			continue
+		fi
+		if [[ $_iostat_state = "INHEADER2" &&
+		    -z ${first/#+(-)/} ]] ; then
+			# Headers never repeat, so if the last committed chunk
+			# was a header, we commit this one as EXTRAHEADER so we
+			# can see it in the error output.
+			if [[ $_iostat_last = "HEADER" ]] ; then
+				_got_iostat "EXTRAHEADER"
+			elif [[ $_iostat_last != "EXTRAHEADER" ]] ; then
+				_got_iostat "HEADER"
+			fi
+			_iostat_state="HEADER"
+			continue
+		fi
+
+		# A pool stat line looks like:
+		#
+		#     pool1        147K   240M      0      0      0      0
+		#
+		# If there are multiple pools, iostat follows them with a
+		# separator of dashed lines:
+		#
+		#     pool1        147K   240M      0      0      0      0
+		#     pool2        147K   240M      0      0      0      0
+		#     ----------  -----  -----  -----  -----  -----  -----
+		#
+		# Stats rows always start after a timestamp or a header. If the
+		# header was emitted, we won't see a timestamp here (it goes
+		# before the header).
+		#
+		# Because our test exercises both pools on their own and
+		# together, we allow pools in either order. In practice they
+		# are sorted, but that's a side-effect of the implementation
+		# (see zpool_compare()), so we're not going to rely on it here.
+		if [[ $first = "pool1" ]] || [[ $first = "pool2" ]] ; then
+
+			# First line, track which one we saw. If it's a
+			# standalone line, it will be committed by the next
+			# NOPOOL or TIMESTAMP above (or the `_got_iostat` after
+			# the loop if this is the last line).
+			if [[ $_iostat_state == "TIMESTAMP" ||
+			    $_iostat_state == "HEADER" ]] ; then
+				if [[ $first = "pool1" ]] ; then
+					_iostat_state="POOL1"
+				elif [[ $first = "pool2" ]] ; then
+					_iostat_state="POOL2"
+				fi
+				continue
+			fi
+
+			# If this is the second pool, we're in a multi-pool
+			# block, and need to look for the separator to close it
+			# out.
+			if [[ $_iostat_state = "POOL1" && $first = "pool2" ]] ||
+			    [[ $_iostat_state = "POOL2" && $first = "pool1" ]] ;
+			    then
+				_iostat_state="INPOOLBOTH"
+				continue
+			fi
+		fi
+
+		# Separator after the stats block.
+		if [[ $_iostat_state = "INPOOLBOTH" &&
+		    -z ${first/#+(-)/} ]] ; then
+			_got_iostat "POOLBOTH"
+			continue
+		fi
+
+		# Anything else will fall through to here. We commit any
+		# in-flight state, then "UNKNOWN", all to help with debugging..
+		if [[ $_iostat_state != "UNKNOWN" ]] ; then
+			_got_iostat $_iostat_state
+			_got_iostat "UNKNOWN"
+		fi
+	done
+
+	# Close out any remaining state.
+	_got_iostat $_iostat_state
+
+	# Compare what we wanted with what we got, and pass/fail the test!
+	if [[ "${_iostat_expect[*]}" != "${_iostat_got[*]}" ]] ; then
+		log_note "expected: ${_iostat_expect[*]}"
+		log_note "     got: ${_iostat_got[*]}"
+		log_fail "zpool iostat did not produce expected output"
+	fi
+}

tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh

@@ -0,0 +1,90 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+# `zpool iostat <N>` should keep running and update the pools it displays as
+# pools are created/destroyed/imported/export.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib
+
+typeset vdev1=$(mktemp)
+typeset vdev2=$(mktemp)
+
+function cleanup {
+	cleanup_iostat
+
+	poolexists pool1 && destroy_pool pool1
+	poolexists pool2 && destroy_pool pool2
+	rm -f $vdev1 $vdev2
+}
+
+log_must mkfile $MINVDEVSIZE $vdev1 $vdev2
+
+expect_iostat "NOPOOL"
+
+start_iostat
+
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL1"
+log_must zpool create pool1 $vdev1
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOLBOTH"
+log_must zpool create pool2 $vdev2
+delay_iostat
+
+expect_iostat "NOPOOL"
+log_must zpool export -a
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL2"
+log_must zpool import -d $vdev2 pool2
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOLBOTH"
+log_must zpool import -d $vdev1 pool1
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL2"
+log_must zpool destroy pool1
+delay_iostat
+
+expect_iostat "NOPOOL"
+log_must zpool destroy pool2
+delay_iostat
+
+stop_iostat
+
+verify_iostat
+
+log_pass "zpool iostat in interval mode follows pool updates"

tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh

@@ -0,0 +1,80 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+# `zpool iostat <pools> <N>` should keep running and only show the listed pools.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib
+
+typeset vdev1=$(mktemp)
+typeset vdev2=$(mktemp)
+
+function cleanup {
+	cleanup_iostat
+
+	poolexists pool1 && destroy_pool pool1
+	poolexists pool2 && destroy_pool pool2
+	rm -f $vdev1 $vdev2
+}
+
+log_must mkfile $MINVDEVSIZE $vdev1 $vdev2
+
+log_must zpool create pool1 $vdev1
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL1"
+start_iostat pool1
+delay_iostat
+
+log_must zpool create pool2 $vdev2
+delay_iostat
+
+expect_iostat "NOPOOL"
+log_must zpool export -a
+delay_iostat
+
+log_must zpool import -d $vdev2 pool2
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL1"
+log_must zpool import -d $vdev1 pool1
+delay_iostat
+
+expect_iostat "NOPOOL"
+log_must zpool destroy pool1
+delay_iostat
+
+log_must zpool destroy pool2
+delay_iostat
+
+stop_iostat
+
+verify_iostat
+
+log_pass "zpool iostat in interval mode with pools follows listed pool updates"