Write label 2,3 uberblocks when vdev expands

When vdev_psize increases, the location of labels 2 and 3 changes
because their location is relative to the end of the device.

The configs for labels 2 and 3 are written during the next spa_sync()
because the vdev is added to the dirty config list.  However, the
uberblock rings are not re-written in their new location, leaving the
device vulnerable to the beginning of the device being overwritten or
damaged.

This patch copies the uberblock ring from label 0 to labels 2 and 3,
in their new locations, at the next sync after vdev_psize increases.

Also, add a test zpool_expand_004_pos.ksh to confirm the uberblocks
are copied.

Reviewed-by: BearBabyLiu <liu.huang@zte.com.cn>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Olaf Faaland <faaland1@llnl.gov>
Closes #5108

tests/zfs-tests/tests/functional/cli_root/zpool_expand/Makefile.am

@@ -5,4 +5,5 @@ dist_pkgdata_SCRIPTS = \
 	cleanup.ksh \
 	zpool_expand_001_pos.ksh \
 	zpool_expand_002_pos.ksh \
-	zpool_expand_003_neg.ksh
+	zpool_expand_003_neg.ksh \
+	zpool_expand_004_pos.ksh

tests/zfs-tests/tests/functional/cli_root/zpool_expand/zpool_expand_004_pos.ksh

@@ -0,0 +1,102 @@
+#! /bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2012, 2015 by Delphix. All rights reserved.
+# Copyright (c) 2017 Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_expand/zpool_expand.cfg
+
+#
+# DESCRIPTION:
+# After vdev expansion, all 4 labels have the same set of uberblocks.
+#
+#
+# STRATEGY:
+# 1) Create 3 files
+# 2) Create a pool backed by the files
+# 3) Expand the files' size with truncate
+# 4) Use zpool online -e to expand the vdevs
+# 5) Check that for all the devices, all 4 labels have the same uberblocks
+#
+
+verify_runnable "global"
+
+function cleanup
+{
+        if poolexists $TESTPOOL1; then
+                log_must zpool destroy $TESTPOOL1
+        fi
+
+	for i in 1 2 3; do
+		[ -e ${TEMPFILE}.$i ] && log_must rm ${TEMPFILE}.$i
+	done
+}
+
+log_onexit cleanup
+
+log_assert "After vdev expansion, all 4 labels have the same set of uberblocks."
+
+for type in " " mirror raidz raidz2; do
+	for i in 1 2 3; do
+		log_must truncate -s $org_size ${TEMPFILE}.$i
+	done
+
+	log_must zpool create $TESTPOOL1 $type $TEMPFILE.1 \
+	    $TEMPFILE.2 $TEMPFILE.3
+
+	sync_pool $TESTPOOL1
+
+	for i in 1 2 3; do
+		log_must truncate -s $exp_size ${TEMPFILE}.$i
+	done
+
+	for i in 1 2 3; do
+		log_must zpool online -e $TESTPOOL1 ${TEMPFILE}.$i
+	done
+
+	sync_pool $TESTPOOL1
+
+
+	for i in 1 2 3; do
+		non_uniform=$(zdb -lu ${TEMPFILE}.$i	| \
+		    grep 'labels = '			| \
+		    grep -c -v 'labels = 0 1 2 3')
+
+		log_note "non-uniform label count: $non_uniform"
+
+		if [[ $non_uniform -ne 0 ]]; then
+			log_fail "After vdev expansion, all labels contents are not identical"
+		fi
+	done
+
+	log_must zpool destroy $TESTPOOL1
+done
+
+log_pass "After vdev expansion, all 4 labels have the same set of uberblocks."