mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-22 10:37:35 +03:00
Parallel pool import
This commit allow spa_load() to drop the spa_namespace_lock so that imports can happen concurrently. Prior to dropping the spa_namespace_lock, the import logic will set the spa_load_thread value to track the thread which is doing the import. Consumers of spa_lookup() retain the same behavior by blocking when either a thread is holding the spa_namespace_lock or the spa_load_thread value is set. This will ensure that critical concurrent operations cannot take place while a pool is being imported. The zpool command is also enhanced to provide multi-threaded support when invoking zpool import -a. Lastly, zinject provides a mechanism to insert artificial delays when importing a pool and new zfs tests are added to verify parallel import functionality. Contributions-by: Don Brady <don.brady@klarasystems.com> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: George Wilson <gwilson@delphix.com> Closes #16093
This commit is contained in:
George Wilson
@@ -466,7 +466,8 @@ tests = ['zpool_import_001_pos', 'zpool_import_002_pos',
|
||||
'import_paths_changed',
|
||||
'import_rewind_config_changed',
|
||||
'import_rewind_device_replaced',
|
||||
'zpool_import_status']
|
||||
'zpool_import_status', 'zpool_import_parallel_pos',
|
||||
'zpool_import_parallel_neg', 'zpool_import_parallel_admin']
|
||||
tags = ['functional', 'cli_root', 'zpool_import']
|
||||
timeout = 1200
|
||||
|
||||
|
||||
@@ -1144,6 +1144,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
|
||||
functional/cli_root/zpool_import/zpool_import_missing_003_pos.ksh \
|
||||
functional/cli_root/zpool_import/zpool_import_rename_001_pos.ksh \
|
||||
functional/cli_root/zpool_import/zpool_import_status.ksh \
|
||||
functional/cli_root/zpool_import/zpool_import_parallel_admin.ksh \
|
||||
functional/cli_root/zpool_import/zpool_import_parallel_neg.ksh \
|
||||
functional/cli_root/zpool_import/zpool_import_parallel_pos.ksh \
|
||||
functional/cli_root/zpool_initialize/cleanup.ksh \
|
||||
functional/cli_root/zpool_initialize/zpool_initialize_attach_detach_add_remove.ksh \
|
||||
functional/cli_root/zpool_initialize/zpool_initialize_fault_export_import_online.ksh \
|
||||
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
#!/bin/ksh -p
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# The contents of this file are subject to the terms of the
|
||||
# Common Development and Distribution License (the "License").
|
||||
# You may not use this file except in compliance with the License.
|
||||
#
|
||||
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
# or https://opensource.org/licenses/CDDL-1.0.
|
||||
# See the License for the specific language governing permissions
|
||||
# and limitations under the License.
|
||||
#
|
||||
# When distributing Covered Code, include this CDDL HEADER in each
|
||||
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
# If applicable, add the following below this CDDL HEADER, with the
|
||||
# fields enclosed by brackets "[]" replaced with your own identifying
|
||||
# information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
|
||||
# Use is subject to license terms.
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2023 Klara, Inc.
|
||||
#
|
||||
|
||||
. $STF_SUITE/include/libtest.shlib
|
||||
. $STF_SUITE/tests/functional/cli_root/zpool_import/zpool_import.cfg
|
||||
. $STF_SUITE/tests/functional/cli_root/zpool_import/zpool_import.kshlib
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Verify that admin commands to different pool are not blocked by import
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create 2 pools
|
||||
# 2. Export one of the pools
|
||||
# 4. Import the pool with an injected delay
|
||||
# 5. Execute some admin commands against both pools
|
||||
# 6. Verify that the admin commands to the non-imported pool don't stall
|
||||
#
|
||||
|
||||
verify_runnable "global"
|
||||
|
||||
function cleanup
|
||||
{
|
||||
zinject -c all
|
||||
destroy_pool $TESTPOOL1
|
||||
destroy_pool $TESTPOOL2
|
||||
}
|
||||
|
||||
function pool_import
|
||||
{
|
||||
typeset dir=$1
|
||||
typeset pool=$2
|
||||
|
||||
SECONDS=0
|
||||
errmsg=$(zpool import -d $dir -f $pool 2>&1 > /dev/null)
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo ${pool}: imported in $SECONDS secs
|
||||
echo $SECONDS > ${DEVICE_DIR}/${pool}-import
|
||||
else
|
||||
echo ${pool}: import failed $errmsg in $SECONDS secs
|
||||
fi
|
||||
}
|
||||
|
||||
function pool_add_device
|
||||
{
|
||||
typeset pool=$1
|
||||
typeset device=$2
|
||||
typeset devtype=$3
|
||||
|
||||
SECONDS=0
|
||||
errmsg=$(zpool add $pool $devtype $device 2>&1 > /dev/null)
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo ${pool}: added $devtype vdev in $SECONDS secs
|
||||
echo $SECONDS > ${DEVICE_DIR}/${pool}-add
|
||||
else
|
||||
echo ${pool}: add $devtype vdev failed ${errmsg}, in $SECONDS secs
|
||||
fi
|
||||
}
|
||||
|
||||
function pool_stats
|
||||
{
|
||||
typeset stats=$1
|
||||
typeset pool=$2
|
||||
|
||||
SECONDS=0
|
||||
errmsg=$(zpool $stats $pool 2>&1 > /dev/null)
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo ${pool}: $stats in $SECONDS secs
|
||||
echo $SECONDS > ${DEVICE_DIR}/${pool}-${stats}
|
||||
else
|
||||
echo ${pool}: $stats failed ${errmsg}, in $SECONDS secs
|
||||
fi
|
||||
}
|
||||
|
||||
function pool_create
|
||||
{
|
||||
typeset pool=$1
|
||||
typeset device=$2
|
||||
|
||||
SECONDS=0
|
||||
errmsg=$(zpool create $pool $device 2>&1 > /dev/null)
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo ${pool}: created in $SECONDS secs
|
||||
echo $SECONDS > ${DEVICE_DIR}/${pool}-create
|
||||
else
|
||||
echo ${pool}: create failed ${errmsg}, in $SECONDS secs
|
||||
fi
|
||||
}
|
||||
|
||||
log_assert "Simple admin commands to different pool not blocked by import"
|
||||
|
||||
log_onexit cleanup
|
||||
|
||||
#
|
||||
# create two pools and export one
|
||||
#
|
||||
log_must zpool create $TESTPOOL1 $VDEV0
|
||||
log_must zpool export $TESTPOOL1
|
||||
log_must zpool create $TESTPOOL2 $VDEV1
|
||||
|
||||
#
|
||||
# import pool asyncronously with an injected 10 second delay
|
||||
#
|
||||
log_must zinject -P import -s 10 $TESTPOOL1
|
||||
pool_import $DEVICE_DIR $TESTPOOL1 &
|
||||
|
||||
sleep 2
|
||||
|
||||
#
|
||||
# run some admin commands on the pools while the import is in progress
|
||||
#
|
||||
|
||||
pool_add_device $TESTPOOL1 $VDEV2 "log" &
|
||||
pool_add_device $TESTPOOL2 $VDEV3 "cache" &
|
||||
pool_stats "status" $TESTPOOL1 &
|
||||
pool_stats "status" $TESTPOOL2 &
|
||||
pool_stats "list" $TESTPOOL1 &
|
||||
pool_stats "list" $TESTPOOL2 &
|
||||
pool_create $TESTPOOL1 $VDEV4 &
|
||||
wait
|
||||
|
||||
log_must zpool sync $TESTPOOL1 $TESTPOOL2
|
||||
|
||||
zpool history $TESTPOOL1
|
||||
zpool history $TESTPOOL2
|
||||
|
||||
log_must test "5" -lt $(<${DEVICE_DIR}/${TESTPOOL1}-import)
|
||||
|
||||
#
|
||||
# verify that commands to second pool did not wait for import to finish
|
||||
#
|
||||
log_must test "2" -gt $(<${DEVICE_DIR}/${TESTPOOL2}-status)
|
||||
log_must test "2" -gt $(<${DEVICE_DIR}/${TESTPOOL2}-list)
|
||||
log_must test "2" -gt $(<${DEVICE_DIR}/${TESTPOOL2}-add)
|
||||
[[ -e ${DEVICE_DIR}/${TESTPOOL1}-create ]] && log_fail "unexpected pool create"
|
||||
|
||||
log_pass "Simple admin commands to different pool not blocked by import"
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
#!/bin/ksh -p
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# The contents of this file are subject to the terms of the
|
||||
# Common Development and Distribution License (the "License").
|
||||
# You may not use this file except in compliance with the License.
|
||||
#
|
||||
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
# or https://opensource.org/licenses/CDDL-1.0.
|
||||
# See the License for the specific language governing permissions
|
||||
# and limitations under the License.
|
||||
#
|
||||
# When distributing Covered Code, include this CDDL HEADER in each
|
||||
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
# If applicable, add the following below this CDDL HEADER, with the
|
||||
# fields enclosed by brackets "[]" replaced with your own identifying
|
||||
# information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
|
||||
# Use is subject to license terms.
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2023 Klara, Inc.
|
||||
#
|
||||
|
||||
. $STF_SUITE/include/libtest.shlib
|
||||
. $STF_SUITE/tests/functional/cli_root/zpool_import/zpool_import.cfg
|
||||
. $STF_SUITE/tests/functional/cli_root/zpool_import/zpool_import.kshlib
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Verify that pool imports by same name only have one winner
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create 4 single disk pools with the same name
|
||||
# 2. Generate some ZIL records (for a longer import)
|
||||
# 3. Export the pools
|
||||
# 4. Import the pools in parallel
|
||||
# 5. Repeat with using matching guids
|
||||
#
|
||||
|
||||
verify_runnable "global"
|
||||
|
||||
POOLNAME="import_pool"
|
||||
DEV_DIR_PREFIX="$DEVICE_DIR/$POOLNAME"
|
||||
VDEVSIZE=$((512 * 1024 * 1024))
|
||||
|
||||
log_assert "parallel pool imports by same name only have one winner"
|
||||
|
||||
# each pool has its own device directory
|
||||
for i in {0..3}; do
|
||||
log_must mkdir -p ${DEV_DIR_PREFIX}$i
|
||||
log_must truncate -s $VDEVSIZE ${DEV_DIR_PREFIX}$i/${DEVICE_FILE}$i
|
||||
done
|
||||
|
||||
function cleanup
|
||||
{
|
||||
zinject -c all
|
||||
log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 0
|
||||
log_must set_tunable64 METASLAB_DEBUG_LOAD 0
|
||||
|
||||
destroy_pool $POOLNAME
|
||||
|
||||
log_must rm -rf $DEV_DIR_PREFIX*
|
||||
}
|
||||
|
||||
log_onexit cleanup
|
||||
|
||||
log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 1
|
||||
log_must set_tunable64 METASLAB_DEBUG_LOAD 1
|
||||
|
||||
function import_pool
|
||||
{
|
||||
typeset dir=$1
|
||||
typeset pool=$2
|
||||
typeset newname=$3
|
||||
|
||||
SECONDS=0
|
||||
errmsg=$(zpool import -N -d $dir -f $pool $newname 2>&1 > /dev/null)
|
||||
if [[ $? -eq 0 ]]; then
|
||||
touch $dir/imported
|
||||
echo "imported $pool in $SECONDS secs"
|
||||
elif [[ $errmsg == *"cannot import"* ]]; then
|
||||
echo "pool import failed: $errmsg, waited $SECONDS secs"
|
||||
touch $dir/failed
|
||||
fi
|
||||
}
|
||||
|
||||
#
|
||||
# create four exported pools with the same name
|
||||
#
|
||||
for i in {0..3}; do
|
||||
log_must zpool create $POOLNAME ${DEV_DIR_PREFIX}$i/${DEVICE_FILE}$i
|
||||
log_must zpool export $POOLNAME
|
||||
done
|
||||
log_must zinject -P import -s 10 $POOLNAME
|
||||
|
||||
#
|
||||
# import the pools in parallel, expecting only one winner
|
||||
#
|
||||
for i in {0..3}; do
|
||||
import_pool ${DEV_DIR_PREFIX}$i $POOLNAME &
|
||||
done
|
||||
wait
|
||||
|
||||
# check the result of background imports
|
||||
typeset num_imports=0
|
||||
typeset num_cannot=0
|
||||
for i in {0..3}; do
|
||||
if [[ -f ${DEV_DIR_PREFIX}$i/imported ]]; then
|
||||
((num_imports += 1))
|
||||
fi
|
||||
if [[ -f ${DEV_DIR_PREFIX}$i/failed ]]; then
|
||||
((num_cannot += 1))
|
||||
loser=$i
|
||||
fi
|
||||
done
|
||||
[[ $num_imports -eq "1" ]] || log_fail "expecting an import"
|
||||
[[ $num_cannot -eq "3" ]] || \
|
||||
log_fail "expecting 3 pool exists errors, found $num_cannot"
|
||||
|
||||
log_note "$num_imports imported and $num_cannot failed (expected)"
|
||||
|
||||
log_pass "parallel pool imports by same name only have one winner"
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
#!/bin/ksh -p
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# The contents of this file are subject to the terms of the
|
||||
# Common Development and Distribution License (the "License").
|
||||
# You may not use this file except in compliance with the License.
|
||||
#
|
||||
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
# or https://opensource.org/licenses/CDDL-1.0.
|
||||
# See the License for the specific language governing permissions
|
||||
# and limitations under the License.
|
||||
#
|
||||
# When distributing Covered Code, include this CDDL HEADER in each
|
||||
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
# If applicable, add the following below this CDDL HEADER, with the
|
||||
# fields enclosed by brackets "[]" replaced with your own identifying
|
||||
# information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
|
||||
# Use is subject to license terms.
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2023 Klara, Inc.
|
||||
#
|
||||
|
||||
. $STF_SUITE/include/libtest.shlib
|
||||
. $STF_SUITE/tests/functional/cli_root/zpool_import/zpool_import.cfg
|
||||
. $STF_SUITE/tests/functional/cli_root/zpool_import/zpool_import.kshlib
|
||||
|
||||
# test uses 8 vdevs
|
||||
export MAX_NUM=8
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Verify that pool imports can occur in parallel
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create 8 pools
|
||||
# 2. Generate some ZIL records
|
||||
# 3. Export the pools
|
||||
# 4. Import half of the pools synchronously to baseline sequential cost
|
||||
# 5. Import the other half asynchronously to demonstrate parallel savings
|
||||
# 6. Export 4 pools
|
||||
# 7. Test zpool import -a
|
||||
#
|
||||
|
||||
verify_runnable "global"
|
||||
|
||||
#
|
||||
# override the minimum sized vdevs
|
||||
#
|
||||
VDEVSIZE=$((512 * 1024 * 1024))
|
||||
increase_device_sizes $VDEVSIZE
|
||||
|
||||
POOLNAME="import_pool"
|
||||
|
||||
function cleanup
|
||||
{
|
||||
zinject -c all
|
||||
log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 0
|
||||
log_must set_tunable64 METASLAB_DEBUG_LOAD 0
|
||||
|
||||
for i in {0..$(($MAX_NUM - 1))}; do
|
||||
destroy_pool $POOLNAME-$i
|
||||
done
|
||||
# reset the devices
|
||||
increase_device_sizes 0
|
||||
increase_device_sizes $FILE_SIZE
|
||||
}
|
||||
|
||||
log_assert "Pool imports can occur in parallel"
|
||||
|
||||
log_onexit cleanup
|
||||
|
||||
log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 1
|
||||
log_must set_tunable64 METASLAB_DEBUG_LOAD 1
|
||||
|
||||
|
||||
#
|
||||
# create some exported pools with import delay injectors
|
||||
#
|
||||
for i in {0..$(($MAX_NUM - 1))}; do
|
||||
log_must zpool create $POOLNAME-$i $DEVICE_DIR/${DEVICE_FILE}$i
|
||||
log_must zpool export $POOLNAME-$i
|
||||
log_must zinject -P import -s 12 $POOLNAME-$i
|
||||
done
|
||||
wait
|
||||
|
||||
#
|
||||
# import half of the pools synchronously
|
||||
#
|
||||
SECONDS=0
|
||||
for i in {0..3}; do
|
||||
log_must zpool import -d $DEVICE_DIR -f $POOLNAME-$i
|
||||
done
|
||||
sequential_time=$SECONDS
|
||||
log_note "sequentially imported 4 pools in $sequential_time seconds"
|
||||
|
||||
#
|
||||
# import half of the pools in parallel
|
||||
#
|
||||
SECONDS=0
|
||||
for i in {4..7}; do
|
||||
log_must zpool import -d $DEVICE_DIR -f $POOLNAME-$i &
|
||||
done
|
||||
wait
|
||||
parallel_time=$SECONDS
|
||||
log_note "asyncronously imported 4 pools in $parallel_time seconds"
|
||||
|
||||
log_must test $parallel_time -lt $(($sequential_time / 3))
|
||||
|
||||
#
|
||||
# export pools with import delay injectors
|
||||
#
|
||||
for i in {4..7}; do
|
||||
log_must zpool export $POOLNAME-$i
|
||||
log_must zinject -P import -s 12 $POOLNAME-$i
|
||||
done
|
||||
wait
|
||||
|
||||
#
|
||||
# now test zpool import -a
|
||||
#
|
||||
SECONDS=0
|
||||
log_must zpool import -a -d $DEVICE_DIR -f
|
||||
parallel_time=$SECONDS
|
||||
log_note "asyncronously imported 4 pools in $parallel_time seconds"
|
||||
|
||||
log_must test $parallel_time -lt $(($sequential_time / 3))
|
||||
|
||||
log_pass "Pool imports occur in parallel"
|
||||
Reference in New Issue
Block a user