mirror_zfs/tests/zfs-tests/tests/functional/mmp/mmp.kshlib

# SPDX-License-Identifier: CDDL-1.0
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or https://opensource.org/licenses/CDDL-1.0.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
# Use is subject to license terms.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/mmp/mmp.cfg


function check_pool_import # pool opts token keyword
{
	typeset pool=${1:-$MMP_POOL}
	typeset opts=$2
	typeset token=$3
	typeset keyword=$4

	zpool import $opts 2>&1 | \
	    awk -v token="$token:" '($1==token) {print}' | \
	    grep -iq "$keyword"
}

function is_pool_imported # pool opts
{
	typeset pool=${1:-$MMP_POOL}
	typeset opts=$2

	check_pool_import "$pool" "$opts" "status" \
	    "The pool is currently imported"
}

function wait_pool_imported # pool opts
{
	typeset pool=${1:-$MMP_POOL}
	typeset opts=$2

	while is_pool_imported "$pool" "$opts"; do
		log_must sleep 5
	done
}

function try_pool_import # pool opts message
{
	typeset pool=${1:-$MMP_POOL}
	typeset opts=$2
	typeset msg=$3

	zpool import $opts $pool 2>&1 | grep -i "$msg"
}

function mmp_set_hostid
{
	typeset hostid=$1

	zgenhostid $1

	[ $(hostid) = "$hostid" ]
}

function mmp_clear_hostid
{
	rm -f $HOSTID_FILE
}

function mmp_pool_create_simple # pool dir
{
	typeset pool=${1:-$MMP_POOL}
	typeset dir=${2:-$MMP_DIR}

	log_must mkdir -p $dir
	log_must rm -f $dir/*
	log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2

	log_must mmp_clear_hostid
	log_must mmp_set_hostid $HOSTID1
	log_must zpool create -f -o cachefile=$MMP_CACHE $pool \
	    mirror $dir/vdev1 $dir/vdev2
	log_must zpool set multihost=on $pool
}

function mmp_pool_create_zhack # pool dir
{
	typeset pool=${1:-$MMP_POOL}
	typeset dir=${2:-$MMP_DIR}
	typeset opts="-d $dir action idle -t120 $pool"

	mmp_pool_create_simple $pool $dir

	log_must mv $MMP_CACHE ${MMP_CACHE}.stale
	log_must zpool export $pool
	log_must mmp_clear_hostid
	log_must mmp_set_hostid $HOSTID2

	log_note "Starting zhack in the background as hostid $HOSTID1"
	log_must eval "ZFS_HOSTID=$HOSTID1 zhack $opts >$MMP_ZHACK_LOG 2>&1 &"

	while ! is_pool_imported "$pool" "-d $dir"; do
		log_must pgrep zhack
		log_must sleep 5
	done
}

function mmp_pool_destroy # pool dir
{
	typeset pool=${1:-$MMP_POOL}
	typeset dir=${2:-$MMP_DIR}

	ZHACKPID=$(pgrep zhack)
	if [ -n "$ZHACKPID" ]; then
		log_must kill $ZHACKPID
		wait $ZHACKPID
	fi

	if poolexists $pool; then
		destroy_pool $pool
        fi

	log_must rm -f $dir/*
	mmp_clear_hostid
}

function mmp_pool_set_hostid # pool hostid
{
	typeset pool=$1
	typeset hostid=$2

	log_must mmp_clear_hostid
	log_must mmp_set_hostid $hostid
	log_must zpool export $pool
	log_must zpool import $pool

	return 0
}

function import_no_activity_check # pool opts
{
	typeset pool=$1
	typeset opts=$2

	RESULT=$(ZFS_LOAD_INFO_DEBUG=1 zpool import $opts $pool)
	typeset rc=$?

	# mmp_result: 3 (ESRCH) no activity check was run not required
	# mmp_result: 6 (ENXIO) no activity check was run hostid not set
	if ! echo "$RESULT" | grep -q "mmp_result: 3" &&
	   ! echo "$RESULT" | grep -q "mmp_result: 6"; then
		log_note "ERROR: $RESULT"
		log_fail "ERROR: import_no_activity_check unexpected activity check"
	fi

	return $rc
}

function import_activity_check # pool opts
{
	typeset pool=$1
	typeset opts=$2

	RESULT=$(ZFS_LOAD_INFO_DEBUG=1 zpool import $opts $pool)
	typeset rc=$?

	# mmp_result: 0 (Success) check was run no activity detected
	# mmp_result: 121 (EREMOTEIO) check was run activity detected
	# mmp_result: 4 (EINTR) check was run but interrupted by user
	if ! echo "$RESULT" | grep -q "mmp_result: 0"; then
		log_note "ERROR: $RESULT"
		log_fail "ERROR: import_activity_check expected activity check"
	fi

	return $rc
}

function clear_mmp_history
{
	log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY_OFF
	log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY
}

function count_skipped_mmp_writes # pool duration
{
	typeset pool=$1
	typeset -i duration=$2

	sleep $duration
	kstat_pool $pool multihost | \
	    awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};'
}

function count_mmp_writes # pool duration
{
	typeset pool=$1
	typeset -i duration=$2

	sleep $duration
	kstat_pool $pool multihost | \
	    awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};'
}

function summarize_uberblock_mmp # device
{
	typeset device=$1

	zdb -luuuu $device | awk '
	BEGIN				{write_fail_present=0; write_fail_missing=0; uber_invalid=0;}
	/Uberblock\[[0-9][0-9]*\]/	{delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};};
	/mmp_fail/			{fail=$3};
	/mmp_seq/			{seq=$3};
	/mmp_write/			{write=$3};
	/mmp_delay/			{delay=$3; if (delay==0) {delay_zero++};};
	/mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++};
	/mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++};
	/mmp_valid/ && delay>0 && write<=0 {write_missing++};
	/mmp_valid/ && delay>0 && fail<=0 {fail_missing++};
	/mmp_valid/ && delay>0 && seq>0 {seq_nonzero++};
	END {
		print "total_uberblocks " total;
		print "delay_zero " delay_zero;
		print "write_fail_present " write_fail_present;
		print "write_fail_missing " write_fail_missing;
		print "write_missing " write_missing;
		print "fail_missing " fail_missing;
		print "seq_nonzero " seq_nonzero;
		print "uberblock_invalid " uber_invalid;
	}'
}

function count_mmp_write_fail_present # device
{
	typeset device=$1

	summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}'
}

function count_mmp_write_fail_missing # device
{
	typeset device=$1

	summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}'
}

function verify_mmp_write_fail_present # device
{
	typeset device=$1

	count=$(count_mmp_write_fail_present $device)
	log_note "present count: $count"
	if [ $count -eq 0 ]; then
		summarize_uberblock_mmp $device
		log_note "----- snip -----"
		zdb -luuuu $device
		log_note "----- snip -----"
		log_fail "No Uberblocks contain valid mmp_write and fail values"
	fi

	count=$(count_mmp_write_fail_missing $device)
	log_note "missing count: $count"
	if [ $count -gt 0 ]; then
		summarize_uberblock_mmp $device
		log_note "----- snip -----"
		zdb -luuuu $device
		log_note "----- snip -----"
		log_fail "Uberblocks missing mmp_write or mmp_fail"
	fi
}