mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-24 19:28:53 +03:00
Always validate checksums for Direct I/O reads
This fixes an oversight in the Direct I/O PR. There is nothing that stops a process from manipulating the contents of a buffer for a Direct I/O read while the I/O is in flight. This can lead checksum verify failures. However, the disk contents are still correct, and this would lead to false reporting of checksum validation failures. To remedy this, all Direct I/O reads that have a checksum verification failure are treated as suspicious. In the event a checksum validation failure occurs for a Direct I/O read, then the I/O request will be reissued though the ARC. This allows for actual validation to happen and removes any possibility of the buffer being manipulated after the I/O has been issued. Just as with Direct I/O write checksum validation failures, Direct I/O read checksum validation failures are reported though zpool status -d in the DIO column. Also the zevent has been updated to have both: 1. dio_verify_wr -> Checksum verification failure for writes 2. dio_verify_rd -> Checksum verification failure for reads. This allows for determining what I/O operation was the culprit for the checksum verification failure. All DIO errors are reported only on the top-level VDEV. Even though FreeBSD can write protect pages (stable pages) it still has the same issue as Linux with Direct I/O reads. This commit updates the following: 1. Propogates checksum failures for reads all the way up to the top-level VDEV. 2. Reports errors through zpool status -d as DIO. 3. Has two zevents for checksum verify errors with Direct I/O. One for read and one for write. 4. Updates FreeBSD ABD code to also check for ABD_FLAG_FROM_PAGES and handle ABD buffer contents validation the same as Linux. 5. Updated manipulate_user_buffer.c to also manipulate a buffer while a Direct I/O read is taking place. 6. Adds a new ZTS test case dio_read_verify that stress tests the new code. 7. Updated man pages. 8. Added an IMPLY statement to zio_checksum_verify() to make sure that Direct I/O reads are not issued as speculative. 9. Removed self healing through mirror, raidz, and dRAID VDEVs for Direct I/O reads. This issue was first observed when installing a Windows 11 VM on a ZFS dataset with the dataset property direct set to always. The zpool devices would report checksum failures, but running a subsequent zpool scrub would not repair any data and report no errors. Reviewed-by: Tony Hutter <hutter2@llnl.gov> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Reviewed-by: Alexander Motin <mav@FreeBSD.org> Signed-off-by: Brian Atkinson <batkinson@lanl.gov> Closes #16598
This commit is contained in:
Brian Atkinson
committed by
Brian Behlendorf
Brian Behlendorf
parent
774dcba86d
commit
26ecd8b993
@@ -1477,6 +1477,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
|
||||
functional/direct/dio_overwrites.ksh \
|
||||
functional/direct/dio_property.ksh \
|
||||
functional/direct/dio_random.ksh \
|
||||
functional/direct/dio_read_verify.ksh \
|
||||
functional/direct/dio_recordsize.ksh \
|
||||
functional/direct/dio_unaligned_block.ksh \
|
||||
functional/direct/dio_unaligned_filesize.ksh \
|
||||
|
||||
@@ -84,8 +84,9 @@ function get_zpool_status_chksum_verify_failures # pool_name vdev_type
|
||||
function get_zed_dio_verify_events # pool
|
||||
{
|
||||
typeset pool=$1
|
||||
typeset op=$2
|
||||
|
||||
val=$(zpool events $pool | grep -c dio_verify)
|
||||
val=$(zpool events $pool | grep -c "dio_verify_${op}")
|
||||
|
||||
echo "$val"
|
||||
}
|
||||
@@ -96,11 +97,12 @@ function get_zed_dio_verify_events # pool
|
||||
# zpool events
|
||||
# After getting that counts will clear the out the ZPool errors and events
|
||||
#
|
||||
function check_dio_write_chksum_verify_failures # pool vdev_type expect_errors
|
||||
function check_dio_chksum_verify_failures # pool vdev_type op expect_errors
|
||||
{
|
||||
typeset pool=$1
|
||||
typeset vdev_type=$2
|
||||
typeset expect_errors=$3
|
||||
typeset op=$4
|
||||
typeset note_str="expecting none"
|
||||
|
||||
if [[ $expect_errors -ne 0 ]]; then
|
||||
@@ -108,10 +110,10 @@ function check_dio_write_chksum_verify_failures # pool vdev_type expect_errors
|
||||
fi
|
||||
|
||||
log_note "Checking for Direct I/O write checksum verify errors \
|
||||
$note_str on ZPool: $pool"
|
||||
$note_str on ZPool: $pool with $vdev_type"
|
||||
|
||||
status_failures=$(get_zpool_status_chksum_verify_failures $pool $vdev_type)
|
||||
zed_dio_verify_events=$(get_zed_dio_verify_events $pool)
|
||||
zed_dio_verify_events=$(get_zed_dio_verify_events $pool $op)
|
||||
|
||||
if [[ $expect_errors -ne 0 ]]; then
|
||||
if [[ $status_failures -eq 0 ||
|
||||
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
#!/bin/ksh -p
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# The contents of this file are subject to the terms of the
|
||||
# Common Development and Distribution License (the "License").
|
||||
# You may not use this file except in compliance with the License.
|
||||
#
|
||||
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
# or https://opensource.org/licenses/CDDL-1.0.
|
||||
# See the License for the specific language governing permissions
|
||||
# and limitations under the License.
|
||||
#
|
||||
# When distributing Covered Code, include this CDDL HEADER in each
|
||||
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
# If applicable, add the following below this CDDL HEADER, with the
|
||||
# fields enclosed by brackets "[]" replaced with your own identifying
|
||||
# information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2024 by Triad National Security, LLC.
|
||||
#
|
||||
|
||||
. $STF_SUITE/include/libtest.shlib
|
||||
. $STF_SUITE/tests/functional/direct/dio.cfg
|
||||
. $STF_SUITE/tests/functional/direct/dio.kshlib
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Verify checksum verify works for Direct I/O reads.
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create a zpool from each vdev type.
|
||||
# 2. Start a Direct I/O read workload while manipulating the user buffer
|
||||
# contents.
|
||||
# 3. Verify there are Direct I/O read verify failures using
|
||||
# zpool status -d and checking for zevents. We also make sure there
|
||||
# are reported no data errors.
|
||||
#
|
||||
|
||||
verify_runnable "global"
|
||||
|
||||
log_assert "Verify checksum verify works for Direct I/O reads."
|
||||
|
||||
log_onexit dio_cleanup
|
||||
|
||||
NUMBLOCKS=300
|
||||
BS=$((128 * 1024)) # 128k
|
||||
|
||||
log_must truncate -s $MINVDEVSIZE $DIO_VDEVS
|
||||
|
||||
# We will verify that there are no checksum errors for every Direct I/O read
|
||||
# while manipulating the buffer contents while the I/O is still in flight and
|
||||
# also that Direct I/O checksum verify failures and dio_verify_rd zevents are
|
||||
# reported.
|
||||
|
||||
|
||||
for type in "" "mirror" "raidz" "draid"; do
|
||||
typeset vdev_type=$type
|
||||
if [[ "${vdev_type}" == "" ]]; then
|
||||
vdev_type="stripe"
|
||||
fi
|
||||
|
||||
log_note "Verifying every Direct I/O read verify with VDEV type \
|
||||
${vdev_type}"
|
||||
|
||||
create_pool $TESTPOOL1 $type $DIO_VDEVS
|
||||
log_must eval "zfs create -o recordsize=128k -o compression=off \
|
||||
$TESTPOOL1/$TESTFS1"
|
||||
|
||||
mntpnt=$(get_prop mountpoint $TESTPOOL1/$TESTFS1)
|
||||
prev_dio_rd=$(get_iostats_stat $TESTPOOL1 direct_read_count)
|
||||
prev_arc_rd=$(get_iostats_stat $TESTPOOL1 arc_read_count)
|
||||
|
||||
# Create the file before trying to manipulate the contents
|
||||
log_must stride_dd -o "$mntpnt/direct-write.iso" -i /dev/urandom \
|
||||
-b $BS -c $NUMBLOCKS -D
|
||||
# Manipulate the buffer contents will reading the file with Direct I/O
|
||||
log_must manipulate_user_buffer -f "$mntpnt/direct-write.iso" \
|
||||
-n $NUMBLOCKS -b $BS -r
|
||||
|
||||
# Getting new Direct I/O and ARC Write counts.
|
||||
curr_dio_rd=$(get_iostats_stat $TESTPOOL1 direct_read_count)
|
||||
curr_arc_rd=$(get_iostats_stat $TESTPOOL1 arc_read_count)
|
||||
total_dio_rd=$((curr_dio_rd - prev_dio_rd))
|
||||
total_arc_rd=$((curr_arc_rd - prev_arc_rd))
|
||||
|
||||
log_note "Making sure there are no checksum errors with the ZPool"
|
||||
log_must check_pool_status $TESTPOOL "errors" "No known data errors"
|
||||
|
||||
log_note "Making sure we have Direct I/O and ARC reads logged"
|
||||
if [[ $total_dio_rd -lt 1 ]]; then
|
||||
log_fail "No Direct I/O reads $total_dio_rd"
|
||||
fi
|
||||
if [[ $total_arc_rd -lt 1 ]]; then
|
||||
log_fail "No ARC reads $total_arc_rd"
|
||||
fi
|
||||
|
||||
log_note "Making sure we have Direct I/O write checksum verifies with ZPool"
|
||||
check_dio_chksum_verify_failures "$TESTPOOL1" "$vdev_type" 1 "rd"
|
||||
destroy_pool $TESTPOOL1
|
||||
done
|
||||
|
||||
log_pass "Verified checksum verify works for Direct I/O reads."
|
||||
@@ -46,7 +46,7 @@ verify_runnable "global"
|
||||
function cleanup
|
||||
{
|
||||
log_must rm -f "$mntpnt/direct-write.iso"
|
||||
check_dio_write_chksum_verify_failures $TESTPOOL "raidz" 0
|
||||
check_dio_chksum_verify_failures $TESTPOOL "raidz" 0 "wr"
|
||||
}
|
||||
|
||||
log_assert "Verify stable pages work for Direct I/O writes."
|
||||
@@ -76,8 +76,8 @@ do
|
||||
|
||||
# Manipulate the user's buffer while running O_DIRECT write
|
||||
# workload with the buffer.
|
||||
log_must manipulate_user_buffer -o "$mntpnt/direct-write.iso" \
|
||||
-n $NUMBLOCKS -b $BS
|
||||
log_must manipulate_user_buffer -f "$mntpnt/direct-write.iso" \
|
||||
-n $NUMBLOCKS -b $BS -w
|
||||
|
||||
# Reading back the contents of the file
|
||||
log_must stride_dd -i $mntpnt/direct-write.iso -o /dev/null \
|
||||
|
||||
@@ -91,8 +91,8 @@ log_must set_tunable32 VDEV_DIRECT_WR_VERIFY 0
|
||||
log_note "Verifying no panics for Direct I/O writes with compression"
|
||||
log_must zfs set compression=on $TESTPOOL/$TESTFS
|
||||
prev_dio_wr=$(get_iostats_stat $TESTPOOL direct_write_count)
|
||||
log_must manipulate_user_buffer -o "$mntpnt/direct-write.iso" -n $NUMBLOCKS \
|
||||
-b $BS
|
||||
log_must manipulate_user_buffer -f "$mntpnt/direct-write.iso" -n $NUMBLOCKS \
|
||||
-b $BS -w
|
||||
curr_dio_wr=$(get_iostats_stat $TESTPOOL direct_write_count)
|
||||
total_dio_wr=$((curr_dio_wr - prev_dio_wr))
|
||||
|
||||
@@ -116,8 +116,8 @@ for i in $(seq 1 $ITERATIONS); do
|
||||
$i of $ITERATIONS with zfs_vdev_direct_write_verify=0"
|
||||
|
||||
prev_dio_wr=$(get_iostats_stat $TESTPOOL direct_write_count)
|
||||
log_must manipulate_user_buffer -o "$mntpnt/direct-write.iso" \
|
||||
-n $NUMBLOCKS -b $BS
|
||||
log_must manipulate_user_buffer -f "$mntpnt/direct-write.iso" \
|
||||
-n $NUMBLOCKS -b $BS -w
|
||||
|
||||
# Reading file back to verify checksum errors
|
||||
filesize=$(get_file_size "$mntpnt/direct-write.iso")
|
||||
@@ -144,7 +144,7 @@ for i in $(seq 1 $ITERATIONS); do
|
||||
fi
|
||||
log_note "Making sure we have no Direct I/O write checksum verifies \
|
||||
with ZPool"
|
||||
check_dio_write_chksum_verify_failures $TESTPOOL "raidz" 0
|
||||
check_dio_chksum_verify_failures $TESTPOOL "raidz" 0 "wr"
|
||||
|
||||
log_must rm -f "$mntpnt/direct-write.iso"
|
||||
done
|
||||
@@ -166,8 +166,8 @@ for i in $(seq 1 $ITERATIONS); do
|
||||
$ITERATIONS with zfs_vdev_direct_write_verify=1"
|
||||
|
||||
prev_dio_wr=$(get_iostats_stat $TESTPOOL direct_write_count)
|
||||
log_must manipulate_user_buffer -o "$mntpnt/direct-write.iso" \
|
||||
-n $NUMBLOCKS -b $BS -e
|
||||
log_must manipulate_user_buffer -f "$mntpnt/direct-write.iso" \
|
||||
-n $NUMBLOCKS -b $BS -e -w
|
||||
|
||||
# Reading file back to verify there no are checksum errors
|
||||
filesize=$(get_file_size "$mntpnt/direct-write.iso")
|
||||
@@ -175,7 +175,7 @@ for i in $(seq 1 $ITERATIONS); do
|
||||
log_must stride_dd -i "$mntpnt/direct-write.iso" -o /dev/null -b $BS \
|
||||
-c $num_blocks
|
||||
|
||||
# Getting new Direct I/O and ARC Write counts.
|
||||
# Getting new Direct I/O write counts.
|
||||
curr_dio_wr=$(get_iostats_stat $TESTPOOL direct_write_count)
|
||||
total_dio_wr=$((curr_dio_wr - prev_dio_wr))
|
||||
|
||||
@@ -188,7 +188,7 @@ for i in $(seq 1 $ITERATIONS); do
|
||||
fi
|
||||
|
||||
log_note "Making sure we have Direct I/O write checksum verifies with ZPool"
|
||||
check_dio_write_chksum_verify_failures "$TESTPOOL" "raidz" 1
|
||||
check_dio_chksum_verify_failures "$TESTPOOL" "raidz" 1 "wr"
|
||||
done
|
||||
|
||||
log_must rm -f "$mntpnt/direct-write.iso"
|
||||
|
||||
Reference in New Issue
Block a user