mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-22 18:40:43 +03:00
OpenZFS 9438 - Holes can lose birth time info if a block has a mix of birth times
As reported by https://github.com/zfsonlinux/zfs/issues/4996, there is yet another hole birth issue. In this one, if a block is entirely holes, but the birth times are not all the same, we lose that information by creating one hole with the current txg as its birth time. The ZoL PR's fix approach is incorrect. Ultimately, the problem here is that when you truncate and write a file in the same transaction group, the dbuf for the indirect block will be zeroed out to deal with the truncation, and then written for the write. During this process, we will lose hole birth time information for any holes in the range. In the case where a dnode is being freed, we need to determine whether the block should be converted to a higher-level hole in the zio pipeline, and if so do it when the dnode is being synced out. Porting Notes: * The DMU_OBJECT_END change in zfs_znode.c was already applied. * Added test cases from #5675 provided by @rincebrain for hole_birth issues. These test cases should be pushed upstream to OpenZFS. * Updated mk_files which is used by several rsend tests so the files created are a little more interesting and may contain holes. Authored by: Paul Dagnelie <pcd@delphix.com> Reviewed by: Matt Ahrens <matt@delphix.com> Reviewed by: George Wilson <george.wilson@delphix.com> Approved by: Robert Mustacchi <rm@joyent.com> Ported-by: Brian Behlendorf <behlendorf1@llnl.gov> OpenZFS-issue: https://www.illumos.org/issues/9438 OpenZFS-commit: https://github.com/openzfs/openzfs/commit/738e2a3c External-issue: DLPX-46861 Closes #7746
This commit is contained in:
Paul Dagnelie
committed by
Brian Behlendorf
Brian Behlendorf
parent
b719768e35
commit
21d48b5eac
@@ -743,7 +743,7 @@ tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
|
||||
'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
|
||||
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
|
||||
'send-c_recv_dedup', 'send_encrypted_files', 'send_encrypted_heirarchy',
|
||||
'send_freeobjects', 'send_realloc_dnode_size']
|
||||
'send_freeobjects', 'send_realloc_dnode_size', 'send_hole_birth']
|
||||
tags = ['functional', 'rsend']
|
||||
|
||||
[tests/functional/scrub_mirror]
|
||||
|
||||
@@ -39,7 +39,8 @@ dist_pkgdata_SCRIPTS = \
|
||||
send-c_zstreamdump.ksh \
|
||||
send-cpL_varied_recsize.ksh \
|
||||
send_freeobjects.ksh \
|
||||
send_realloc_dnode_size.ksh
|
||||
send_realloc_dnode_size.ksh \
|
||||
send_hole_birth.ksh
|
||||
|
||||
dist_pkgdata_DATA = \
|
||||
rsend.cfg \
|
||||
|
||||
@@ -153,6 +153,20 @@ function cleanup_pools
|
||||
destroy_pool $POOL3
|
||||
}
|
||||
|
||||
function cmp_md5s {
|
||||
typeset file1=$1
|
||||
typeset file2=$2
|
||||
|
||||
eval md5sum $file1 | awk '{ print $1 }' > $BACKDIR/md5_file1
|
||||
eval md5sum $file2 | awk '{ print $1 }' > $BACKDIR/md5_file2
|
||||
diff $BACKDIR/md5_file1 $BACKDIR/md5_file2
|
||||
typeset -i ret=$?
|
||||
|
||||
rm -f $BACKDIR/md5_file1 $BACKDIR/md5_file2
|
||||
|
||||
return $ret
|
||||
}
|
||||
|
||||
#
|
||||
# Detect if the given two filesystems have same sub-datasets
|
||||
#
|
||||
@@ -388,13 +402,36 @@ function mk_files
|
||||
maxsize=$2
|
||||
file_id_offset=$3
|
||||
fs=$4
|
||||
bs=512
|
||||
|
||||
for ((i=0; i<$nfiles; i=i+1)); do
|
||||
dd if=/dev/urandom \
|
||||
of=/$fs/file-$maxsize-$((i+$file_id_offset)) \
|
||||
bs=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1)) \
|
||||
count=1 >/dev/null 2>&1 || log_fail \
|
||||
"Failed to create /$fs/file-$maxsize-$((i+$file_id_offset))"
|
||||
file_name="/$fs/file-$maxsize-$((i+$file_id_offset))"
|
||||
file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
|
||||
|
||||
#
|
||||
# Create an interesting mix of files which contain both
|
||||
# data blocks and holes for more realistic test coverage.
|
||||
# Half the files are created as sparse then partially filled,
|
||||
# the other half is dense then a hole is punched in the file.
|
||||
#
|
||||
if [ $((RANDOM % 2)) -eq 0 ]; then
|
||||
truncate -s $file_size $file_name || \
|
||||
log_fail "Failed to create $file_name"
|
||||
dd if=/dev/urandom of=$file_name \
|
||||
bs=$bs count=$(($file_size / 2 / $bs)) \
|
||||
seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
|
||||
conv=notrunc >/dev/null 2>&1 || \
|
||||
log_fail "Failed to create $file_name"
|
||||
else
|
||||
dd if=/dev/urandom of=$file_name \
|
||||
bs=$file_size count=1 >/dev/null 2>&1 || \
|
||||
log_fail "Failed to create $file_name"
|
||||
dd if=/dev/zero of=$file_name \
|
||||
bs=$bs count=$(($file_size / 2 / $bs)) \
|
||||
seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
|
||||
conv=notrunc >/dev/null 2>&1 || \
|
||||
log_fail "Failed to create $file_name"
|
||||
fi
|
||||
done
|
||||
echo Created $nfiles files of random sizes up to $maxsize bytes
|
||||
}
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
#!/bin/ksh -p
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# The contents of this file are subject to the terms of the
|
||||
# Common Development and Distribution License (the "License").
|
||||
# You may not use this file except in compliance with the License.
|
||||
#
|
||||
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
# or http://www.opensolaris.org/os/licensing.
|
||||
# See the License for the specific language governing permissions
|
||||
# and limitations under the License.
|
||||
#
|
||||
# When distributing Covered Code, include this CDDL HEADER in each
|
||||
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
# If applicable, add the following below this CDDL HEADER, with the
|
||||
# fields enclosed by brackets "[]" replaced with your own identifying
|
||||
# information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright 2008 Sun Microsystems, Inc. All rights reserved.
|
||||
# Use is subject to license terms.
|
||||
#
|
||||
|
||||
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Verify send streams which contain holes.
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create an initial file for the full send and snapshot.
|
||||
# 2. Permute the file with holes and snapshot.
|
||||
# 3. Send the full and incremental snapshots to a new pool.
|
||||
# 4. Verify the contents of the files match.
|
||||
#
|
||||
|
||||
sendpool=$POOL
|
||||
sendfs=$sendpool/sendfs
|
||||
|
||||
recvpool=$POOL2
|
||||
recvfs=$recvpool/recvfs
|
||||
|
||||
verify_runnable "both"
|
||||
|
||||
log_assert "Test hole_birth"
|
||||
log_onexit cleanup
|
||||
|
||||
function cleanup
|
||||
{
|
||||
cleanup_pool $sendpool
|
||||
cleanup_pool $recvpool
|
||||
set_tunable64 send_holes_without_birth_time 1
|
||||
}
|
||||
|
||||
function send_and_verify
|
||||
{
|
||||
log_must eval "zfs send $sendfs@snap1 > $BACKDIR/pool-snap1"
|
||||
log_must eval "zfs receive -F $recvfs < $BACKDIR/pool-snap1"
|
||||
|
||||
log_must eval "zfs send -i $sendfs@snap1 $sendfs@snap2 " \
|
||||
">$BACKDIR/pool-snap1-snap2"
|
||||
log_must eval "zfs receive $recvfs < $BACKDIR/pool-snap1-snap2"
|
||||
|
||||
log_must cmp_md5s /$sendfs/file1 /$recvfs/file1
|
||||
}
|
||||
|
||||
# By default sending hole_birth times is disabled. This functionality needs
|
||||
# to be re-enabled for this test case to verify correctness. Once we're
|
||||
# comfortable that all hole_birth bugs has been resolved this behavior may
|
||||
# be re-enabled by default.
|
||||
log_must set_tunable64 send_holes_without_birth_time 0
|
||||
|
||||
# Incremental send truncating the file and adding new data.
|
||||
log_must zfs create -o recordsize=4k $sendfs
|
||||
|
||||
log_must truncate -s 1G /$sendfs/file1
|
||||
log_must dd if=/dev/urandom of=/$sendfs/file1 bs=4k count=11264 seek=1152
|
||||
log_must zfs snapshot $sendfs@snap1
|
||||
|
||||
log_must truncate -s 4194304 /$sendfs/file1
|
||||
log_must dd if=/dev/urandom of=/$sendfs/file1 bs=4k count=152 seek=384 \
|
||||
conv=notrunc
|
||||
log_must dd if=/dev/urandom of=/$sendfs/file1 bs=4k count=10 seek=1408 \
|
||||
conv=notrunc
|
||||
log_must zfs snapshot $sendfs@snap2
|
||||
|
||||
send_and_verify
|
||||
log_must cleanup_pool $sendpool
|
||||
log_must cleanup_pool $recvpool
|
||||
|
||||
# Incremental send appending a hole and data.
|
||||
log_must zfs create -o recordsize=512 $sendfs
|
||||
|
||||
log_must dd if=/dev/urandom of=/$sendfs/file1 bs=128k count=1 seek=1
|
||||
log_must zfs snapshot $sendfs@snap1
|
||||
|
||||
log_must dd if=/dev/urandom of=/$sendfs/file1 bs=128k count=1
|
||||
log_must dd if=/dev/urandom of=/$sendfs/file1 bs=128k count=1 seek=3
|
||||
log_must zfs snapshot $sendfs@snap2
|
||||
|
||||
send_and_verify
|
||||
log_must cleanup_pool $sendpool
|
||||
log_must cleanup_pool $recvpool
|
||||
|
||||
# Incremental send truncating the file and adding new data.
|
||||
log_must zfs create -o recordsize=512 $sendfs
|
||||
|
||||
log_must truncate -s 300M /$sendfs/file1
|
||||
log_must dd if=/dev/urandom of=/$sendfs/file1 bs=512 count=128k conv=notrunc
|
||||
log_must zfs snapshot $sendfs@snap1
|
||||
|
||||
log_must truncate -s 10M /$sendfs/file1
|
||||
log_must dd if=/dev/urandom of=/$sendfs/file1 bs=512 count=1 seek=96k \
|
||||
conv=notrunc
|
||||
log_must zfs snapshot $sendfs@snap2
|
||||
|
||||
send_and_verify
|
||||
|
||||
log_pass "Test hole_birth"
|
||||
Reference in New Issue
Block a user