From b0eac56a4d69c6f2ae7cb1937cf59e05c52e3a60 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 7 Feb 2017 20:12:53 -0500 Subject: [PATCH] Move ziltest.sh to the ZTS framework The ziltest.sh script is a test case designed to verify the correct functioning of the ZIL. For historical reasons it was never added to the test suite and was always run independantly. This change rectifies that. The existing ziltest.sh has been translated in to `slog_015_pos.ksh` and added to the existing slog test cases. Reviewed-by: Don Brady Reviewed-by: Chunwei Chen Reviewed-by: George Melikov Signed-off-by: Brian Behlendorf Closes #5758 --- TEST | 8 - scripts/Makefile.am | 1 - scripts/ziltest.sh | 305 ------------------ tests/runfiles/linux.run | 2 +- .../tests/functional/slog/Makefile.am | 3 +- .../zfs-tests/tests/functional/slog/slog.cfg | 4 +- .../tests/functional/slog/slog.kshlib | 1 + .../tests/functional/slog/slog_015_pos.ksh | 201 ++++++++++++ 8 files changed, 207 insertions(+), 318 deletions(-) delete mode 100755 scripts/ziltest.sh create mode 100755 tests/zfs-tests/tests/functional/slog/slog_015_pos.ksh diff --git a/TEST b/TEST index 85440271b..db70180f6 100644 --- a/TEST +++ b/TEST @@ -13,10 +13,6 @@ #TEST_ZTEST_DIR="/var/tmp/" #TEST_ZTEST_OPTIONS="-V" -### ziltest -#TEST_ZILTEST_SKIP="yes" -#TEST_ZILTEST_OPTIONS="" - ### zconfig #TEST_ZCONFIG_SKIP="yes" TEST_ZCONFIG_OPTIONS="-c -s10" @@ -79,12 +75,8 @@ Amazon*) CentOS-7*) # ZFS enabled xfstests fails to build TEST_XFSTESTS_SKIP="yes" - # Sporadic VERIFY(!zilog_is_dirty(zilog)) failed - TEST_ZILTEST_SKIP="yes" ;; CentOS-6*) - # Sporadic VERIFY(!zilog_is_dirty(zilog)) failed - TEST_ZILTEST_SKIP="yes" ;; Debian*) ;; diff --git a/scripts/Makefile.am b/scripts/Makefile.am index b55600926..c8218db11 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -6,7 +6,6 @@ pkgdatadir = $(datadir)/@PACKAGE@ dist_pkgdata_SCRIPTS = \ $(top_builddir)/scripts/common.sh \ $(top_srcdir)/scripts/zconfig.sh \ - $(top_srcdir)/scripts/ziltest.sh \ $(top_srcdir)/scripts/zimport.sh \ $(top_srcdir)/scripts/zfs.sh \ $(top_srcdir)/scripts/zfs-tests.sh \ diff --git a/scripts/ziltest.sh b/scripts/ziltest.sh deleted file mode 100755 index bb816293f..000000000 --- a/scripts/ziltest.sh +++ /dev/null @@ -1,305 +0,0 @@ -#!/bin/bash -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License, Version 1.0 only -# (the "License"). You may not use this file except in compliance -# with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# -# Copyright 2007 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# -# Linux version -# -# To run just type ziltest.sh -# -# - creates a 200MB pool in /var/tmp/ -# - prints information on: -# working set files -# ZIL records written -# ZIL block usage -# verification results -# - returns status of 0 on success -# -########################################################################## -# -# Here's how it all works: -# -# The general idea is to build up -# an intent log from a bunch of diverse user commands -# without actually committing them to the file system. -# Then copy the file system, replay the intent -# log and compare the file system and the copy. -# -# To enable this automated testing of the intent log -# requires some but minimal support from the file system. -# In particular, a "freeze" command is required to flush -# the in-flight transactions; to stop the actual -# committing of transactions; and to ensure no deltas are -# discarded. All deltas past a freeze point are kept for -# replay and comparison later. Here is the flow: -# -# create an empty file system (FS) -# freeze FS -# run various user commands that create files, directories and ACLs -# copy FS to temporary location (COPY) -# unmount filesystem -# -# remount FS -# compare FS against the COPY -# - -PATH=/usr/bin -PATH=$PATH:/usr/sbin -PATH=$PATH:/bin -PATH=$PATH:/sbin -export PATH - -# ==================================================================== -# SETUP -# ==================================================================== -CMD=$(basename "$0") -POOL=ziltestpool.$$ -DEVSIZE=${DEVSIZE-200m} -POOLDIR=/var/tmp -POOLFILE=$POOLDIR/ziltest_poolfile.$$ -SLOGFILE=$POOLDIR/ziltest_slog.$$ -FS=$POOL/fs -ROOT=/$FS -COPY=/var/tmp/${POOL} -KEEP=no - -cleanup() -{ - zfs destroy -rf $FS - echo "$CMD: pool I/O summary & status:" - echo "----------------------------------------------------" - zpool iostat $POOL - echo - zpool status $POOL - echo "----------------------------------------------------" - echo - zpool destroy -f $POOL - rm -rf $COPY - rm $POOLFILE $SLOGFILE -} - -bail() -{ - test $KEEP = no && cleanup - echo "$1" - exit 1 -} - -test $# -eq 0 || bail "usage: $CMD" - -# ==================================================================== -# PREP -# -# Create a pool using a file based vdev -# Create a destination for runtime copy of FS -# Freeze transaction syncing in the pool -# ==================================================================== -truncate -s "$DEVSIZE" $POOLFILE || bail "can't make $POOLFILE" -truncate -s "$DEVSIZE" $SLOGFILE || bail "can't make $SLOGFILE" -zpool create $POOL $POOLFILE log $SLOGFILE || bail "can't create pool -$POOL" -zpool list $POOL - -zfs set compression=on $POOL || bail "can't enable compression on $POOL" -zfs create $FS || bail "can't create $FS" -mkdir -p $COPY || bail "can't create $COPY" - -# -# This dd command works around an issue where ZIL records aren't created -# after freezing the pool unless a ZIL header already exists. Create a file -# synchronously to force ZFS to write one out. -# -dd if=/dev/zero of=$ROOT/sync conv=fdatasync,fsync bs=1 count=1 2> /dev/null - -zpool freeze $POOL || bail "can't freeze $POOL" - -# ==================================================================== -# TESTS -# -# Add operations here that will add commit records to the ZIL -# -# Use $ROOT for all file name prefixes -# ==================================================================== - -# -# TX_CREATE -# -touch $ROOT/a - -# -# TX_RENAME -# -mv $ROOT/a $ROOT/b - -# -# TX_SYMLINK -# -touch $ROOT/c -ln -s $ROOT/c $ROOT/d - -# -# TX_LINK -# -touch $ROOT/e -ln $ROOT/e $ROOT/f - -# -# TX_MKDIR -# -mkdir $ROOT/dir_to_delete - -# -# TX_RMDIR -# -rmdir $ROOT/dir_to_delete - -# -# Create a simple validation payload -# -PAYLOAD=$(modinfo -F filename zfs) -cp "$PAYLOAD" "$ROOT/payload" -CHECKSUM_BEFORE=$(sha256sum -b "$PAYLOAD") - -# -# TX_WRITE (small file with ordering) -# -if is_linux; then - cp /proc/self/mounts $ROOT/small_file -else - cp /etc/mtab $ROOT/small_file -fi -cp /etc/profile $ROOT/small_file - -# -# TX_CREATE, TX_MKDIR, TX_REMOVE, TX_RMDIR -# -cp -R /usr/share/dict $ROOT -rm -rf $ROOT/dict - -# -# TX_SETATTR -# -touch $ROOT/setattr -chmod 567 $ROOT/setattr -chgrp root $ROOT/setattr -touch -cm -t 201311271200 $ROOT/setattr - -# -# TX_TRUNCATE (to zero) -# -cp /etc/services $ROOT/truncated_file -> $ROOT/truncated_file - -# -# Write to an open but removed file -# -(sleep 2; date) > $ROOT/date & sleep 1; rm $ROOT/date; wait - -# -# TX_WRITE (large file) -# -dd if=/usr/share/lib/termcap of=$ROOT/large bs=128k oflag=sync 2> /dev/null - -# -# Write zeroes, which compresss to holes, in the middle of a file -# -dd if=$POOLFILE of=$ROOT/holes.1 bs=128k count=8 2> /dev/null -dd if=/dev/zero of=$ROOT/holes.1 bs=128k count=2 2> /dev/null - -dd if=$POOLFILE of=$ROOT/holes.2 bs=128k count=8 2> /dev/null -dd if=/dev/zero of=$ROOT/holes.2 bs=128k count=2 oseek=2 2> /dev/null - -dd if=$POOLFILE of=$ROOT/holes.3 bs=128k count=8 2> /dev/null -dd if=/dev/zero of=$ROOT/holes.3 bs=128k count=2 oseek=2 conv=notrunc 2> /dev/null - -# -# TX_MKXATTR -# -mkdir $ROOT/xattr.dir -attr -qs fileattr -V HelloWorld $ROOT/xattr.dir -attr -qs tmpattr -V HelloWorld $ROOT/xattr.dir -attr -qr tmpattr $ROOT/xattr.dir - -touch $ROOT/xattr.file -attr -qs fileattr -V HelloWorld $ROOT/xattr.file -attr -qs tmpattr -V HelloWorld $ROOT/xattr.file -attr -qr tmpattr $ROOT/xattr.file -rm $ROOT/xattr.file - - -# ==================================================================== -# REPLAY -# ==================================================================== - -KEEP=yes # keep stuff around if we fail, so we can look at it - -cd $ROOT -find . | cpio -pdmu --quiet $COPY -echo -cd / - -zfs unmount $FS || bail "can't unmount $FS" - -echo "$CMD: transactions to replay:" -echo "----------------------------------------------------" -zdb -ivv $FS || bail "can't run zdb on $POOL" -echo "----------------------------------------------------" -echo - -# -# Export and reimport the pool to unfreeze it and claim log blocks. -# It has to be import -f because we can't write a frozen pool's labels! -# -zpool export $POOL || bail "can't export $POOL" -zpool import -f -d $POOLDIR $POOL || bail "can't import $POOL" - -# ==================================================================== -# VERIFY -# ==================================================================== - -echo "$CMD: current block usage:" -echo "----------------------------------------------------" -zdb -bcv $POOL || bail "blocks were leaked!" -echo "----------------------------------------------------" -echo - -echo "$CMD: Copy of xattrs:" -echo "----------------------------------------------------" -attr -l $ROOT/xattr.dir || bail "can't list xattrs" -echo "----------------------------------------------------" -echo - -echo "$CMD: Results of workingset diff:" -echo "----------------------------------------------------" -diff -r $ROOT $COPY > /dev/null || diff -r $ROOT $COPY || bail "replay diffs!" - -echo "$CHECKSUM_BEFORE" | sha256sum -c || bail "payload checksums don't match" -echo "payload checksum matched" -echo "----------------------------------------------------" -echo - -cleanup - -exit 0 diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index d9d5d58d6..079bc1a81 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -585,7 +585,7 @@ tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos', [tests/functional/slog] tests = ['slog_001_pos', 'slog_002_pos', 'slog_003_pos', 'slog_004_pos', 'slog_005_pos', 'slog_006_pos', 'slog_007_pos', 'slog_008_neg', - 'slog_009_neg', 'slog_010_neg', 'slog_011_neg'] + 'slog_009_neg', 'slog_010_neg', 'slog_011_neg', 'slog_015_pos'] # DISABLED: # clone_001_pos - https://github.com/zfsonlinux/zfs/issues/3484 diff --git a/tests/zfs-tests/tests/functional/slog/Makefile.am b/tests/zfs-tests/tests/functional/slog/Makefile.am index c37047280..23c07b297 100644 --- a/tests/zfs-tests/tests/functional/slog/Makefile.am +++ b/tests/zfs-tests/tests/functional/slog/Makefile.am @@ -17,4 +17,5 @@ dist_pkgdata_SCRIPTS = \ slog_011_neg.ksh \ slog_012_neg.ksh \ slog_013_pos.ksh \ - slog_014_pos.ksh + slog_014_pos.ksh \ + slog_015_pos.ksh diff --git a/tests/zfs-tests/tests/functional/slog/slog.cfg b/tests/zfs-tests/tests/functional/slog/slog.cfg index 28cce1aae..5ac9a46c5 100644 --- a/tests/zfs-tests/tests/functional/slog/slog.cfg +++ b/tests/zfs-tests/tests/functional/slog/slog.cfg @@ -30,8 +30,8 @@ export SIZE=64M -export VDIR=/disk-slog -export VDIR2=/disk2-slog +export VDIR=$TEST_BASE_DIR/disk-slog +export VDIR2=$TEST_BASE_DIR/disk2-slog export VDEV="$VDIR/a $VDIR/b $VDIR/c" export SDEV="$VDIR/d" diff --git a/tests/zfs-tests/tests/functional/slog/slog.kshlib b/tests/zfs-tests/tests/functional/slog/slog.kshlib index 124bfe6d2..0aca19aa2 100644 --- a/tests/zfs-tests/tests/functional/slog/slog.kshlib +++ b/tests/zfs-tests/tests/functional/slog/slog.kshlib @@ -39,6 +39,7 @@ function cleanup if datasetexists $TESTPOOL2 ; then log_must $ZPOOL destroy -f $TESTPOOL2 fi + rm -rf $TESTDIR } # diff --git a/tests/zfs-tests/tests/functional/slog/slog_015_pos.ksh b/tests/zfs-tests/tests/functional/slog/slog_015_pos.ksh new file mode 100755 index 000000000..71833e7c1 --- /dev/null +++ b/tests/zfs-tests/tests/functional/slog/slog_015_pos.ksh @@ -0,0 +1,201 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +. $STF_SUITE/tests/functional/slog/slog.kshlib + +# +# DESCRIPTION: +# Verify slogs are replayed correctly. This test is a direct +# adaptation of the ziltest.sh script for the ZFS Test Suite. +# +# The general idea is to build up an intent log from a bunch of +# diverse user commands without actually committing them to the +# file system. Then copy the file system, replay the intent +# log and compare the file system and the copy. +# +# To enable this automated testing of the intent log some minimal +# support is required of the file system. In particular, a +# "freeze" command is required to flush the in-flight transactions; +# to stop the actual committing of transactions; and to ensure no +# deltas are discarded. All deltas past a freeze point are kept +# for replay and comparison later. Here is the flow: +# +# STRATEGY: +# 1. Create an empty file system (TESTFS) +# 2. Freeze TESTFS +# 3. Run various user commands that create files, directories and ACLs +# 4. Copy TESTFS to temporary location (TESTDIR) +# 5. Unmount filesystem +# +# 6. Remount TESTFS +# 7. Compare TESTFS against the TESTDIR copy +# + +verify_runnable "global" + +log_assert "Replay of intent log succeeds." +log_onexit cleanup + +# +# 1. Create an empty file system (TESTFS) +# +log_must $ZPOOL create $TESTPOOL $VDEV log mirror $LDEV +log_must $ZFS set compression=on $TESTPOOL +log_must $ZFS create $TESTPOOL/$TESTFS +log_must $MKDIR -p $TESTDIR + +# +# This dd command works around an issue where ZIL records aren't created +# after freezing the pool unless a ZIL header already exists. Create a file +# synchronously to force ZFS to write one out. +# +log_must $GNUDD if=/dev/zero of=/$TESTPOOL/$TESTFS/sync \ + conv=fdatasync,fsync bs=1 count=1 + +# +# 2. Freeze TESTFS +# +log_must $ZPOOL freeze $TESTPOOL + +# +# 3. Run various user commands that create files, directories and ACLs +# + +# TX_CREATE +log_must touch /$TESTPOOL/$TESTFS/a + +# TX_RENAME +log_must mv /$TESTPOOL/$TESTFS/a /$TESTPOOL/$TESTFS/b + +# TX_SYMLINK +log_must touch /$TESTPOOL/$TESTFS/c +log_must ln -s /$TESTPOOL/$TESTFS/c /$TESTPOOL/$TESTFS/d + +# TX_LINK +log_must touch /$TESTPOOL/$TESTFS/e +log_must ln /$TESTPOOL/$TESTFS/e /$TESTPOOL/$TESTFS/f + +# TX_MKDIR +log_must mkdir /$TESTPOOL/$TESTFS/dir_to_delete + +# TX_RMDIR +log_must rmdir /$TESTPOOL/$TESTFS/dir_to_delete + +# Create a simple validation payload +log_must $DD if=/dev/urandom of=/$TESTPOOL/$TESTFS/payload bs=1k count=8 +CHECKSUM_BEFORE=$(sha256sum -b /$TESTPOOL/$TESTFS/payload) + +# TX_WRITE (small file with ordering) +log_must $MKFILE 1k /$TESTPOOL/$TESTFS/small_file +log_must $MKFILE 512b /$TESTPOOL/$TESTFS/small_file + +# TX_CREATE, TX_MKDIR, TX_REMOVE, TX_RMDIR +log_must cp -R /usr/share/dict /$TESTPOOL/$TESTFS +log_must rm -rf /$TESTPOOL/$TESTFS/dict + +# TX_SETATTR +log_must touch /$TESTPOOL/$TESTFS/setattr +log_must chmod 567 /$TESTPOOL/$TESTFS/setattr +log_must chgrp root /$TESTPOOL/$TESTFS/setattr +log_must touch -cm -t 201311271200 /$TESTPOOL/$TESTFS/setattr + +# TX_TRUNCATE (to zero) +log_must $MKFILE 4k /$TESTPOOL/$TESTFS/truncated_file +log_must $TRUNCATE -s 0 /$TESTPOOL/$TESTFS/truncated_file + +# TX_WRITE (large file) +log_must $DD if=/dev/urandom of=/$TESTPOOL/$TESTFS/large \ + bs=128k count=64 oflag=sync + +# Write zeroes, which compresss to holes, in the middle of a file +log_must $DD if=/dev/urandom of=/$TESTPOOL/$TESTFS/holes.1 bs=128k count=8 +log_must $DD if=/dev/zero of=/$TESTPOOL/$TESTFS/holes.1 bs=128k count=2 + +log_must $DD if=/dev/urandom of=/$TESTPOOL/$TESTFS/holes.2 bs=128k count=8 +log_must $DD if=/dev/zero of=/$TESTPOOL/$TESTFS/holes.2 bs=128k count=2 seek=2 + +log_must $DD if=/dev/urandom of=/$TESTPOOL/$TESTFS/holes.3 bs=128k count=8 +log_must $DD if=/dev/zero of=/$TESTPOOL/$TESTFS/holes.3 bs=128k count=2 \ + seek=2 conv=notrunc + +# TX_MKXATTR +log_must $MKDIR /$TESTPOOL/$TESTFS/xattr.dir +log_must attr -qs fileattr -V HelloWorld /$TESTPOOL/$TESTFS/xattr.dir +log_must attr -qs tmpattr -V HelloWorld /$TESTPOOL/$TESTFS/xattr.dir +log_must attr -qr tmpattr /$TESTPOOL/$TESTFS/xattr.dir + +log_must touch /$TESTPOOL/$TESTFS/xattr.file +log_must attr -qs fileattr -V HelloWorld /$TESTPOOL/$TESTFS/xattr.file +log_must attr -qs tmpattr -V HelloWorld /$TESTPOOL/$TESTFS/xattr.file +log_must attr -qr tmpattr /$TESTPOOL/$TESTFS/xattr.file + +# +# 4. Copy TESTFS to temporary location (TESTDIR) +# +log_must cp -a /$TESTPOOL/$TESTFS/* $TESTDIR + +# +# 5. Unmount filesystem and export the pool +# +# At this stage TESTFS is empty again and unfrozen, and the +# intent log contains a complete set of deltas to replay it. +# +log_must $ZFS unmount /$TESTPOOL/$TESTFS + +log_note "Verify transactions to replay:" +log_must $ZDB -iv $TESTPOOL/$TESTFS + +log_must $ZPOOL export $TESTPOOL + +# +# 6. Remount TESTFS +# +# Import the pool to unfreeze it and claim log blocks. It has to be +# `zpool import -f` because we can't write a frozen pool's labels! +# +log_must $ZPOOL import -f -d $VDIR $TESTPOOL + +# +# 7. Compare TESTFS against the TESTDIR copy +# +log_note "Verify current block usage:" +log_must $ZDB -bcv $TESTPOOL + +log_note "Verify copy of xattrs:" +log_must attr -l /$TESTPOOL/$TESTFS/xattr.dir +log_must attr -l /$TESTPOOL/$TESTFS/xattr.file + +log_note "Verify working set diff:" +log_must diff -r /$TESTPOOL/$TESTFS $TESTDIR >/dev/null || \ + diff -r /$TESTPOOL/$TESTFS $TESTDIR + +log_note "Verify file checksum:" +log_note "$CHECKSUM_BEFORE" +log_must echo "$CHECKSUM_BEFORE" | sha256sum -c + +log_pass "Replay of intent log succeeds."