c/mirror_zfs
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_zfs.git synced 2026-05-22 02:27:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Implement Redacted Send/Receive

Browse Source 
Redacted send/receive allows users to send subsets of their data to 
a target system. One possible use case for this feature is to not 
transmit sensitive information to a data warehousing, test/dev, or 
analytics environment. Another is to save space by not replicating 
unimportant data within a given dataset, for example in backup tools 
like zrepl.

Redacted send/receive is a three-stage process. First, a clone (or 
clones) is made of the snapshot to be sent to the target. In this 
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction 
snapshot" (or snapshots). Second, the new zfs redact command is used 
to create a redaction bookmark. The redaction bookmark stores the 
list of blocks in a snapshot that were modified by the redaction 
snapshot(s). Finally, the redaction bookmark is passed as a parameter 
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive 
or unwanted information, and those blocks are not included in the send 
stream.  When sending from the redaction bookmark, the blocks it 
contains are considered as candidate blocks in addition to those 
blocks in the destination snapshot that were modified since the 
creation_txg of the redaction bookmark.  This step is necessary to 
allow the target to rehydrate data in the case where some blocks are 
accidentally or unnecessarily modified in the redaction snapshot.

The changes to bookmarks to enable fast space estimation involve 
adding deadlists to bookmarks. There is also logic to manage the 
life cycles of these deadlists.

The new size estimation process operates in cases where previously 
an accurate estimate could not be provided. In those cases, a send 
is performed where no data blocks are read, reducing the runtime 
significantly and providing a byte-accurate size estimate.

Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
This commit is contained in:
Paul Dagnelie
2019-06-19 09:48:13 -07:00
committed by Brian Behlendorf
parent c1b5801bb5
commit 30af21b025
103 changed files with 11513 additions and 2668 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/zfs-tests/cmd/Makefile.am
+3 -1
View File
@@ -8,6 +8,7 @@ SUBDIRS = \
file_check \
file_trunc \
file_write \
get_diff \
largest_file \
libzfs_input_check \
mkbusy \
@@ -24,4 +25,5 @@ SUBDIRS = \
rename_dir \
rm_lnkcnt_zero_file \
threadsappend \
xattrtest
xattrtest \
stride_dd
tests/zfs-tests/cmd/get_diff/Makefile.am
+6
View File
@@ -0,0 +1,6 @@
include $(top_srcdir)/config/Rules.am
pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/bin
pkgexec_PROGRAMS = get_diff
get_diff_SOURCES = get_diff.c
tests/zfs-tests/cmd/get_diff/get_diff.c
+109
View File
@@ -0,0 +1,109 @@
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/
/*
* Copyright (c) 2018 by Delphix. All rights reserved.
*/
#include <stdio.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
static void
usage(char *msg, int exit_value)
{
(void) fprintf(stderr, "get_diff file redacted_file\n");
(void) fprintf(stderr, "%s\n", msg);
exit(exit_value);
}
/*
* This utility compares two files, an original and its redacted counterpart
* (in that order). It compares the files 512 bytes at a time, printing out
* any ranges (as offset and length) where the redacted file does not match
* the original. This output is used to verify that the expected ranges of
* a redacted file do not contain the original data.
*/
int
main(int argc, char *argv[])
{
off_t diff_off = 0, diff_len = 0, off = 0;
int fd1, fd2;
char *fname1, *fname2;
char buf1[DEV_BSIZE], buf2[DEV_BSIZE];
ssize_t bytes;
if (argc != 3)
usage("Incorrect number of arguments.", 1);
if ((fname1 = argv[1]) == NULL)
usage("Filename missing.", 1);
if ((fd1 = open(fname1, O_LARGEFILE | O_RDONLY)) < 0) {
perror("open1 failed");
exit(1);
}
if ((fname2 = argv[2]) == NULL)
usage("Redacted filename missing.", 1);
if ((fd2 = open(fname2, O_LARGEFILE | O_RDONLY)) < 0) {
perror("open2 failed");
exit(1);
}
while ((bytes = pread(fd1, buf1, DEV_BSIZE, off)) > 0) {
if (pread(fd2, buf2, DEV_BSIZE, off) < 0) {
if (errno == EIO) {
/*
* A read in a redacted section of a file will
* fail with EIO. If we get EIO, continue on
* but ensure that a comparison of buf1 and
* buf2 will fail, indicating a redacted block.
*/
buf2[0] = ~buf1[0];
} else {
perror("pread failed");
exit(1);
}
}
if (memcmp(buf1, buf2, bytes) == 0) {
if (diff_len != 0) {
(void) fprintf(stdout, "%lld,%lld\n",
(long long)diff_off, (long long)diff_len);
assert(off == diff_off + diff_len);
diff_len = 0;
}
diff_off = 0;
} else {
if (diff_len == 0)
diff_off = off;
assert(off == diff_off + diff_len);
diff_len += bytes;
}
off += bytes;
}
if (diff_len != 0 && diff_len != 0) {
(void) fprintf(stdout, "%lld,%lld\n", (long long)diff_off,
(long long)diff_len);
}
(void) close(fd1);
(void) close(fd2);
return (0);
}
tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c
+36
View File
@@ -690,6 +690,34 @@ zfs_destroy(const char *dataset)
return (err == 0 ? 0 : errno);
}
static void
test_redact(const char *snapshot1, const char *snapshot2)
{
nvlist_t *required = fnvlist_alloc();
nvlist_t *snapnv = fnvlist_alloc();
char bookmark[MAXNAMELEN + 32];
fnvlist_add_string(required, "bookname", "testbookmark");
fnvlist_add_boolean(snapnv, snapshot2);
fnvlist_add_nvlist(required, "snapnv", snapnv);
IOC_INPUT_TEST(ZFS_IOC_REDACT, snapshot1, required, NULL, 0);
nvlist_free(snapnv);
nvlist_free(required);
strncpy(bookmark, snapshot1, sizeof (bookmark) - 1);
*strchr(bookmark, '@') = '\0';
strncat(bookmark, "#testbookmark", sizeof (bookmark));
zfs_destroy(bookmark);
}
static void
test_get_bookmark_props(const char *bookmark)
{
IOC_INPUT_TEST(ZFS_IOC_GET_BOOKMARK_PROPS, bookmark, NULL, NULL, 0);
}
static void
zfs_ioc_input_tests(const char *pool)
{
@@ -700,6 +728,7 @@ zfs_ioc_input_tests(const char *pool)
char bookmark[ZFS_MAX_DATASET_NAME_LEN + 32];
char backup[ZFS_MAX_DATASET_NAME_LEN];
char clone[ZFS_MAX_DATASET_NAME_LEN];
char clonesnap[ZFS_MAX_DATASET_NAME_LEN + 32];
int tmpfd, err;
/*
@@ -710,6 +739,7 @@ zfs_ioc_input_tests(const char *pool)
(void) snprintf(snapshot, sizeof (snapshot), "%s@snapshot", dataset);
(void) snprintf(bookmark, sizeof (bookmark), "%s#bookmark", dataset);
(void) snprintf(clone, sizeof (clone), "%s/test-fs-clone", pool);
(void) snprintf(clonesnap, sizeof (clonesnap), "%s@snap", clone);
(void) snprintf(backup, sizeof (backup), "%s/backup", pool);
err = lzc_create(dataset, DMU_OST_ZFS, NULL, NULL, 0);
@@ -747,6 +777,7 @@ zfs_ioc_input_tests(const char *pool)
test_bookmark(pool, snapshot, bookmark);
test_get_bookmarks(dataset);
test_get_bookmark_props(bookmark);
test_destroy_bookmarks(pool, bookmark);
test_hold(pool, snapshot);
@@ -754,6 +785,9 @@ zfs_ioc_input_tests(const char *pool)
test_release(pool, snapshot);
test_clone(snapshot, clone);
test_snapshot(pool, clonesnap);
test_redact(snapshot, clonesnap);
zfs_destroy(clonesnap);
zfs_destroy(clone);
test_rollback(dataset, snapshot);
@@ -909,6 +943,8 @@ validate_ioc_values(void)
ZFS_IOC_BASE + 78 == ZFS_IOC_POOL_DISCARD_CHECKPOINT &&
ZFS_IOC_BASE + 79 == ZFS_IOC_POOL_INITIALIZE &&
ZFS_IOC_BASE + 80 == ZFS_IOC_POOL_TRIM &&
ZFS_IOC_BASE + 81 == ZFS_IOC_REDACT &&
ZFS_IOC_BASE + 82 == ZFS_IOC_GET_BOOKMARK_PROPS &&
LINUX_IOC_BASE + 1 == ZFS_IOC_EVENTS_NEXT &&
LINUX_IOC_BASE + 2 == ZFS_IOC_EVENTS_CLEAR &&
LINUX_IOC_BASE + 3 == ZFS_IOC_EVENTS_SEEK);
tests/zfs-tests/cmd/stride_dd/.gitignore
+1
View File
@@ -0,0 +1 @@
/stride_dd
tests/zfs-tests/cmd/stride_dd/Makefile.am
+7
View File
@@ -0,0 +1,7 @@
include $(top_srcdir)/config/Rules.am
pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/bin
pkgexec_PROGRAMS = stride_dd
stride_dd_SOURCES = stride_dd.c
stride_dd_LDADD = -lrt
tests/zfs-tests/cmd/stride_dd/stride_dd.c
+214
View File
@@ -0,0 +1,214 @@
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/
/*
* Copyright (c) 2018 by Delphix. All rights reserved.
*/
#include <sys/types.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
static int bsize = 0;
static int count = 0;
static char *ifile = NULL;
static char *ofile = NULL;
static int stride = 0;
static int seek = 0;
static char *execname = "stride_dd";
static void usage(void);
static void parse_options(int argc, char *argv[]);
static void
usage(void)
{
(void) fprintf(stderr,
"usage: %s -i inputfile -o outputfile -b blocksize -c count \n"
" -s stride [ -k seekblocks]\n"
"\n"
"Simplified version of dd that supports the stride option.\n"
"A stride of n means that for each block written, n - 1 blocks\n"
"are skipped in both the input and output file. A stride of 1\n"
"means that blocks are read and written consecutively.\n"
"All numeric parameters must be integers.\n"
"\n"
" inputfile: File to read from\n"
" outputfile: File to write to\n"
" blocksize: Size of each block to read/write\n"
" count: Number of blocks to read/write\n"
" stride: Read/write a block then skip (stride - 1) blocks\n"
" seekblocks: Number of blocks to skip at start of output\n",
execname);
(void) exit(1);
}
static void
parse_options(int argc, char *argv[])
{
int c;
int errflag = 0;
execname = argv[0];
extern char *optarg;
extern int optind, optopt;
while ((c = getopt(argc, argv, ":b:c:i:o:s:k:")) != -1) {
switch (c) {
case 'b':
bsize = atoi(optarg);
break;
case 'c':
count = atoi(optarg);
break;
case 'i':
ifile = optarg;
break;
case 'o':
ofile = optarg;
break;
case 's':
stride = atoi(optarg);
break;
case 'k':
seek = atoi(optarg);
break;
case ':':
(void) fprintf(stderr,
"Option -%c requires an operand\n", optopt);
errflag++;
break;
case '?':
default:
(void) fprintf(stderr,
"Unrecognized option: -%c\n", optopt);
errflag++;
break;
}
if (errflag) {
(void) usage();
}
}
if (bsize <= 0 || count <= 0 || stride <= 0 || ifile == NULL ||
ofile == NULL || seek < 0) {
(void) fprintf(stderr,
"Required parameter(s) missing or invalid.\n");
(void) usage();
}
}
int
main(int argc, char *argv[])
{
int i;
int ifd;
int ofd;
void *buf;
int c;
parse_options(argc, argv);
ifd = open(ifile, O_RDONLY);
if (ifd == -1) {
(void) fprintf(stderr, "%s: %s: ", execname, ifile);
perror("open");
exit(2);
}
ofd = open(ofile, O_WRONLY | O_CREAT, 0666);
if (ofd == -1) {
(void) fprintf(stderr, "%s: %s: ", execname, ofile);
perror("open");
exit(2);
}
/*
* We use valloc because some character block devices expect a
* page-aligned buffer.
*/
int err = posix_memalign(&buf, 4096, bsize);
if (err != 0) {
(void) fprintf(stderr,
"%s: %s\n", execname, strerror(err));
exit(2);
}
if (seek > 0) {
if (lseek(ofd, seek * bsize, SEEK_CUR) == -1) {
perror("output lseek");
exit(2);
}
}
for (i = 0; i < count; i++) {
c = read(ifd, buf, bsize);
if (c != bsize) {
perror("read");
exit(2);
}
if (c != bsize) {
if (c < 0) {
perror("read");
} else {
(void) fprintf(stderr,
"%s: unexpected short read, read %d "
"bytes, expected %d\n", execname,
c, bsize);
}
exit(2);
}
c = write(ofd, buf, bsize);
if (c != bsize) {
if (c < 0) {
perror("write");
} else {
(void) fprintf(stderr,
"%s: unexpected short write, wrote %d "
"bytes, expected %d\n", execname,
c, bsize);
}
exit(2);
}
if (stride > 1) {
if (lseek(ifd, (stride - 1) * bsize, SEEK_CUR) == -1) {
perror("input lseek");
exit(2);
}
if (lseek(ofd, (stride - 1) * bsize, SEEK_CUR) == -1) {
perror("output lseek");
exit(2);
}
}
}
free(buf);
(void) close(ofd);
(void) close(ifd);
return (0);
}
tests/zfs-tests/include/commands.cfg
+4 -1
View File
@@ -1,4 +1,5 @@
#
# Copyright (c) 2016, 2018 by Delphix. All rights reserved.
# These variables are used by zfs-tests.sh to constrain which utilities
# may be used by the suite. The suite will create a directory which is
# the only element of $PATH and create symlinks from that dir to the
@@ -163,6 +164,7 @@ export ZFSTEST_FILES='chg_usr_exec
file_check
file_trunc
file_write
get_diff
largest_file
libzfs_input_check
mkbusy
@@ -180,4 +182,5 @@ export ZFSTEST_FILES='chg_usr_exec
rm_lnkcnt_zero_file
threadsappend
user_ns_exec
xattrtest'
xattrtest
stride_dd'
tests/zfs-tests/include/libtest.shlib
+2 -1
View File
@@ -405,7 +405,8 @@ function create_recv_clone
log_must eval "zfs send $snap | zfs recv -u $recvfs"
log_must mkfile 1m "$mountpoint/data"
log_must zfs snapshot $incr
log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 > $sendfile"
log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 \
iflag=fullblock > $sendfile"
log_mustnot eval "zfs recv -su $recvfs < $sendfile"
destroy_dataset "$sendfs" "-r"
log_must rm -f "$sendfile"
tests/zfs-tests/tests/functional/Makefile.am
+1
View File
@@ -52,6 +52,7 @@ SUBDIRS = \
projectquota \
quota \
raidz \
redacted_send \
redundancy \
refquota \
refreserv \
tests/zfs-tests/tests/functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh
Executable
+77
View File
@@ -0,0 +1,77 @@
#!/bin/ksh -p
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2017 by Delphix. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
#
# DESCRIPTION:
# When a snapshot is destroyed, we used to recurse all clones
# that are downstream of the destroyed snapshot (e.g. to remove
# its key and merge its deadlist entries to the previous one).
# This recursion would break the stack on deeply nested clone
# hierarchies. To avoid this problem today, we keep heap-allocated
# records of all the clones as we traverse their hierarchy.
#
# This test ensures and showcases that our new method works with
# deeply nested clone hierarchies.
#
# STRATEGY:
# 1. Create an fs and take a snapshot of it (snapshot foo)
# 2. Take a second snapshot of the same fs (snapshot bar) on
# top of snapshot foo
# 3. Create a clone of snapshot bar and then take a snapshot
# of it.
# 4. Create a clone of the newly-created snapshot and then
# take a snapshot of it.
# 5. Repeat step [4] many times to create a deeply nested hierarchy.
# 6. Destroy snapshot foo.
#
verify_runnable "both"
typeset FS0=$TESTPOOL/0
typeset FOO=foo
typeset BAR=BAR
typeset FS0SNAPFOO=$FS0@$FOO
typeset FS0SNAPBAR=$FS0@$BAR
typeset -i numds=300
log_must zfs create $FS0
function test_cleanup
{
log_must zfs destroy -Rf $FS0
return 0
}
log_must zfs snapshot $FS0SNAPFOO
log_must zfs snapshot $FS0SNAPBAR
log_onexit test_cleanup
for (( i=1; i<numds; i++ )); do
log_must zfs clone $TESTPOOL/$((i-1))@$BAR $TESTPOOL/$i
log_must zfs snapshot $TESTPOOL/$i@$BAR
done
log_must zfs destroy $FS0SNAPFOO
log_pass "Snapshot deletion doesn't break the stack in deeply nested " \
"clone hierarchies."
tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh
-1
View File
@@ -65,7 +65,6 @@ done
for opt in ${opts[@]}; do
log_mustnot eval "zfs send -b$opt $SENDFS > /dev/null"
log_mustnot eval "zfs send -b$opt $SENDFS#bm > /dev/null"
log_mustnot eval "zfs send -b$opt -i $SENDFS#bm $SENDFS@s2 > /dev/null"
done
# Do 3..6 in a loop to verify various combination of "zfs send" options
tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh
+3 -1
View File
@@ -15,7 +15,7 @@
#
#
# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
# Copyright (c) 2012, 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
@@ -36,6 +36,7 @@ verify_runnable "both"
function cleanup
{
log_must set_tunable32 zfs_override_estimate_recordsize 8192
for ds in $datasets; do
destroy_dataset $ds "-rf"
done
@@ -90,6 +91,7 @@ function verify_size_estimates
log_assert "Verify 'zfs send -nvP' generates valid stream estimates"
log_onexit cleanup
log_must set_tunable32 zfs_override_estimate_recordsize 0
typeset -l block_count=0
typeset -l block_size
typeset -i PERCENT=1
tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg
+3
View File
@@ -77,6 +77,9 @@ typeset -a properties=(
"feature@obsolete_counts"
"feature@zpool_checkpoint"
"feature@spacemap_v2"
"feature@redaction_bookmarks"
"feature@redacted_datasets"
"feature@bookmark_written"
)
# Additional properties added for Linux.
tests/zfs-tests/tests/functional/redacted_send/Makefile.am
+25
View File
@@ -0,0 +1,25 @@
pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/redacted_send
dist_pkgdata_SCRIPTS = \
setup.ksh \
cleanup.ksh \
redacted_compressed.ksh \
redacted_contents.ksh \
redacted_deleted.ksh \
redacted_disabled_feature.ksh \
redacted_embedded.ksh \
redacted_holes.ksh \
redacted_incrementals.ksh \
redacted_largeblocks.ksh \
redacted_many_clones.ksh \
redacted_mixed_recsize.ksh \
redacted_mounts.ksh \
redacted_negative.ksh \
redacted_origin.ksh \
redacted_props.ksh \
redacted_resume.ksh \
redacted_size.ksh \
redacted_volume.ksh
dist_pkgdata_DATA = \
redacted.cfg \
redacted.kshlib
tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh
Executable
+33
View File
@@ -0,0 +1,33 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
destroy_pool $POOL
destroy_pool $POOL2
log_must set_tunable32 zfs_allow_redacted_dataset_mount 0
log_pass
tests/zfs-tests/tests/functional/redacted_send/redacted.cfg
+86
View File
@@ -0,0 +1,86 @@
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
export DISK1=$(echo $DISKS | awk '{print $1}')
export DISK2=$(echo $DISKS | awk '{print $2}')
export POOL=$TESTPOOL
export POOL2=$TESTPOOL2
export FS=$TESTFS
export FS2=$TESTFS2
#
# These are the byte ranges that differ between files and their redacted
# counterparts. See compare_files() for more detail.
#
typeset RANGE0="0,2097152"
typeset RANGE1="0,131072"
typeset RANGE2="1048576,2097152"
typeset RANGE3="0,131072
1966080,131072
3932160,131072"
typeset RANGE4="0,131072
262144,131072
524288,131072
786432,131072"
typeset RANGE5="0,1048576
7340032,1048576"
typeset RANGE6="393216,131072
655360,131072
917504,131072
1179648,131072
1441792,393216
1966080,393216
2621440,262144
3145728,262144
3670016,262144
4194304,262144
4718592,262144
5242880,262144"
typeset RANGE7="1048576,6291456"
typeset RANGE8="4063232,131072"
typeset RANGE9="0,131072
262144,131072
524288,131072
786432,131072
1048576,131072
1310720,131072
1572864,131072
1835008,131072
2097152,131072
2359296,131072
2621440,131072
2883584,131072
3145728,131072
3407872,131072
3670016,131072
3932160,131072"
typeset RANGE10="0,393216"
typeset RANGE11="0,1048576"
typeset RANGE12="0,2097152"
typeset RANGE13="0,16384"
typeset RANGE14=""
typeset RANGE15="0,4194304"
typeset RANGE16="0,6291456"
tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib
+270
View File
@@ -0,0 +1,270 @@
#!/bin/ksh
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright (c) 2016, 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
. $STF_SUITE/tests/functional/redacted_send/redacted.cfg
function setup_dataset
{
typeset ds_name=$1
typeset opts=$2
typeset file_create_func=$3
typeset sendfs="$POOL/$ds_name"
[[ -n $file_create_func ]] || file_create_func=setup_common
log_must zfs create $opts $sendfs
$file_create_func $sendfs
log_must zfs snapshot $sendfs@snap
log_must zfs clone $opts $sendfs@snap $POOL/${ds_name}_clone
log_must zfs snapshot $POOL/${ds_name}_clone@snap
}
function setup_common
{
typeset sendfs=$1
typeset mntpnt=$(get_prop mountpoint $sendfs)
typeset bs=$(get_prop recsize $sendfs)
log_must dd if=/dev/urandom of=$mntpnt/f1 bs=$bs count=16
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=$bs count=32
}
function setup_embedded
{
typeset sendfs=$1
typeset recsize
typeset mntpnt=$(get_prop mountpoint $sendfs)
for recsize in 512 1024 2048 4096 8192 16384; do
if is_linux; then
log_must dd if=/dev/urandom of=$mntpnt/$recsize bs=8 \
count=1 seek=$(((recsize / 8) - 1))
else
log_must mkholes -d $((recsize - 8)):8 $mntpnt/$recsize
fi
done
}
function setup_holes
{
typeset sendfs=$1
typeset mntpnt=$(get_prop mountpoint $sendfs)
typeset M=$((1024 * 1024))
if is_linux; then
log_must dd if=/dev/urandom of=$mntpnt/f1 bs=8M count=1
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1M count=1
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1M count=1 seek=7 \
conv=notrunc
log_must dd if=/dev/urandom of=$mntpnt/f3 bs=1M count=6 seek=1
log_must truncate $mntpnt/f3 --size=$((8 * M))
log_must truncate $mntpnt/f4 --size=$((8 * M))
else
log_must mkholes -d 0:$((8 * M)) $mntpnt/f1
log_must mkholes -d 0:$M -d $((7 * M)):$M $mntpnt/f2
log_must mkholes -d $M:$((6 * M)) -h $((7 * M)):$M $mntpnt/f3
log_must mkholes -h 0:$((8 * M)) $mntpnt/f4
fi
log_must zfs create $sendfs/manyrm
for i in {1..256}; do
log_must stride_dd -i /dev/urandom -o $mntpnt/manyrm/f$i -b 512 \
-c $(random 100) -s $(random 4)
done
log_must zfs snapshot $sendfs/manyrm@snap
log_must zfs clone $sendfs/manyrm@snap $sendfs/manyrm_clone
log_must zfs snapshot $sendfs/manyrm_clone@snap
}
function setup_incrementals
{
typeset sendfs=$1
typeset mntpnt=$(get_prop mountpoint $sendfs)
typeset bs=$(get_prop recsize $sendfs)
log_must dd if=/dev/urandom of=$mntpnt/f1 bs=$bs count=16
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=$bs count=32
log_must mkdir $mntpnt/d1
log_must eval "cat $mntpnt/f1 $mntpnt/f2 >$mntpnt/d1/f1"
log_must zfs snapshot $sendfs@snap0
log_must zfs clone $sendfs@snap0 $POOL/hole
mntpnt=$(get_prop mountpoint $POOL/hole)
log_must dd if=/dev/zero of=$mntpnt/f2 bs=$bs count=16 conv=notrunc
log_must zfs snapshot $POOL/hole@snap
log_must zfs clone $sendfs@snap0 $POOL/stride3
mntpnt=$(get_prop mountpoint $POOL/stride3)
log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $bs -c 11 -s 3
log_must zfs snapshot $POOL/stride3@snap
log_must zfs clone $sendfs@snap0 $POOL/stride5
mntpnt=$(get_prop mountpoint $POOL/stride5)
log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $bs -c 7 -s 5
log_must zfs snapshot $POOL/stride5@snap
log_must zfs clone $sendfs@snap0 $POOL/int
log_must zfs snapshot $POOL/int@snap
log_must zfs clone $POOL/int@snap $POOL/rm
mntpnt=$(get_prop mountpoint $POOL/rm)
log_must rm -rf $mntpnt/[df][12]
log_must zfs snapshot $POOL/rm@snap
log_must zfs clone $POOL/int@snap $POOL/write
mntpnt=$(get_prop mountpoint $POOL/write)
log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=16 conv=notrunc
log_must dd if=/dev/urandom of=$mntpnt/d1/f1 bs=512 count=16 seek=16 \
conv=notrunc
log_must zfs snapshot $POOL/write@snap
}
function setup_mounts
{
typeset sendfs=$1
typeset mntpnt=$(get_prop mountpoint $sendfs)
log_must touch $mntpnt/empty
log_must dd if=/dev/urandom of=$mntpnt/contents1 bs=512 count=2
log_must dd if=/dev/urandom of=$mntpnt/contents2 bs=512 count=2
log_must mkdir $mntpnt/dir1
log_must touch $mntpnt/dir1/empty
log_must dd if=/dev/urandom of=$mntpnt/dir1/contents1 bs=512 count=2
log_must dd if=/dev/urandom of=$mntpnt/dir1/contents2 bs=512 count=2
log_must mkdir $mntpnt/dir1/dir2
log_must touch $mntpnt/dir1/dir2/empty
log_must dd if=/dev/urandom of=$mntpnt/dir1/dir2/file bs=512 count=2
log_must zfs create -s -V 16p $sendfs/vol
log_must zfs snapshot $sendfs/vol@snap
log_must zfs clone $sendfs/vol@snap $sendfs/vol_clone
log_must zfs snapshot $sendfs/vol_clone@snap
}
function mount_redacted
{
typeset flag=''
while getopts "f" opt; do
case $opt in
f)
flag='-f'
;;
esac
done
shift $(($OPTIND - 1))
typeset ds=$1
log_must set_tunable32 zfs_allow_redacted_dataset_mount 1
zfs mount $flag -oro $ds || return 1
log_must set_tunable32 zfs_allow_redacted_dataset_mount 0
return 0
}
function unmount_redacted
{
typeset ds=$1
zfs unmount $ds
}
#
# This function calls a utility that prints out the ranges where a file
# and its redacted counterpart differ, each range on a new line like this:
#
# 0,131072
# 1966080,131072
# 3932160,131072
#
# The output is then checked against a variable containing the expected
# output to verify the redacted ranges are the ones expected.
#
function compare_files
{
typeset sendfs=$1
typeset recvfs=$2
typeset file=$3
typeset expected="$4"
typeset tmpfile="$tmpdir/get_file.out"
log_must mount_redacted -f $recvfs
typeset file1="$(get_prop mountpoint $sendfs)/$file"
typeset file2="$(get_prop mountpoint $recvfs)/$file"
log_note "Comparing $file1 and $file2"
[[ -f $file1 ]] || log_fail "File $file1 does not exist."
[[ -f $file2 ]] || log_fail "File $file2 does not exist."
log_must eval "get_diff $file1 $file2 >$tmpfile"
typeset range="$(cat $tmpfile)"
log_must unmount_redacted $recvfs
[[ "$expected" = "$range" ]] || log_fail "Unexpected range: $range"
}
function redacted_cleanup
{
typeset ds_list=$@
typeset ds
# Verify the receiving pool can still be exported and imported.
log_must zpool export $POOL2
log_must zpool import $POOL2
for ds in $ds_list; do
datasetexists $ds && log_must zfs destroy -R $ds
done
log_must set_tunable32 zfs_allow_redacted_dataset_mount 0
rm -f $(get_prop mountpoint $POOL)/tmp/*
}
# Retrieve the redaction list of a bookmark or snapshot, using
# the property or zdb output, as requested.
function get_guid_list
{
typeset filename=$1
typeset dataset=$2
typeset use_zdb=${3:-false}
if $use_zdb; then
guid_list=$(zdb -vvvv $dataset | sed -e 's/,//g' \
-ne 's/^.*Snapshots: \[\(.*\)\]/\1/p')
else
guid_list=$(get_prop redact_snaps $dataset)
fi
for guid in $(echo $guid_list | tr ',' ' '); do
echo $guid
done | sort >$filename
}
tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh
Executable
+71
View File
@@ -0,0 +1,71 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify that compressed send streams are redacted correctly.
#
# Strategy:
# 1. Receive a redacted compressed send stream, verifying compression and
# redaction.
# 2. Receive an incremental on the full receive, verifying compression and
# redaction.
#
typeset ds_name="compressed"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name "-o compress=lz4"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset clone_mnt="$(get_prop mountpoint $clone)"
log_onexit redacted_cleanup $sendfs $recvfs
log_must stride_dd -i /dev/urandom -o $clone_mnt/f1 -b $((128 * 1024)) -c 4 -s 2
log_must zfs snapshot $clone@snap1
log_must rm $clone_mnt/f2
log_must zfs snapshot $clone@snap2
log_must zfs redact $sendfs@snap book1 $clone@snap1 $clone@snap2
log_must eval "zfs send -c --redact book1 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must stream_has_features $stream compressed lz4 redacted
compare_files $sendfs $recvfs "f1" "$RANGE4"
verify_stream_size $stream $sendfs
log_must mount_redacted -f $recvfs
verify_stream_size $stream $recvfs
log_must unmount_redacted $recvfs
log_must eval "zfs send -c -i $sendfs@snap $clone@snap1 >$stream"
log_must eval "zfs recv $POOL2/inc1 <$stream"
log_must stream_has_features $stream compressed lz4
typeset mntpnt=$(get_prop mountpoint $POOL2)
log_must diff $clone_mnt/f1 $mntpnt/inc1/f1
log_must diff $send_mnt/f2 $mntpnt/inc1/f2
log_must eval "zfs send -c -i $sendfs@snap $clone@snap2 >$stream"
log_must eval "zfs recv $POOL2/inc2 <$stream"
log_must stream_has_features $stream compressed lz4
log_must diff $clone_mnt/f1 $mntpnt/inc1/f1
[[ -f $mntpnt/inc2/f2 ]] && log_fail "File f2 should not exist."
log_pass "Compressed send streams are redacted correctly."
tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh
Executable
+162
View File
@@ -0,0 +1,162 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify redaction works as expected for various scenarios.
#
# Strategy:
# 1. An unmodified file does not get redacted at all.
# 2. Empty redaction list redacts everything.
# 3. A file removed in the clone redacts the whole file.
# 4. A file moved in the clone does not redact the file.
# 5. A copied, then removed file in the clone redacts the whole file.
# 6. Overwriting a file with identical contents redacts the file.
# 7. A paritally modified block redacts the entire block.
# 8. Only overlapping areas of modified ranges are redacted.
# 9. Send from the root dataset of a pool work correctly.
#
typeset ds_name="contents"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name ''
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
log_onexit redacted_cleanup $sendfs $recvfs
# An unmodified file does not get redacted at all.
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book1 $clone@snap1
log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must mount_redacted -f $recvfs
log_must diff $send_mnt/f1 $recv_mnt/f1
log_must diff $send_mnt/f2 $recv_mnt/f2
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Removing a file in the clone redacts the entire file.
log_must rm "$clone_mnt/f1"
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book3 $clone@snap1
log_must eval "zfs send --redact book3 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f1" "$RANGE0"
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Moving a file in the clone does not redact the file.
log_must mv "$clone_mnt/f1" "$clone_mnt/f1.moved"
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book4 $clone@snap1
log_must eval "zfs send --redact book4 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must mount_redacted -f $recvfs
[[ -f $recv_mnt/f1.moved ]] && log_fail "Found moved file in redacted receive."
log_must diff $send_mnt/f1 $recv_mnt/f1
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Copying, then removing a file in the clone does redact the file.
log_must cp "$clone_mnt/f1" "$clone_mnt/f1.copied"
log_must rm "$clone_mnt/f1"
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book5 $clone@snap1
log_must eval "zfs send --redact book5 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f1" "$RANGE0"
log_must mount_redacted -f $recvfs
[[ -f $recv_mnt/f1.copied ]] && log_fail "Found moved file in redacted receive."
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Overwriting the contents of a block with identical contents redacts the file.
log_must cp "$clone_mnt/f1" "$clone_mnt/f1.copied"
log_must cp "$clone_mnt/f1.copied" "$clone_mnt/f1"
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book6 $clone@snap1
log_must eval "zfs send --redact book6 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f1" "$RANGE0"
log_must mount_redacted -f $recvfs
[[ -f $recv_mnt/f1.copied ]] && log_fail "Found moved file in redacted receive."
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Modifying some of a block redacts the whole block.
log_must dd if=/dev/urandom of=$clone_mnt/f1 conv=notrunc seek=2 count=1 bs=32k
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book7 $clone@snap1
log_must eval "zfs send --redact book7 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f1" "$RANGE1"
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Only overlapping areas of modified ranges are redacted.
log_must dd if=/dev/urandom of=$clone_mnt/f2 bs=1024k count=3 conv=notrunc
log_must zfs snapshot $clone@snap1
log_must zfs clone $sendfs@snap $clone/new
typeset mntpnt="$(get_prop mountpoint $clone/new)"
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1024k seek=1 count=3 \
conv=notrunc
log_must zfs snapshot $clone/new@snap
log_must zfs redact $sendfs@snap book8 $clone@snap1 $clone/new@snap
log_must eval "zfs send --redact book8 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f2" "$RANGE2"
log_must zfs destroy -R $clone/new
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# FizzBuzz version
log_must zfs clone $sendfs@snap $POOL/stride3
mntpnt="$(get_prop mountpoint $POOL/stride3)"
log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $((128 * 1024)) -c 11 -s 3
log_must zfs snapshot $POOL/stride3@snap
log_must zfs clone $sendfs@snap $POOL/stride5
mntpnt="$(get_prop mountpoint $POOL/stride5)"
log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $((128 * 1024)) -c 7 -s 5
log_must zfs snapshot $POOL/stride5@snap
log_must zfs redact $sendfs@snap book8a $POOL/stride3@snap $POOL/stride5@snap
log_must eval "zfs send --redact book8a $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f2" "$RANGE3"
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Send from the root dataset of a pool work correctly.
log_must dd if=/dev/urandom of=/$POOL/f1 bs=128k count=4
log_must zfs snapshot $POOL@snap
log_must zfs clone $POOL@snap $POOL/clone
log_must dd if=/dev/urandom of=/$POOL/clone/f1 bs=128k count=1 conv=notrunc
log_must zfs snapshot $POOL/clone@snap
log_must zfs redact $POOL@snap book9 $POOL/clone@snap
log_must eval "zfs send --redact book9 $POOL@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $POOL $recvfs "f1" "$RANGE1"
log_must zfs destroy -R $POOL@snap
log_pass "Redaction works as expected for various scenarios."
tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh
Executable
+103
View File
@@ -0,0 +1,103 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2017, 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify redaction works as expected with respect to deleted files
#
# Strategy:
# 1. A file on the delete queue counts as deleted when using it to calculate
# redaction.
# 2. A file that is removed in the tosnap of an incremental, where the fromsnap
# is a redaction bookmark that contains references to that file, does not
# result in records for that file.
#
typeset ds_name="deleted"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset clone2="$POOL/${ds_name}_clone2"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name ''
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
log_onexit redacted_cleanup $sendfs $recvfs
#
# A file on the delete queue counts as deleted when using it to calculate
# redaction.
#
#
# Open file descriptor 5 for appending to $clone_mnt/f1 so that it will go on
# the delete queue when we rm it.
#
exec 5>>$clone_mnt/f1
log_must dd if=/dev/urandom of=$clone_mnt/f1 bs=512 count=1 conv=notrunc
log_must rm $clone_mnt/f1
log_must zfs snapshot $clone@snap1
# Close file descriptor 5
exec 5>&-
log_must zfs redact $sendfs@snap book1 $clone@snap1
log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must mount_redacted -f $recvfs
#
# We have temporarily disabled redaction blkptrs, so this will not
# fail as was originally intended. We should uncomment this line
# when we reenable redaction blkptrs.
#
#log_mustnot dd if=$recv_mnt/f1 of=/dev/null bs=512 count=1
log_must diff $send_mnt/f2 $recv_mnt/f2
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
#
# A file that is removed in the tosnap of an incremental, where the fromsnap
# is a redaction bookmark that contains references to that file, does not
# result in records for that file.
#
log_must zfs clone $sendfs@snap $clone2
typeset clone2_mnt="$(get_prop mountpoint $clone2)"
log_must rm -rf $clone2_mnt/*
log_must zfs snapshot $clone2@snap
log_must zfs redact $sendfs@snap book2 $clone2@snap
log_must zfs destroy -R $clone2
log_must eval "zfs send --redact book2 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must rm $send_mnt/f1
log_must zfs snapshot $sendfs@snap2
log_must zfs clone $sendfs@snap2 $clone2
typeset clone2_mnt="$(get_prop mountpoint $clone2)"
log_must rm $clone2_mnt/*
log_must zfs snapshot $clone2@snap
log_must zfs redact $sendfs@snap2 book3 $clone2@snap
log_must zfs destroy -R $clone2
log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must mount_redacted -f $recvfs
log_must diff <(ls $send_mnt) <(ls $recv_mnt)
log_must zfs destroy -R $recvfs
log_must zfs rollback -R $sendfs@snap
log_pass "Verify Redaction works as expected with respect to deleted files."
tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh
Executable
+71
View File
@@ -0,0 +1,71 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify the functionality of the redaction_bookmarks and redacted_datasets
# features.
#
# Strategy:
# 1. Create a pool with all features disabled.
# 2. Verify redacted send fails.
# 3. Enable redaction_bookmarks and verify redacted sends works.
# 4. Verify recepit of a redacted stream fails.
# 5. Enable recacted_datasets and verify zfs receive works.
#
typeset ds_name="disabled"
typeset sendfs="$POOL/$ds_name"
typeset sendfs1="$POOL2/${ds_name}1"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset clone1="$POOL2/${ds_name}_clone1"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name ''
function cleanup
{
destroy_pool $POOL2
create_pool $POOL2 $DISK2
log_must zfs snapshot $POOL2@init
redacted_cleanup $sendfs $recvfs
}
log_onexit cleanup
destroy_pool $POOL2
log_must zpool create -d $POOL2 $DISK2
log_must zfs create $sendfs1
log_must zfs snapshot $sendfs1@snap
log_must zfs clone $sendfs1@snap $clone1
log_must zfs snapshot $clone1@snap
log_mustnot zfs redact $sendfs1@snap book1 $clone1@snap
log_must zpool set feature@redaction_bookmarks=enabled $POOL2
log_must zfs redact $sendfs1@snap book1 $clone1@snap
log_must zfs redact $sendfs@snap book1 $clone@snap
log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
log_mustnot eval "zfs recv $recvfs <$stream"
log_must zpool set feature@redacted_datasets=enabled $POOL2
log_must eval "zfs recv $recvfs <$stream"
log_pass "The redacted send/recv features work correctly."
tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh
Executable
+103
View File
@@ -0,0 +1,103 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify embedded blocks and redacted send work correctly together.
#
# Strategy:
# 1. Create recsize sized files with embedded blocks from size 512b to 16k.
# 2. Receive a redacted send stream with nothing redacted.
# 3. Verify the received files match the source, contain embedded blocks, and
# that the stream has the redacted and embedded data features.
# 4. Receive a redacted send stream with files 512, 2048 and 8192 redacted.
# 5. Verify that the redacted files no longer match, but the others still
# contain embedded blocks and the stream has the redacted and embedded
# data features.
#
typeset ds_name="embedded"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name '-o compress=lz4' setup_embedded
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
typeset recsize send_obj recv_obj
log_onexit redacted_cleanup $sendfs $recvfs
log_must zfs redact $sendfs@snap book1 $clone@snap
log_must eval "zfs send -e --redact book1 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must stream_has_features $stream redacted embed_data
log_must mount_redacted -f $recvfs
for recsize in 512 1024 2048 4096 8192 16384; do
send_obj=$(get_objnum $send_mnt/$recsize)
recv_obj=$(get_objnum $recv_mnt/$recsize)
log_must diff $send_mnt/$recsize $recv_mnt/$recsize
log_must eval "zdb -ddddd $sendfs $send_obj >$tmpdir/send.zdb"
log_must eval "zdb -ddddd $recvfs $recv_obj >$tmpdir/recv.zdb"
grep -q "EMBEDDED" $tmpdir/send.zdb || \
log_fail "Obj $send_obj not embedded in $sendfs"
grep -q "EMBEDDED" $tmpdir/recv.zdb || \
log_fail "Obj $recv_obj not embedded in $recvfs"
cat $stream | zstreamdump -v | log_must grep -q \
"WRITE_EMBEDDED object = $send_obj offset = 0"
done
log_must zfs destroy -R $recvfs
for recsize in 512 2048 8192; do
log_must dd if=/dev/urandom of=$clone_mnt/$recsize bs=$recsize count=1
done
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book2 $clone@snap1
log_must eval "zfs send -e --redact book2 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must stream_has_features $stream redacted embed_data
log_must mount_redacted -f $recvfs
for recsize in 512 2048 8192; do
log_mustnot diff $send_mnt/$recsize $recv_mnt/$recsize
done
for recsize in 1024 4096 16384; do
send_obj=$(get_objnum $send_mnt/$recsize)
recv_obj=$(get_objnum $recv_mnt/$recsize)
log_must diff $send_mnt/$recsize $recv_mnt/$recsize
log_must eval "zdb -ddddd $sendfs $send_obj >$tmpdir/send.zdb"
log_must eval "zdb -ddddd $recvfs $recv_obj >$tmpdir/recv.zdb"
grep -q "EMBEDDED" $tmpdir/send.zdb || \
log_fail "Obj $send_obj not embedded in $sendfs"
grep -q "EMBEDDED" $tmpdir/recv.zdb || \
log_fail "Obj $recv_obj not embedded in $recvfs"
cat $stream | zstreamdump -v | log_must grep -q \
"WRITE_EMBEDDED object = $send_obj offset = 0"
done
log_pass "Embedded blocks and redacted send work correctly together."
tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh
Executable
+120
View File
@@ -0,0 +1,120 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify redacted send streams reliably handle holes.
#
# Strategy:
# 1. Holes written at the beginning and end of a non-sparse file in the
# redacted list are correctly redacted.
# 2. Holes written throughout a non-sparse file in the redacted list are
# correctly redacted.
# 3. Data written into a hole in a sparse file in the redacted list are
# correctly redacted.
# 4. Holes in metadata blocks.
#
typeset ds_name="holes"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name '' setup_holes
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
typeset M=$((1024 * 1024))
log_onexit redacted_cleanup $sendfs $recvfs
# Write holes at the start and end of a non-sparse file.
if is_linux; then
log_must dd if=/dev/zero of=$clone_mnt/f1 bs=1M count=1 conv=notrunc
log_must dd if=/dev/zero of=$clone_mnt/f1 bs=1M count=1 conv=notrunc seek=7
else
log_must mkholes -h 0:$M -h $((7 * M)):$M $clone_mnt/f1
fi
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book1 $clone@snap1
log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f1" "$RANGE5"
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Write two overlapping sets of holes into the same non-sparse file.
log_must stride_dd -i /dev/zero -o $clone_mnt/f1 -b $((128 * 1024)) -c 8 -s 2 -k 3
log_must stride_dd -i /dev/zero -o $clone_mnt/f1 -b $((256 * 1024)) -c 8 -s 2 -k 6
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book2 $clone@snap1
log_must eval "zfs send --redact book2 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f1" "$RANGE6"
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Write data into the middle of a hole.
if is_linux; then
log_must dd if=/dev/urandom of=$clone_mnt/f2 bs=1M count=2 seek=3 \
conv=notrunc
else
log_must mkholes -d $((3 * M)):$((2 * M)) $clone_mnt/f2
fi
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book3 $clone@snap1
log_must eval "zfs send --redact book3 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f2" "$RANGE14"
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Remove a file with holes.
log_must rm $clone_mnt/f3
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book4 $clone@snap1
log_must eval "zfs send --redact book4 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f3" "$RANGE7"
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
# Create a hole in a L0 metadata block by removing files.
log_must rm $send_mnt/manyrm_clone/f{32..96}
log_must zfs snapshot $sendfs/manyrm_clone@snap1
log_must zfs redact $sendfs/manyrm@snap book6 $sendfs/manyrm_clone@snap1
log_must eval "zfs send --redact book6 $sendfs/manyrm@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
log_must mount_redacted -f $recvfs
for i in {1..31} {97..256}; do
diff $send_mnt/manyrm/f$i $recv_mnt/f$i || log_fail \
"File f$i did not match in the send and recv datasets."
done
for i in {32..96}; do
file_size=$(stat -c %s $send_mnt/manyrm/f$i)
redacted_size=$(stat -c %s $recv_mnt/f$i)
[[ $file_size -eq $redacted_size ]] || log_fail \
"File f$i has size $file_size and redacted size $redacted_size"
done
log_must zfs rollback -R $clone@snap
log_must zfs destroy -R $recvfs
log_pass "Redacted send streams reliably handle holes."
tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh
Executable
+152
View File
@@ -0,0 +1,152 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify that incrementals (redacted and normal) work with redacted datasets.
#
# Strategy:
# 1. Test normal incrementals from the original snap to a subset of the
# redaction list.
# 2. Test receipt of intermediate clones, and their children.
# 3. Test receipt with origin snap specified by '-o origin='.
# 4. Test incrementals from redaction bookmarks.
#
typeset ds_name="incrementals"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name '' setup_incrementals
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
log_onexit redacted_cleanup $sendfs $recvfs $POOL2/rfs
# Setup a redacted send using a redaction list at varying depth.
log_must zfs redact $sendfs@snap0 book1 $POOL/rm@snap $POOL/stride3@snap \
$POOL/stride5@snap
log_must eval "zfs send --redact book1 $sendfs@snap0 >$stream"
log_must eval "zfs receive $POOL2/rfs <$stream"
# Verify receipt of normal incrementals to redaction list members.
log_must eval "zfs send -i $sendfs@snap0 $POOL/stride3@snap >$stream"
log_must eval "zfs recv $POOL2/rstride3 <$stream"
log_must diff -r /$POOL/stride3 /$POOL2/rstride3
log_must eval "zfs send -i $sendfs@snap0 $POOL/stride5@snap >$stream"
log_must eval "zfs recv $POOL2/rstride5 <$stream"
log_must diff -r /$POOL/stride5 /$POOL2/rstride5
# But not a normal child that we weren't redacted with respect to.
log_must eval "zfs send -i $sendfs@snap0 $POOL/hole@snap >$stream"
log_mustnot eval "zfs recv $POOL2/rhole@snap <$stream"
# Verify we can receive an intermediate clone redacted with respect to a
# subset of the original redaction list.
log_must zfs redact $POOL/int@snap book2 $POOL/rm@snap
log_must eval "zfs send -i $sendfs@snap0 --redact book2 $POOL/int@snap >$stream"
log_must eval "zfs recv $POOL2/rint <$stream"
compare_files $POOL/int $POOL2/rint "f1" "$RANGE0"
compare_files $POOL/int $POOL2/rint "f2" "$RANGE15"
compare_files $POOL/int $POOL2/rint "d1/f1" "$RANGE16"
log_must mount_redacted -f $POOL2/rint
# Verify we can receive grandchildren on the child.
log_must eval "zfs send -i $POOL/int@snap $POOL/rm@snap >$stream"
log_must eval "zfs receive $POOL2/rrm <$stream"
log_must diff -r /$POOL/rm /$POOL2/rrm
# But not a grandchild that the received child wasn't redacted with respect to.
log_must eval "zfs send -i $POOL/int@snap $POOL/write@snap >$stream"
log_mustnot eval "zfs recv $POOL2/rwrite<$stream"
# Verify we cannot receive an intermediate clone that isn't redacted with
# respect to a subset of the original redaction list.
log_must zfs redact $POOL/int@snap book4 $POOL/rm@snap $POOL/write@snap
log_must eval "zfs send -i $sendfs@snap0 --redact book4 $POOL/int@snap >$stream"
log_mustnot eval "zfs recv $POOL2/rint <$stream"
log_must zfs redact $POOL/int@snap book5 $POOL/write@snap
log_must eval "zfs send -i $sendfs@snap0 --redact book5 $POOL/int@snap >$stream"
log_mustnot eval "zfs recv $POOL2/rint <$stream"
log_mustnot zfs redact $POOL/int@snap book6 $POOL/hole@snap
# Verify we can receive a full clone of the grandchild on the child.
log_must eval "zfs send $POOL/write@snap >$stream"
log_must eval "zfs recv -o origin=$POOL2/rint@snap $POOL2/rwrite <$stream"
log_must diff -r /$POOL/write /$POOL2/rwrite
# Along with other origins.
log_must eval "zfs recv -o origin=$POOL2/rfs@snap0 $POOL2/rwrite1 <$stream"
log_must diff -r /$POOL/write /$POOL2/rwrite1
log_must eval "zfs recv -o origin=$POOL2@init $POOL2/rwrite2 <$stream"
log_must diff -r /$POOL/write /$POOL2/rwrite2
log_must zfs destroy -R $POOL2/rwrite2
log_must zfs destroy -R $POOL2/rfs
# Write some data for tests of incremental sends from bookmarks
log_must zfs snapshot $sendfs@snap1
log_must zfs clone $sendfs@snap1 $POOL/hole1
typeset mntpnt=$(get_prop mountpoint $POOL/hole1)
log_must dd if=/dev/zero of=$mntpnt/f2 bs=128k count=16 conv=notrunc
log_must zfs snapshot $POOL/hole1@snap
log_must zfs clone $sendfs@snap1 $POOL/write1
mntpnt=$(get_prop mountpoint $POOL/write1)
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=128k count=16 conv=notrunc
log_must zfs snapshot $POOL/write1@snap
log_must zfs clone $POOL/int@snap $POOL/write2
mntpnt=$(get_prop mountpoint $POOL/write2)
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=128k count=16 conv=notrunc
log_must zfs snapshot $POOL/write2@snap
# Setup a redacted send using a redaction list at varying depth.
log_must zfs redact $sendfs@snap0 book7 $POOL/rm@snap $POOL/stride3@snap \
$POOL/stride5@snap
log_must eval "zfs send --redact book7 $sendfs@snap0 >$stream"
log_must eval "zfs receive $POOL2/rfs <$stream"
# Verify we can receive a redacted incremental sending from the bookmark.
log_must zfs redact $sendfs@snap1 book8 $POOL/write1@snap
log_must eval "zfs send -i $sendfs#book7 --redact book8 $sendfs@snap1 >$stream"
log_must eval "zfs receive $POOL2/rfs <$stream"
# The stride3 and stride5 snaps redact 3 128k blocks at block offsets 0 15 and
# 30 of f2. The write1 snap only covers the first two of those three blocks.
compare_files $sendfs $POOL2/rfs "f2" "$RANGE12"
log_must mount_redacted -f $POOL2/rfs
log_must diff $send_mnt/f1 /$POOL2/rfs/f1
log_must diff $send_mnt/d1/f1 /$POOL2/rfs/d1/f1
unmount_redacted $POOL2/rfs
# Verify we can receive a normal child we weren't redacted with respect to by
# sending from the bookmark.
log_must eval "zfs send -i $sendfs#book7 $POOL/hole1@snap >$stream"
log_must eval "zfs recv $POOL2/rhole1 <$stream"
log_must diff -r /$POOL/hole1 /$POOL2/rhole1
# Verify we can receive an intermediate clone redacted with respect to a
# non-subset if we send from the bookmark.
log_must zfs redact $POOL/int@snap book9 $POOL/write2@snap
log_must eval "zfs send -i $sendfs#book7 --redact book9 $POOL/int@snap >$stream"
log_must eval "zfs receive $POOL2/rint <$stream"
compare_files $sendfs $POOL2/rint "f2" "$RANGE12"
log_pass "Incrementals (redacted and normal) work with redacted datasets."
tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh
Executable
+63
View File
@@ -0,0 +1,63 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify large blocks and redacted send work correctly together.
#
# Strategy:
# 1. Create a dataset and clone with a 1m recordsize, modifying a few k
# within the first 1m of a 16m file.
# 2. Verify that the whole first 1m of the file is redacted.
# 3. Receive an incremental stream from the original snap to the snap it
# was redacted with respect to.
# 4. Verify that the received dataset matches the clone
#
typeset ds_name="largeblocks"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name '-o recsize=1m'
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
log_onexit redacted_cleanup $sendfs $recvfs
log_must dd if=/dev/urandom of=$clone_mnt/f1 bs=32k count=3 seek=8 conv=notrunc
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book1 $clone@snap1
log_must eval "zfs send -L --redact book1 $sendfs@snap >$stream"
log_must stream_has_features $stream redacted large_blocks
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f1" "$RANGE11"
log_must mount_redacted -f $recvfs
log_must diff $send_mnt/f2 $recv_mnt/f2
unmount_redacted $recvfs
log_must eval "zfs send -L -i $sendfs@snap $clone@snap1 >$stream"
log_must stream_has_features $stream large_blocks
log_must eval "zfs recv $recvfs/new <$stream"
log_must diff -r $clone_mnt $recv_mnt/new
log_pass "Large blocks and redacted send work correctly together."
tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh
Executable
+68
View File
@@ -0,0 +1,68 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify redacted send can deal with a large redaction list.
#
# Strategy:
# 1. Create 64 clones of sendfs each of which modifies two blocks in a file.
# The first modification is at an offset unique to each clone, and the
# second (the last block in the file) is common to them all.
# 2. Verify a redacted stream with a reasonable redaction list length can
# be correctly processed.
# 3. Verify that if the list is too long, the send fails gracefully.
#
typeset ds_name="many_clones"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name ''
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
typeset redaction_list=''
typeset mntpnt
log_onexit redacted_cleanup $sendfs $recvfs
# Fill in both the last block, and a different block in every clone.
for i in {1..64}; do
log_must zfs clone $sendfs@snap ${clone}$i
mntpnt=$(get_prop mountpoint ${clone}$i)
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=$i \
conv=notrunc
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=63 \
conv=notrunc
log_must zfs snapshot ${clone}$i@snap
done
# The limit isn't necessarily 32 snapshots. The maximum number of snapshots in
# the redacted list is determined in dsl_bookmark_create_redacted_check().
log_must zfs redact $sendfs@snap book1 $clone{1..32}@snap
log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
compare_files $sendfs $recvfs "f2" "$RANGE8"
log_mustnot zfs redact $sendfs@snap book2 $clone{1..64}@snap
log_pass "Redacted send can deal with a large redaction list."
tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh
Executable
+77
View File
@@ -0,0 +1,77 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify redacted send works with datasets of different sizes.
#
# Strategy:
# 1. Create two dataset one with recsize 512, and one 1m and create a 2m file.
# 2. For each dataset, create clones of both 512 and 1m recsize and modify
# the first 16k of the file.
# 3. Send each original dataset, redacted with respect to each of the clones
# into both a dataset inheriting a 512 recsize and a 1m one.
# 4. Verify that the smallest unit of redaction is that of the origin fs.
#
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
typeset mntpnt
log_onexit redacted_cleanup $POOL/512 $POOL/1m $POOL2/512 $POOL2/1m
# Set up the datasets we'll send and redact from.
log_must zfs create -o recsize=512 $POOL/512
mntpnt=$(get_prop mountpoint $POOL/512)
log_must dd if=/dev/urandom of=$mntpnt/f1 bs=1024k count=2
log_must zfs snapshot $POOL/512@snap
log_must zfs clone -o recsize=1m $POOL/512@snap $POOL/1mclone
mntpnt=$(get_prop mountpoint $POOL/1mclone)
log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=32 conv=notrunc
log_must zfs snapshot $POOL/1mclone@snap
log_must zfs create -o recsize=1m $POOL/1m
mntpnt=$(get_prop mountpoint $POOL/1m)
log_must dd if=/dev/urandom of=$mntpnt/f1 bs=1024k count=2
log_must zfs snapshot $POOL/1m@snap
log_must zfs clone -o recsize=512 $POOL/1m@snap $POOL/512clone
mntpnt=$(get_prop mountpoint $POOL/512clone)
log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=32 conv=notrunc
log_must zfs snapshot $POOL/512clone@snap
# Create datasets that allow received datasets to inherit recordsize.
log_must zfs create -o recsize=512 $POOL2/512
log_must zfs create -o recsize=1m $POOL2/1m
# Do the sends and verify the contents.
log_must zfs redact $POOL/512@snap book1 $POOL/1mclone@snap
log_must eval "zfs send --redact book1 $POOL/512@snap>$stream"
log_must eval "zfs recv $POOL2/512/recva <$stream"
compare_files $POOL/512 $POOL2/512/recva "f1" "$RANGE13"
log_must eval "zfs recv $POOL2/1m/recvb <$stream"
compare_files $POOL/512 $POOL2/1m/recvb "f1" "$RANGE13"
log_must zfs redact $POOL/1m@snap book2 $POOL/512clone@snap
log_must eval "zfs send --redact book2 $POOL/1m@snap >$stream"
log_must eval "zfs recv $POOL2/512/recvc <$stream"
compare_files $POOL/1m $POOL2/512/recvc "f1" "$RANGE11"
log_must eval "zfs recv $POOL2/1m/recvd <$stream"
compare_files $POOL/1m $POOL2/1m/recvd "f1" "$RANGE11"
log_pass "Redaction works correctly with different recordsizes."
tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh
Executable
+109
View File
@@ -0,0 +1,109 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify that received redacted datasets are not mounted by default, but
# can still be mounted after setting zfs_allow_redacted_dataset_mount.
#
# Strategy:
# 1. Verify a received redacted stream isn't mounted by default.
# 2. Set zfs_allow_redacted_dataset_mount and verify it can't be mounted
# without the -f flag, but can with -f.
# 3. Receive a redacted volume.
# 4. Verify the device file isn't present until the kernel variable is set.
# 5. Verify the files in the send fs are also present in the recv fs.
#
typeset ds_name="mounts"
typeset sendfs="$POOL/$ds_name"
typeset sendvol="$sendfs/vol"
typeset recvfs="$POOL2/$ds_name"
typeset recvvol="$POOL2/vol"
typeset clone="$POOL/${ds_name}_clone"
typeset clonevol="${sendvol}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name '' setup_mounts
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
typeset recv_vol_file="/dev/zvol/$recvvol"
log_onexit redacted_cleanup $sendfs $recvfs $recvvol
log_must rm $clone_mnt/empty $clone_mnt/contents1
log_must dd if=/dev/urandom of=$clone_mnt/contents2 bs=512 count=1 conv=notrunc
log_must rm $clone_mnt/dir1/contents1
log_must rm -rf $clone_mnt/dir1/dir2
log_must dd if=/dev/urandom of=$clone_mnt/dir1/contents2 bs=512 count=1 \
conv=notrunc
log_must dd if=/dev/urandom of=$clone_mnt/dir1/empty bs=512 count=1
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendfs@snap book1 $clone@snap
log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
log_must eval "zfs receive $recvfs <$stream"
log_mustnot ismounted $recvfs
log_mustnot mount_redacted $recvfs
log_mustnot ismounted $recvfs
log_must mount_redacted -f $recvfs
log_must ismounted $recvfs
# Verify that the send and recv fs both have the same files under their
# mountpoints by comparing find output with the name of the mountpoint
# deleted.
contents=$(log_must find $recv_mnt)
contents_orig=$(log_must find $send_mnt)
log_must diff <(echo ${contents//$recv_mnt/}) \
<(echo ${contents_orig//$send_mnt/})
log_must zfs redact $sendvol@snap book2 $clonevol@snap
log_must eval "zfs send --redact book2 $sendvol@snap >$stream"
log_must eval "zfs receive $recvvol <$stream"
[[ -b $recv_vol_file ]] && log_fail "Volume device file should not exist."
log_must set_tunable32 zfs_allow_redacted_dataset_mount 1
log_must zpool export $POOL2
log_must zpool import $POOL2
udevadm settle
# The device file isn't guaranteed to show up right away.
if [[ ! -b $recv_vol_file ]]; then
udevadm settle
for t in 10 5 3 2 1; do
log_note "Polling $t seconds for device file."
udevadm settle
sleep $t
[[ -b $recv_vol_file ]] && break
done
fi
[[ -b $recv_vol_file ]] || log_fail "Volume device file should exist."
log_must dd if=/dev/urandom of=$send_mnt/dir1/contents1 bs=512 count=2
log_must rm $send_mnt/dir1/dir2/empty
log_must zfs snapshot $sendfs@snap2
log_must eval "zfs send -i $sendfs#book1 $sendfs@snap2 >$stream"
log_must eval "zfs receive $recvfs <$stream"
log_must mount_redacted -f $recvfs
log_must ismounted $recvfs
contents=$(log_must find $recv_mnt)
contents_orig=$(log_must find $send_mnt)
log_must diff <(echo ${contents//$recv_mnt/}) \
<(echo ${contents_orig//$send_mnt/})
log_pass "Received redacted streams can be mounted."
tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh
Executable
+80
View File
@@ -0,0 +1,80 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Test that redacted send correctly detects invalid arguments.
#
typeset sendfs="$POOL2/sendfs"
typeset recvfs="$POOL2/recvfs"
typeset clone1="$POOL2/clone1"
typeset clone2="$POOL2/clone2"
typeset clone3="$POOL2/clone3"
typeset clone3="$POOL2/clone4"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
log_onexit redacted_cleanup $sendfs $recvfs $clone3
log_must zfs create $sendfs
log_must zfs snapshot $sendfs@snap1
log_must zfs snapshot $sendfs@snap2
log_must zfs snapshot $sendfs@snap3
log_must zfs clone $sendfs@snap2 $clone1
log_must zfs snapshot $clone1@snap
log_must zfs bookmark $clone1@snap $clone1#book
log_must zfs clone $sendfs@snap2 $clone2
log_must zfs snapshot $clone2@snap
# Incompatible flags
log_must zfs redact $sendfs@snap2 book $clone1@snap
log_mustnot eval "zfs send -R --redact book $sendfs@snap2 >/dev/null"
typeset arg
for arg in "$sendfs" "$clone1#book"; do
log_mustnot eval "zfs send --redact book $arg >/dev/null"
done
# Bad redaction list arguments
log_mustnot zfs redact $sendfs@snap1
log_mustnot zfs redact $sendfs@snap1 book
log_mustnot zfs redact $sendfs#book1 book4 $clone1
log_mustnot eval "zfs send --redact $sendfs#book $sendfs@snap >/dev/null"
# Redaction snapshots not a descendant of tosnap
log_mustnot zfs redact $sendfs@snap2 book $sendfs@snap2
log_must zfs redact $sendfs@snap2 book2 $clone1@snap $clone2@snap
log_must eval "zfs send --redact book2 $sendfs@snap2 >$stream"
log_must zfs redact $sendfs@snap2 book3 $clone1@snap $clone2@snap
log_must eval "zfs send -i $sendfs@snap1 --redact book3 $sendfs@snap2 \
>/dev/null"
log_mustnot zfs redact $sendfs@snap3 $sendfs@snap3 $clone1@snap
# Full redacted sends of redacted datasets are not allowed.
log_must eval "zfs recv $recvfs <$stream"
log_must zfs snapshot $recvfs@snap
log_must zfs clone $recvfs@snap $clone3
log_must zfs snapshot $clone3@snap
log_mustnot zfs redact $recvfs@snap book5 $clone3@snap
# Nor may a redacted dataset appear in the redaction list.
log_mustnot zfs redact testpool2/recvfs@snap2 book7 testpool2/recvfs@snap
log_pass "Verify that redacted send correctly detects invalid arguments."
tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh
Executable
+87
View File
@@ -0,0 +1,87 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Test that receiving sends from redaction bookmarks and redacted datasets
# works correctly in certain edge cases.
# 1. Send A(B,C,D) to pool2.
# 2. Verify send from A(B, C, D) can be received onto it.
# 3. Verify send from A(B, C) can be received onto it.
# 4. Verify send from A() can be received onto it.
# 5. Verify send from A(E) cannot be received onto it.
# 6. Verify send from redaction bookmark for A(B, C) can be received onto it.
# 7. Verify send from redaction bookmark for A() can be received onto it.
# 8. Verify send from redaction bookmark for A(E) cannot be received onto it.
#
typeset ds_name="origin"
typeset sendfs="$POOL/$ds_name"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name '' setup_incrementals
typeset dsA=$sendfs@snap0
typeset dsB=$POOL/hole@snap
typeset dsC=$POOL/rm@snap
typeset dsD=$POOL/write@snap
typeset dsE=$POOL/stride3@snap
typeset dsF=$POOL/stride5@snap
typeset targ=$POOL2/targfs@snap
log_onexit redacted_cleanup $sendfs $POOL2/rBCD $POOL2/targfs \
$POOL2/rBC $POOL2/rE
# Set up all the filesystems and clones.
log_must zfs redact $dsA BCD $dsB $dsC $dsD
log_must eval "zfs send --redact BCD $dsA >$stream"
log_must eval "zfs receive $POOL2/rBCD <$stream"
log_must eval "zfs receive $targ <$stream"
log_must zfs redact $dsA BC $dsB $dsC
log_must eval "zfs send --redact BC $dsA >$stream"
log_must eval "zfs receive $POOL2/rBC <$stream"
log_must zfs redact $dsA E $dsE
log_must eval "zfs send --redact E $dsA >$stream"
log_must eval "zfs receive $POOL2/rE <$stream"
log_must eval "zfs send $dsF >$stream"
log_must eval "zfs receive -o origin=$POOL2/rBCD@snap0 $POOL2/BCDrF <$stream"
log_must eval "zfs receive -o origin=$POOL2/rBC@snap0 $POOL2/BCrF <$stream"
log_must eval "zfs receive -o origin=$POOL2/rE@snap0 $POOL2/ErF <$stream"
# Run tests from redacted datasets.
log_must eval "zfs send -i $POOL2/rBCD@snap0 $POOL2/BCDrF@snap >$stream"
log_must eval "zfs receive -o origin=$targ $POOL2/tdBCD <$stream"
log_must eval "zfs send -i $POOL2/rBC@snap0 $POOL2/BCrF@snap >$stream"
log_must eval "zfs receive -o origin=$targ $POOL2/tdBC <$stream"
log_must eval "zfs send -i $POOL2/rE@snap0 $POOL2/ErF@snap >$stream"
log_mustnot eval "zfs receive -o origin=$targ $POOL2/tdE <$stream"
# Run tests from redaction bookmarks.
log_must eval "zfs send -i $sendfs#BC $dsF >$stream"
log_must eval "zfs receive -o origin=$targ $POOL2/tbBC <$stream"
log_must eval "zfs send -i $sendfs#E $dsF >$stream"
log_mustnot eval "zfs receive -o origin=$targ $POOL2/tbE <$stream"
log_pass "Verify sends from redacted datasets and bookmarks work correctly."
tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh
Executable
+77
View File
@@ -0,0 +1,77 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify the list of redacted snapshot guids as properties.
#
# Strategy:
# 1. Create a redacted dataset and receive it into another pool.
# 2. Verify that the redaction list in the book mark (according to zdb)
# matches the list shown in the redact_snaps property.
# 3. Verify that the received snapshot has a matching redaction list.
#
typeset ds_name="props"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name ''
typeset mntpnt
log_onexit redacted_cleanup $sendfs $recvfs
# Verify a plain dataset, snapshot or bookmark has an empty list.
log_must zfs snapshot $sendfs@empty_snapshot
log_must zfs bookmark $sendfs@empty_snapshot $sendfs#empty_bookmark
found_list=$(get_prop redact_snaps $sendfs)
[[ $found_list = "-" ]] || log_fail "Unexpected dataset list: $found_list"
found_list=$(get_prop redact_snaps $sendfs@empty_snapshot)
[[ $found_list = "-" ]] || log_fail "Unexpected snapshot list: $found_list"
found_list=$(get_prop redact_snaps $sendfs#empty_bookmark)
[[ $found_list = "-" ]] || log_fail "Unexpected bookmark list: $found_list"
# Fill in a different block in every clone.
for i in {1..16}; do
log_must zfs clone $sendfs@snap ${clone}$i
mntpnt=$(get_prop mountpoint ${clone}$i)
log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=$i \
conv=notrunc
log_must zfs snapshot ${clone}$i@snap
done
log_must zfs redact $sendfs@snap book1 $clone{1..16}@snap
log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
log_must eval "zfs recv $recvfs <$stream"
get_guid_list $tmpdir/prop_list $sendfs#book1
get_guid_list $tmpdir/zdb_list $sendfs#book1 true
get_guid_list $tmpdir/recvd_prop_list $recvfs@snap
count=$(wc -l $tmpdir/prop_list | awk '{print $1}')
[[ $count -eq 16 ]] || log_fail "Found incorrect number of redaction snapshots."
diff $tmpdir/prop_list $tmpdir/zdb_list || \
log_fail "Property list differed from zdb output"
diff $tmpdir/prop_list $tmpdir/recvd_prop_list || \
log_fail "Received property list differed from sent"
log_pass "The redaction list is consistent between sent and received datasets."
tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh
Executable
+87
View File
@@ -0,0 +1,87 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify that resumable send works correctly with redacted streams.
#
# Strategy:
# 1. Do a full redacted resumable send.
# 2. Verify the received contents are correct.
# 3. Do an incremental redacted resumable send.
# 4. Verify the received contents are correct.
# 5. Verify that recv -A removes a partially received dataset.
#
typeset ds_name="resume"
typeset sendfs="$POOL/$ds_name"
typeset recvfs="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset clone1="$POOL/${ds_name}_clone1"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
setup_dataset $ds_name ''
typeset clone_mnt="$(get_prop mountpoint $clone)"
typeset send_mnt="$(get_prop mountpoint $sendfs)"
typeset recv_mnt="/$POOL2/$ds_name"
log_onexit redacted_cleanup $sendfs $recvfs
log_must stride_dd -i /dev/urandom -o $clone_mnt/f2 -b 512 -c 64 -s 512
log_must zfs snapshot $clone@snap1
# Do the full resumable send
log_must zfs redact $sendfs@snap book1 $clone@snap1
resume_test "zfs send --redact book1 $sendfs@snap" $tmpdir $recvfs
log_must mount_redacted -f $recvfs
log_must set_tunable32 zfs_allow_redacted_dataset_mount 1
log_must diff $send_mnt/f1 $recv_mnt/f1
log_must eval "get_diff $send_mnt/f2 $recv_mnt/f2 >$tmpdir/get_diff.out"
typeset range=$(cat $tmpdir/get_diff.out)
[[ "$RANGE9" = "$range" ]] || log_fail "Unexpected range: $range"
log_must dd if=/dev/urandom of=$send_mnt/f3 bs=1024k count=3
log_must zfs snapshot $sendfs@snap2
log_must zfs clone $sendfs@snap2 $clone1
typeset clone1_mnt="$(get_prop mountpoint $clone1)"
log_must dd if=/dev/urandom of=$clone1_mnt/f3 bs=128k count=3 conv=notrunc
log_must zfs snapshot $clone1@snap
# Do the incremental resumable send
log_must zfs redact $sendfs@snap2 book2 $clone1@snap
resume_test "zfs send --redact book2 -i $sendfs#book1 $sendfs@snap2" \
$tmpdir $recvfs
log_must diff $send_mnt/f1 $recv_mnt/f1
log_must diff $send_mnt/f2 $recv_mnt/f2
log_must eval "get_diff $send_mnt/f3 $recv_mnt/f3 >$tmpdir/get_diff.out"
range=$(cat $tmpdir/get_diff.out)
[[ "$RANGE10" = "$range" ]] || log_fail "Unexpected range: $range"
# Test recv -A works properly
log_mustnot zfs recv -A $recvfs
log_must zfs destroy -R $recvfs
log_mustnot zfs recv -A $recvfs
log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
dd if=$stream bs=64k count=1 | log_mustnot zfs receive -s $recvfs
[[ "-" = $(get_prop receive_resume_token $recvfs) ]] && \
log_fail "Receive token not found."
log_must zfs recv -A $recvfs
log_must datasetnonexists $recvfs
log_pass "Resumable send works correctly with redacted streams."
tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh
Executable
+64
View File
@@ -0,0 +1,64 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify that send size estimates of redacted sends work correctly
#
# Strategy:
# 1. Perform a redacted send with -nv and without, and verify the
# size estimate is the same as the size of the actual send.
# 2. Receive an incremental send from the redaction bookmark with
# -nv and without, and verify the size estimate is the same as
# the size of the actual send.
#
ds_name="sizes"
typeset sendfs="$POOL/$ds_name"
typeset clone="$POOL/${ds_name}_clone2"
setup_dataset $ds_name "-o compress=lz4"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset size=$(mktemp $tmpdir/size.XXXX)
typeset size2=$(mktemp $tmpdir/size.XXXX)
log_onexit redacted_cleanup $sendfs $clone
log_must zfs clone $sendfs@snap $clone
typeset clone_mnt="$(get_prop mountpoint $clone)"
log_must rm -rf $clone_mnt/*
log_must zfs snapshot $clone@snap
log_must zfs redact $sendfs@snap book $clone@snap
log_must eval "zfs send -nvP --redact book $sendfs@snap | \
grep '^size' | awk '{print \$2}' >$size"
log_must eval "zfs send --redact book $sendfs@snap | wc --bytes \
>$size2"
bytes1=$(cat $size | tr -d '[[:space:]]')
bytes2=$(cat $size2 | tr -d '[[:space:]]')
[[ "$bytes1" -eq "$bytes2" ]] || \
log_fail "Full sizes differ: estimate $bytes1 and actual $bytes2"
log_must zfs snapshot $sendfs@snap2
log_must eval "zfs send -nvP -i $sendfs#book $sendfs@snap2 | \
grep '^size' | awk '{print \$2}' >$size"
log_must eval "zfs send -i $sendfs#book $sendfs@snap2 | wc --bytes >$size2"
bytes1=$(cat $size | tr -d '[[:space:]]')
bytes2=$(cat $size2 | tr -d '[[:space:]]')
[[ "$bytes1" -eq "$bytes2" ]] || \
log_fail "Incremental sizes differ: estimate $bytes1 and actual $bytes2"
log_pass "Size estimates of redacted sends estimate accurately."
tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh
Executable
+105
View File
@@ -0,0 +1,105 @@
#!/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
#
# Description:
# Verify that redacted send works on volumes.
#
# Strategy:
# 1. Write to a volume, then make a clone of that volume.
# 2. Receive a redacted stream that sends all blocks.
# 3. Receive a redacted stream that redacts the first half of the written area.
#
typeset ds_name="volume"
typeset sendvol="$POOL/$ds_name"
typeset recvvol="$POOL2/$ds_name"
typeset clone="$POOL/${ds_name}_clone"
typeset tmpdir="$(get_prop mountpoint $POOL)/tmp"
typeset stream=$(mktemp $tmpdir/stream.XXXX)
typeset send_file="/dev/zvol/$sendvol"
typeset recv_file="/dev/zvol/$recvvol"
typeset clone_file="/dev/zvol/$clone"
log_onexit redacted_cleanup $sendvol $recvvol
log_must zfs create -b 8k -V 1g $sendvol
sleep 10
log_must zpool export $POOL
log_must zpool import $POOL
udevadm settle
if [[ ! -b $send_file ]]; then
udevadm settle
for t in 10 5 3 2 1; do
log_note "Polling $t seconds for device file."
udevadm settle
sleep $t
[[ -b $send_file ]] && break
done
fi
log_must dd if=/dev/urandom of=$send_file bs=8k count=64
log_must zfs snapshot $sendvol@snap
log_must zfs clone $sendvol@snap $clone
log_must zfs snapshot $clone@snap
log_must set_tunable32 zfs_allow_redacted_dataset_mount 1
log_must zfs redact $sendvol@snap book1 $clone@snap
log_must eval "zfs send --redact book1 $sendvol@snap >$stream"
log_must eval "zfs recv $recvvol <$stream"
sleep 10
log_must zpool export $POOL2
log_must zpool import $POOL2
udevadm settle
if [[ ! -b $recv_file ]]; then
udevadm settle
for t in 10 5 3 2 1; do
log_note "Polling $t seconds for device file."
udevadm settle
sleep $t
[[ -b $recv_file ]] && break
done
fi
log_must dd if=$send_file of=$tmpdir/send.dd bs=8k count=64
log_must dd if=$recv_file of=$tmpdir/recv.dd bs=8k count=64
log_must diff $tmpdir/send.dd $tmpdir/recv.dd
log_must zfs destroy -R $recvvol
log_must dd if=/dev/urandom of=$clone_file bs=8k count=32
log_must zfs snapshot $clone@snap1
log_must zfs redact $sendvol@snap book2 $clone@snap1
log_must eval "zfs send --redact book2 $sendvol@snap >$stream"
log_must eval "zfs recv $recvvol <$stream"
sleep 10
log_must zpool export $POOL2
log_must zpool import $POOL2
udevadm settle
if [[ ! -b $recv_file ]]; then
udevadm settle
for t in 10 5 3 2 1; do
log_note "Polling $t seconds for device file."
udevadm settle
sleep $t
[[ -b $recv_file ]] && break
done
fi
log_must dd if=$send_file of=$tmpdir/send.dd bs=8k count=32 skip=32
log_must dd if=$recv_file of=$tmpdir/recv.dd bs=8k count=32 skip=32
log_must diff $tmpdir/send.dd $tmpdir/recv.dd
log_pass "Redacted send works correctly with volumes."
tests/zfs-tests/tests/functional/redacted_send/setup.ksh
Executable
+36
View File
@@ -0,0 +1,36 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright (c) 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib
verify_disk_count "$DISKS" 2
create_pool $POOL $DISK1
log_must zfs snapshot $POOL@init
create_pool $POOL2 $DISK2
log_must zfs snapshot $POOL2@init
log_must zfs create $POOL/tmp
log_pass
tests/zfs-tests/tests/functional/rsend/rsend.kshlib
+2 -2
View File
@@ -25,7 +25,7 @@
#
#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
# Copyright (c) 2013, 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
@@ -635,7 +635,7 @@ function resume_test
stream_num=$((stream_num+1))
token=$(zfs get -Hp -o value receive_resume_token $recvfs)
log_must eval "zfs send -v -t $token >/$streamfs/$stream_num"
log_must eval "zfs send -t $token >/$streamfs/$stream_num"
[[ -f /$streamfs/$stream_num ]] || \
log_fail "NO FILE /$streamfs/$stream_num"
done
tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh
Executable
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/ksh
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2014, 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
#
# Description:
# Verify that error conditions don't cause panics in zfs send
#
# Strategy:
# 1. Perform a zfs incremental send from a bookmark that doesn't exist
#
verify_runnable "both"
log_neg eval "zfs send -i \#bla $POOl/$FS@final > /dev/null"
log_pass "Ensure that error conditions cause appropriate failures."