From fc460bfbaf5bf869b694feba39c5bc30400ae3ea Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Tue, 4 Mar 2025 17:02:34 -0800 Subject: [PATCH] Add more DDT tests The new Fast Dedup feature has a lot of moving parts, and only some of them have tests. We have some tests for prefetch and quota, and a generic ZAP shrinking test, but we don't have anything for the pruning command or specific to DDT zap shrinking. Here we add a couple small new tests for zpool ddtprune and DDT-specific ZAP shrinking. Sponsored-by: Klara, Inc. Sponsored-by: iXsystems, Inc. Reviewed-by: Allan Jude Reviewed-by: Alexander Motin Reviewed-by: Tony Hutter Signed-off-by: Paul Dagnelie Closes #17049 --- cmd/zdb/zdb.c | 2 + tests/runfiles/common.run | 2 +- tests/zfs-tests/tests/Makefile.am | 2 + .../tests/functional/dedup/dedup_prune.ksh | 96 +++++++++++++++++++ .../functional/dedup/dedup_zap_shrink.ksh | 82 ++++++++++++++++ 5 files changed, 183 insertions(+), 1 deletion(-) create mode 100755 tests/zfs-tests/tests/functional/dedup/dedup_prune.ksh create mode 100755 tests/zfs-tests/tests/functional/dedup/dedup_zap_shrink.ksh diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index fbc92833e..d594cd112 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -2059,6 +2059,8 @@ dump_ddt_object(ddt_t *ddt, ddt_type_t type, ddt_class_t class) if (dump_opt['D'] < 3) return; + (void) printf("%s: object=%llu\n", name, + (u_longlong_t)ddt->ddt_object[type][class]); zpool_dump_ddt(NULL, &ddt->ddt_histogram[type][class]); if (dump_opt['D'] < 4) diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index efc0290de..462704b59 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -683,7 +683,7 @@ tags = ['functional', 'deadman'] [tests/functional/dedup] tests = ['dedup_fdt_create', 'dedup_fdt_import', 'dedup_legacy_create', 'dedup_legacy_import', 'dedup_legacy_fdt_upgrade', - 'dedup_legacy_fdt_mixed', 'dedup_quota'] + 'dedup_legacy_fdt_mixed', 'dedup_quota', 'dedup_prune', 'dedup_zap_shrink'] pre = post = tags = ['functional', 'dedup'] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index cb53913c7..6e34063ad 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -1444,7 +1444,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/dedup/dedup_legacy_import.ksh \ functional/dedup/dedup_legacy_fdt_upgrade.ksh \ functional/dedup/dedup_legacy_fdt_mixed.ksh \ + functional/dedup/dedup_prune.ksh \ functional/dedup/dedup_quota.ksh \ + functional/dedup/dedup_zap_shrink.ksh \ functional/delegate/cleanup.ksh \ functional/delegate/setup.ksh \ functional/delegate/zfs_allow_001_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/dedup/dedup_prune.ksh b/tests/zfs-tests/tests/functional/dedup/dedup_prune.ksh new file mode 100755 index 000000000..44dbecafd --- /dev/null +++ b/tests/zfs-tests/tests/functional/dedup/dedup_prune.ksh @@ -0,0 +1,96 @@ +#!/bin/ksh -p +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara Inc. +# + +# DESCRIPTION: +# Verify that zpool ddtprune successfully reduces the number of entries +# in the DDT. +# +# STRATEGY: +# 1. Create a pool with dedup=on +# 2. Add duplicate entries to the DDT +# 3. Verify ddtprune doesn't remove duplicate entries +# 4. Convert some entries to non-duplicate +# 5. Verify ddtprune removes non-duplicate entries +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/events/events_common.kshlib + +verify_runnable "both" + +log_assert "Verify DDT pruning correctly removes non-duplicate entries" + +# We set the dedup log txg interval to 1, to get a log flush every txg, +# effectively disabling the log. Without this it's hard to predict when +# entries appear in the DDT ZAP +log_must save_tunable DEDUP_LOG_TXG_MAX +log_must set_tunable32 DEDUP_LOG_TXG_MAX 1 + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + log_must restore_tunable DEDUP_LOG_TXG_MAX +} + +function ddt_entries +{ + typeset -i entries=$(zpool status -D $TESTPOOL | \ + grep "dedup: DDT entries" | awk '{print $4}') + + echo ${entries} +} + +log_onexit cleanup + +log_must zpool create -f -o feature@block_cloning=disabled $TESTPOOL $DISKS + +log_must zfs create -o recordsize=512 -o dedup=on $TESTPOOL/$TESTFS +typeset mountpoint=$(get_prop mountpoint $TESTPOOL/$TESTFS) +log_must dd if=/dev/urandom of=$mountpoint/f1 bs=512k count=1 +log_must cp $mountpoint/f1 $mountpoint/f2 +sync_pool $TESTPOOL +entries=$(ddt_entries) +log_note "ddt entries before: $entries" + +log_must zpool ddtprune -p 100 $TESTPOOL +sync_pool $TESTPOOL +new_entries=$(ddt_entries) +[[ "$entries" -eq "$new_entries" ]] || \ + log_fail "DDT entries changed from $entries to $new_entries" + +log_must truncate -s 128k $mountpoint/f2 +sync_pool $TESTPOOL +sleep 1 +log_must zpool ddtprune -p 100 $TESTPOOL +sync_pool $TESTPOOL + +new_entries=$(ddt_entries) +[[ "$((entries / 4))" -eq "$new_entries" ]] || \ + log_fail "DDT entries did not shrink enough: $entries -> $new_entries" + + +log_pass "DDT pruning correctly removes non-duplicate entries" diff --git a/tests/zfs-tests/tests/functional/dedup/dedup_zap_shrink.ksh b/tests/zfs-tests/tests/functional/dedup/dedup_zap_shrink.ksh new file mode 100755 index 000000000..5f2352937 --- /dev/null +++ b/tests/zfs-tests/tests/functional/dedup/dedup_zap_shrink.ksh @@ -0,0 +1,82 @@ +#! /bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2024, 2025, Klara Inc. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# Create a large number of entries in the DDT. Then remove all entries and +# check that the DDT zap was shrunk. Use zdb to check that the zap object +# contains only one leaf block using zdb. +# + +verify_runnable "global" + +log_assert "Create a large number of entries in the DDT. " \ + "Ensure DDT ZAP object shrank after removing entries." + +# We set the dedup log txg interval to 1, to get a log flush every txg, +# effectively disabling the log. Without this it's hard to predict when +# entries appear in the DDT ZAP +log_must save_tunable DEDUP_LOG_TXG_MAX +log_must set_tunable32 DEDUP_LOG_TXG_MAX 1 + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + log_must restore_tunable DEDUP_LOG_TXG_MAX +} + +log_onexit cleanup + +log_must create_pool $TESTPOOL $DISKS +log_must zfs create -o dedup=sha256 -o recordsize=512 $TESTPOOL/$TESTFS +typeset mountpoint=$(get_prop mountpoint $TESTPOOL/$TESTFS) + +log_must dd if=/dev/urandom of=$mountpoint/file bs=512k count=1 +sync_pool $TESTPOOL + +zap_obj=$(zdb -DDD $TESTPOOL | grep "DDT-sha256-zap-unique" | sed -n 's/.*object=//p') + +nleafs=$(zdb -dddd $TESTPOOL "$zap_obj" | grep "Leaf blocks:" | awk -F\: '{print($2);}') +log_must test 1 -lt $nleafs + +nleafs_old=$nleafs + +log_must truncate -s 512 $mountpoint/file +sync_pool $TESTPOOL +nleafs=$(zdb -dddd $TESTPOOL "$zap_obj" | grep "Leaf blocks:" | awk -F\: '{print($2);}') +log_must test $nleafs -lt $nleafs_old + +log_must zpool export $TESTPOOL +log_must zpool import $TESTPOOL + +nleafs=$(zdb -dddd $TESTPOOL "$zap_obj" | grep "Leaf blocks:" | awk -F\: '{print($2);}') +log_must test $nleafs -lt $nleafs_old + +log_pass "ZAP object shrank after removing entries."