Add more DDT tests

The new Fast Dedup feature has a lot of moving parts, and only some of
them have tests. We have some tests for prefetch and quota, and a
generic ZAP shrinking test, but we don't have anything for the pruning
command or specific to DDT zap shrinking. Here we add a couple small new
tests for zpool ddtprune and DDT-specific ZAP shrinking.

Sponsored-by: Klara, Inc.
Sponsored-by: iXsystems, Inc.
Reviewed-by: Allan Jude <allan@klarasystems.com>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Paul Dagnelie <paul.dagnelie@klarasystems.com>
Closes #17049

cmd/zdb/zdb.c

@@ -2059,6 +2059,8 @@ dump_ddt_object(ddt_t *ddt, ddt_type_t type, ddt_class_t class)
 	if (dump_opt['D'] < 3)
 		return;
 
+	(void) printf("%s: object=%llu\n", name,
+	    (u_longlong_t)ddt->ddt_object[type][class]);
 	zpool_dump_ddt(NULL, &ddt->ddt_histogram[type][class]);
 
 	if (dump_opt['D'] < 4)

tests/runfiles/common.run

@@ -683,7 +683,7 @@ tags = ['functional', 'deadman']
 [tests/functional/dedup]
 tests = ['dedup_fdt_create', 'dedup_fdt_import', 'dedup_legacy_create',
     'dedup_legacy_import', 'dedup_legacy_fdt_upgrade',
-    'dedup_legacy_fdt_mixed', 'dedup_quota']
+    'dedup_legacy_fdt_mixed', 'dedup_quota', 'dedup_prune', 'dedup_zap_shrink']
 pre =
 post =
 tags = ['functional', 'dedup']

tests/zfs-tests/tests/Makefile.am

@@ -1444,7 +1444,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/dedup/dedup_legacy_import.ksh \
 	functional/dedup/dedup_legacy_fdt_upgrade.ksh \
 	functional/dedup/dedup_legacy_fdt_mixed.ksh \
+	functional/dedup/dedup_prune.ksh \
 	functional/dedup/dedup_quota.ksh \
+	functional/dedup/dedup_zap_shrink.ksh \
 	functional/delegate/cleanup.ksh \
 	functional/delegate/setup.ksh \
 	functional/delegate/zfs_allow_001_pos.ksh \

tests/zfs-tests/tests/functional/dedup/dedup_prune.ksh

@@ -0,0 +1,96 @@
+#!/bin/ksh -p
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara Inc.
+#
+
+# DESCRIPTION:
+#	Verify that zpool ddtprune successfully reduces the number of entries
+#	in the DDT.
+#
+# STRATEGY:
+#	1. Create a pool with dedup=on
+#	2. Add duplicate entries to the DDT
+#	3. Verify ddtprune doesn't remove duplicate entries
+#	4. Convert some entries to non-duplicate
+#	5. Verify ddtprune removes non-duplicate entries
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+log_assert "Verify DDT pruning correctly removes non-duplicate entries"
+
+# We set the dedup log txg interval to 1, to get a log flush every txg,
+# effectively disabling the log. Without this it's hard to predict when
+# entries appear in the DDT ZAP
+log_must save_tunable DEDUP_LOG_TXG_MAX
+log_must set_tunable32 DEDUP_LOG_TXG_MAX 1
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+	log_must restore_tunable DEDUP_LOG_TXG_MAX
+}
+
+function ddt_entries
+{
+	typeset -i entries=$(zpool status -D $TESTPOOL | \
+		grep "dedup: DDT entries" | awk '{print $4}')
+
+	echo ${entries}
+}
+
+log_onexit cleanup
+
+log_must zpool create -f -o feature@block_cloning=disabled $TESTPOOL $DISKS
+
+log_must zfs create -o recordsize=512 -o dedup=on $TESTPOOL/$TESTFS
+typeset mountpoint=$(get_prop mountpoint $TESTPOOL/$TESTFS)
+log_must dd if=/dev/urandom of=$mountpoint/f1 bs=512k count=1
+log_must cp $mountpoint/f1 $mountpoint/f2
+sync_pool $TESTPOOL
+entries=$(ddt_entries)
+log_note "ddt entries before: $entries"
+
+log_must zpool ddtprune -p 100 $TESTPOOL
+sync_pool $TESTPOOL
+new_entries=$(ddt_entries)
+[[ "$entries" -eq "$new_entries" ]] || \
+	log_fail "DDT entries changed from $entries to $new_entries"
+
+log_must truncate -s 128k $mountpoint/f2
+sync_pool $TESTPOOL
+sleep 1
+log_must zpool ddtprune -p 100 $TESTPOOL
+sync_pool $TESTPOOL
+
+new_entries=$(ddt_entries)
+[[ "$((entries / 4))" -eq "$new_entries" ]] || \
+	log_fail "DDT entries did not shrink enough: $entries -> $new_entries"
+
+
+log_pass "DDT pruning correctly removes non-duplicate entries"

tests/zfs-tests/tests/functional/dedup/dedup_zap_shrink.ksh

@@ -0,0 +1,82 @@
+#! /bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2024, 2025, Klara Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# Create a large number of entries in the DDT. Then remove all entries and
+# check that the DDT zap was shrunk. Use zdb to check that the zap object
+# contains only one leaf block using zdb.
+#
+
+verify_runnable "global"
+
+log_assert "Create a large number of entries in the DDT. " \
+	"Ensure DDT ZAP object shrank after removing entries."
+
+# We set the dedup log txg interval to 1, to get a log flush every txg,
+# effectively disabling the log. Without this it's hard to predict when
+# entries appear in the DDT ZAP
+log_must save_tunable DEDUP_LOG_TXG_MAX
+log_must set_tunable32 DEDUP_LOG_TXG_MAX 1
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+	log_must restore_tunable DEDUP_LOG_TXG_MAX
+}
+
+log_onexit cleanup
+
+log_must create_pool $TESTPOOL $DISKS
+log_must zfs create -o dedup=sha256 -o recordsize=512 $TESTPOOL/$TESTFS
+typeset mountpoint=$(get_prop mountpoint $TESTPOOL/$TESTFS)
+
+log_must dd if=/dev/urandom of=$mountpoint/file bs=512k count=1
+sync_pool $TESTPOOL
+
+zap_obj=$(zdb -DDD $TESTPOOL | grep "DDT-sha256-zap-unique" | sed -n 's/.*object=//p')
+
+nleafs=$(zdb -dddd $TESTPOOL "$zap_obj" | grep "Leaf blocks:" | awk -F\: '{print($2);}')
+log_must test 1 -lt $nleafs
+
+nleafs_old=$nleafs
+
+log_must truncate -s 512 $mountpoint/file
+sync_pool $TESTPOOL
+nleafs=$(zdb -dddd $TESTPOOL "$zap_obj" | grep "Leaf blocks:" | awk -F\: '{print($2);}')
+log_must test $nleafs -lt $nleafs_old
+
+log_must zpool export $TESTPOOL
+log_must zpool import $TESTPOOL
+
+nleafs=$(zdb -dddd $TESTPOOL "$zap_obj" | grep "Leaf blocks:" | awk -F\: '{print($2);}')
+log_must test $nleafs -lt $nleafs_old
+
+log_pass "ZAP object shrank after removing entries."