Add --enable-asan and --enable-ubsan switches

`configure` now accepts `--enable-asan` and `--enable-ubsan` switches
which results in passing `-fsanitize=address`
and `-fsanitize=undefined`, respectively, to the compiler. Those
flags are enabled in GitHub workflows for ZTS and zloop. Errors
reported by both instrumentations are corrected, except for:

- Memory leak reporting is (temporarily) suppressed. The cost of
  fixing them is relatively high compared to the gains.

- Checksum computing functions in `module/zcommon/zfs_fletcher*`
  have UBSan errors suppressed. It is completely impractical
  to enforce 64-byte payload alignment there due to performance
  impact.

- There's no ASan heap poisoning in `module/zstd/lib/zstd.c`. A custom
  memory allocator is used there rendering that measure
  unfeasible.

- Memory leaks detection has to be suppressed for `cmd/zvol_id`.
  `zvol_id` is run by udev with the help of `ptrace(2)`. Tracing is
  incompatible with memory leaks detection.

Reviewed-by: Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
Reviewed-by: George Melikov <mail@gmelikov.ru>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: szubersk <szuberskidamian@gmail.com>
Closes #12928

tests/zfs-tests/include/commands.cfg

@@ -51,6 +51,7 @@ export SYSTEM_FILES_COMMON='arp
     iostat
     kill
     ksh
+    ldd
     ln
     logname
     ls

tests/zfs-tests/include/default.cfg.in

@@ -155,6 +155,21 @@ done
 
 export MAX_PARTITIONS=8
 
+if [ "@ASAN_ENABLED@" = "yes" ]; then
+	export ASAN_OPTIONS=abort_on_error=true:halt_on_error=true:allocator_may_return_null=true:disable_coredump=false:detect_stack_use_after_return=true
+
+	# TODO
+	# disable memory leaks detection
+	# there are quite many of them and they are not as
+	# destructive to CLI programs as they are to daemons
+	export ASAN_OPTIONS="$ASAN_OPTIONS:detect_leaks=false"
+fi
+
+if [ "@UBSAN_ENABLED@" = "yes" ]; then
+	export UBSAN_OPTIONS=abort_on_error=true:halt_on_error=true:print_stacktrace=true
+fi
+
+
 case $(uname -o) in
 GNU/Linux)
 	unpack_opts="--sparse -xf"

tests/zfs-tests/tests/functional/alloc_class/alloc_class_013_pos.ksh

@@ -41,8 +41,8 @@ log_must display_status "$TESTPOOL"
 #
 
 log_must zfs create -o dedup=on -V 2G $TESTPOOL/$TESTVOL
-
-log_must eval "new_fs $ZVOL_DEVDIR/$TESTPOOL/$TESTVOL >/dev/null 2>&1"
+block_device_wait "$ZVOL_DEVDIR/$TESTPOOL/$TESTVOL"
+log_must eval "new_fs $ZVOL_DEVDIR/$TESTPOOL/$TESTVOL >/dev/null"
 
 sync_pool
 log_must zpool list -v $TESTPOOL

tests/zfs-tests/tests/functional/cli_root/zfs/zfs_002_pos.ksh

@@ -97,7 +97,6 @@ if is_linux; then
 	ulimit -c unlimited
 	echo "$corefile" >/proc/sys/kernel/core_pattern
 	echo 0 >/proc/sys/kernel/core_uses_pid
-	export ASAN_OPTIONS="abort_on_error=1:disable_coredump=0"
 elif is_freebsd; then
 	ulimit -c unlimited
 	savedcorefile=$(sysctl -n kern.corefile)

tests/zfs-tests/tests/functional/cli_root/zpool/zpool_002_pos.ksh

@@ -95,7 +95,6 @@ elif is_freebsd; then
 fi
 ulimit -c unlimited
 
-export ASAN_OPTIONS="abort_on_error=1:disable_coredump=0"
 export ZFS_ABORT=yes
 
 for subcmd in "${cmds[@]}" "${badparams[@]}"; do

tests/zfs-tests/tests/functional/cli_root/zpool/zpool_003_pos.ksh

@@ -89,7 +89,6 @@ elif is_freebsd; then
 fi
 ulimit -c unlimited
 
-export ASAN_OPTIONS="abort_on_error=1:disable_coredump=0"
 export ZFS_ABORT=yes
 
 zpool >/dev/null 2>&1

tests/zfs-tests/tests/functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_pos.ksh

@@ -35,7 +35,7 @@
 DISK1=${DISKS%% *}
 
 log_must zpool create -f $TESTPOOL $DISK1
-log_must zpool trim $TESTPOOL
+log_must zpool trim -r 1 "$TESTPOOL"
 
 [[ -z "$(trim_progress $TESTPOOL $DISK1)" ]] && \
     log_fail "TRIM did not start"

tests/zfs-tests/tests/functional/pam/pam_basic.ksh

@@ -22,6 +22,10 @@
 
 . $STF_SUITE/tests/functional/pam/utilities.kshlib
 
+if [ -n "$ASAN_OPTIONS" ]; then
+	export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}')
+fi
+
 log_mustnot ismounted "$TESTPOOL/pam/${username}"
 keystatus unavailable
 

tests/zfs-tests/tests/functional/pam/pam_nounmount.ksh

@@ -22,6 +22,10 @@
 
 . $STF_SUITE/tests/functional/pam/utilities.kshlib
 
+if [ -n "$ASAN_OPTIONS" ]; then
+	export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}')
+fi
+
 log_mustnot ismounted "$TESTPOOL/pam/${username}"
 keystatus unavailable
 

tests/zfs-tests/tests/functional/pam/pam_short_password.ksh

@@ -27,6 +27,10 @@
 
 . $STF_SUITE/tests/functional/pam/utilities.kshlib
 
+if [ -n "$ASAN_OPTIONS" ]; then
+	export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}')
+fi
+
 if [[ -z pamservice ]]; then
 	pamservice=pam_zfs_key_test
 fi

tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in

@@ -16,6 +16,10 @@
 
 . $STF_SUITE/include/libtest.shlib
 
+if [ -n "$ASAN_OPTIONS" ]; then
+	export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}')
+fi
+
 #
 # DESCRIPTION:
 #	Verify the libzfs_core Python test suite can be run successfully

tests/zfs-tests/tests/functional/tmpfile/tmpfile_002_pos.c

@@ -58,7 +58,7 @@ main(void)
 		exit(3);
 	}
 
-	if ((ret = system("sudo zpool freeze $TESTPOOL"))) {
+	if ((ret = system("sudo -E zpool freeze $TESTPOOL"))) {
 		if (ret == -1)
 			perror("system \"zpool freeze\"");
 		else
@@ -69,7 +69,7 @@ main(void)
 
 	close(fd);
 
-	if ((ret = system("sudo zpool export $TESTPOOL"))) {
+	if ((ret = system("sudo -E zpool export $TESTPOOL"))) {
 		if (ret == -1)
 			perror("system \"zpool export\"");
 		else
@@ -78,7 +78,7 @@ main(void)
 		exit(4);
 	}
 
-	if ((ret = system("sudo zpool import $TESTPOOL"))) {
+	if ((ret = system("sudo -E zpool import $TESTPOOL"))) {
 		if (ret == -1)
 			perror("system \"zpool import\"");
 		else