Add --enable-asan and --enable-ubsan switches

`configure` now accepts `--enable-asan` and `--enable-ubsan` switches
which results in passing `-fsanitize=address`
and `-fsanitize=undefined`, respectively, to the compiler. Those
flags are enabled in GitHub workflows for ZTS and zloop. Errors
reported by both instrumentations are corrected, except for:

- Memory leak reporting is (temporarily) suppressed. The cost of
  fixing them is relatively high compared to the gains.

- Checksum computing functions in `module/zcommon/zfs_fletcher*`
  have UBSan errors suppressed. It is completely impractical
  to enforce 64-byte payload alignment there due to performance
  impact.

- There's no ASan heap poisoning in `module/zstd/lib/zstd.c`. A custom
  memory allocator is used there rendering that measure
  unfeasible.

- Memory leaks detection has to be suppressed for `cmd/zvol_id`.
  `zvol_id` is run by udev with the help of `ptrace(2)`. Tracing is
  incompatible with memory leaks detection.

Reviewed-by: Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
Reviewed-by: George Melikov <mail@gmelikov.ru>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: szubersk <szuberskidamian@gmail.com>
Closes #12928

config/Rules.am

@@ -29,6 +29,7 @@ AM_CFLAGS += $(NO_OMIT_FRAME_POINTER)
 AM_CFLAGS += $(IMPLICIT_FALLTHROUGH)
 AM_CFLAGS += $(DEBUG_CFLAGS)
 AM_CFLAGS += $(ASAN_CFLAGS)
+AM_CFLAGS += $(UBSAN_CFLAGS)
 AM_CFLAGS += $(CODE_COVERAGE_CFLAGS) $(NO_FORMAT_ZERO_LENGTH)
 if BUILD_FREEBSD
 AM_CFLAGS += -fPIC -Werror -Wno-unknown-pragmas -Wno-enum-conversion
@@ -58,8 +59,17 @@ AM_CPPFLAGS += -D"__xpg_basename(...)=__xpg_basename(__VA_ARGS__) __attribute__(
 AM_CPPFLAGS += -D"basename(...)=basename(__VA_ARGS__) __attribute__((deprecated(\"basename(3) is underspecified. Use zfs_basename() instead!\")))"
 AM_CPPFLAGS += -D"dirname(...)=dirname(__VA_ARGS__) __attribute__((deprecated(\"dirname(3) is underspecified. Use zfs_dirnamelen() instead!\")))"
 
+if ASAN_ENABLED
+AM_CPPFLAGS += -DZFS_ASAN_ENABLED
+endif
+
+if UBSAN_ENABLED
+AM_CPPFLAGS += -DZFS_UBSAN_ENABLED
+endif
+
 AM_LDFLAGS  = $(DEBUG_LDFLAGS)
 AM_LDFLAGS += $(ASAN_LDFLAGS)
+AM_LDFLAGS += $(UBSAN_LDFLAGS)
 
 if BUILD_FREEBSD
 AM_LDFLAGS += -fstack-protector-strong -shared

config/Substfiles.am

@@ -17,7 +17,9 @@ subst_sed_cmd = \
 	-e 's|@DEFAULT_INIT_NFS_SERVER[@]|$(DEFAULT_INIT_NFS_SERVER)|g' \
 	-e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g' \
 	-e 's|@LIBFETCH_DYNAMIC[@]|$(LIBFETCH_DYNAMIC)|g' \
-	-e 's|@LIBFETCH_SONAME[@]|$(LIBFETCH_SONAME)|g'
+	-e 's|@LIBFETCH_SONAME[@]|$(LIBFETCH_SONAME)|g' \
+	-e 's|@ASAN_ENABLED[@]|$(ASAN_ENABLED)|g' \
+	-e 's|@UBSAN_ENABLED[@]|$(UBSAN_ENABLED)|g'
 
 SUBSTFILES =
 CLEANFILES = $(SUBSTFILES)

config/always-compiler-options.m4

@@ -45,6 +45,53 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_ASAN], [
 	AC_SUBST([ASAN_ZFS])
 ])
 
+dnl #
+dnl # Enabled -fsanitize=undefined if supported by gcc.
+dnl #
+dnl # LDFLAGS needs -fsanitize=undefined at all times so libraries compiled with
+dnl # it will be linked successfully. CFLAGS will vary by binary being built.
+dnl #
+dnl # The UBSAN_OPTIONS environment variable can be used to further control
+dnl # the behavior of binaries and libraries build with -fsanitize=undefined.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_UBSAN], [
+	AC_MSG_CHECKING([whether to build with -fsanitize=undefined support])
+	AC_ARG_ENABLE([ubsan],
+		[AS_HELP_STRING([--enable-ubsan],
+		[Enable -fsanitize=undefined support  @<:@default=no@:>@])],
+		[],
+		[enable_ubsan=no])
+
+	AM_CONDITIONAL([UBSAN_ENABLED], [test x$enable_ubsan = xyes])
+	AC_SUBST([UBSAN_ENABLED], [$enable_ubsan])
+	AC_MSG_RESULT($enable_ubsan)
+
+	AS_IF([ test "$enable_ubsan" = "yes" ], [
+		AC_MSG_CHECKING([whether $CC supports -fsanitize=undefined])
+		saved_cflags="$CFLAGS"
+		CFLAGS="$CFLAGS -Werror -fsanitize=undefined"
+		AC_LINK_IFELSE([
+			AC_LANG_SOURCE([[ int main() { return 0; } ]])
+		], [
+			UBSAN_CFLAGS="-fsanitize=undefined"
+			UBSAN_LDFLAGS="-fsanitize=undefined"
+			UBSAN_ZFS="_with_ubsan"
+			AC_MSG_RESULT([yes])
+		], [
+			AC_MSG_ERROR([$CC does not support -fsanitize=undefined])
+		])
+		CFLAGS="$saved_cflags"
+	], [
+		UBSAN_CFLAGS=""
+		UBSAN_LDFLAGS=""
+		UBSAN_ZFS="_without_ubsan"
+	])
+
+	AC_SUBST([UBSAN_CFLAGS])
+	AC_SUBST([UBSAN_LDFLAGS])
+	AC_SUBST([UBSAN_ZFS])
+])
+
 dnl #
 dnl # Check if gcc supports -Wframe-larger-than=<size> option.
 dnl #

config/zfs-build.m4

@@ -218,6 +218,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_OMIT_FRAME_POINTER
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_IPA_SRA
 	ZFS_AC_CONFIG_ALWAYS_CC_ASAN
+	ZFS_AC_CONFIG_ALWAYS_CC_UBSAN
 	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
 	ZFS_AC_CONFIG_ALWAYS_SYSTEM
 	ZFS_AC_CONFIG_ALWAYS_ARCH
@@ -323,6 +324,7 @@ AC_DEFUN([ZFS_AC_RPM], [
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUG_KMEM_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUG_KMEM_TRACKING_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(ASAN_ZFS) 1"'
+	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(UBSAN_ZFS) 1"'
 
 	RPM_DEFINE_UTIL=' --define "_initconfdir $(initconfdir)"'