Add --enable-asan and --enable-ubsan switches

`configure` now accepts `--enable-asan` and `--enable-ubsan` switches
which results in passing `-fsanitize=address`
and `-fsanitize=undefined`, respectively, to the compiler. Those
flags are enabled in GitHub workflows for ZTS and zloop. Errors
reported by both instrumentations are corrected, except for:

- Memory leak reporting is (temporarily) suppressed. The cost of
  fixing them is relatively high compared to the gains.

- Checksum computing functions in `module/zcommon/zfs_fletcher*`
  have UBSan errors suppressed. It is completely impractical
  to enforce 64-byte payload alignment there due to performance
  impact.

- There's no ASan heap poisoning in `module/zstd/lib/zstd.c`. A custom
  memory allocator is used there rendering that measure
  unfeasible.

- Memory leaks detection has to be suppressed for `cmd/zvol_id`.
  `zvol_id` is run by udev with the help of `ptrace(2)`. Tracing is
  incompatible with memory leaks detection.

Reviewed-by: Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
Reviewed-by: George Melikov <mail@gmelikov.ru>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: szubersk <szuberskidamian@gmail.com>
Closes #12928

module/icp/algs/modes/gcm.c

@@ -348,8 +348,14 @@ gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
 			ctx->gcm_pt_buf = NULL;
 			return (CRYPTO_HOST_MEMORY);
 		}
-		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
-		vmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
+
+		if (ctx->gcm_pt_buf != NULL) {
+			bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
+			vmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
+		} else {
+			ASSERT0(ctx->gcm_pt_buf_len);
+		}
+
 		ctx->gcm_pt_buf = new;
 		ctx->gcm_pt_buf_len = new_len;
 		bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len],
@@ -554,8 +560,15 @@ gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len,
 			 * There's not a block full of data, pad rest of
 			 * buffer with zero
 			 */
-			bzero(authp, block_size);
-			bcopy(&(auth_data[processed]), authp, remainder);
+
+			if (auth_data != NULL) {
+				bzero(authp, block_size);
+				bcopy(&(auth_data[processed]),
+				    authp, remainder);
+			} else {
+				ASSERT0(remainder);
+			}
+
 			datap = (uint8_t *)authp;
 			remainder = 0;
 		} else {

module/icp/io/sha2_mod.c

@@ -710,8 +710,13 @@ sha2_mac_init_ctx(sha2_hmac_ctx_t *ctx, void *keyval, uint_t length_in_bytes)
 
 	(void) bzero(ipad, block_size);
 	(void) bzero(opad, block_size);
-	(void) bcopy(keyval, ipad, length_in_bytes);
-	(void) bcopy(keyval, opad, length_in_bytes);
+
+	if (keyval != NULL) {
+		(void) bcopy(keyval, ipad, length_in_bytes);
+		(void) bcopy(keyval, opad, length_in_bytes);
+	} else {
+		ASSERT0(length_in_bytes);
+	}
 
 	/* XOR key with ipad (0x36) and opad (0x5c) */
 	for (i = 0; i < blocks_per_int64; i ++) {

module/zcommon/zfs_fletcher.c

@@ -300,18 +300,21 @@ fletcher_2_byteswap(const void *buf, uint64_t size,
 	(void) fletcher_2_incremental_byteswap((void *) buf, size, zcp);
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_scalar_init(fletcher_4_ctx_t *ctx)
 {
 	ZIO_SET_CHECKSUM(&ctx->scalar, 0, 0, 0, 0);
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 {
 	memcpy(zcp, &ctx->scalar, sizeof (zio_cksum_t));
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_scalar_native(fletcher_4_ctx_t *ctx, const void *buf,
     uint64_t size)
@@ -335,6 +338,7 @@ fletcher_4_scalar_native(fletcher_4_ctx_t *ctx, const void *buf,
 	ZIO_SET_CHECKSUM(&ctx->scalar, a, b, c, d);
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_scalar_byteswap(fletcher_4_ctx_t *ctx, const void *buf,
     uint64_t size)

module/zcommon/zfs_fletcher_aarch64_neon.c

@@ -48,12 +48,14 @@
 #include <sys/strings.h>
 #include <zfs_fletcher.h>
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_aarch64_neon_init(fletcher_4_ctx_t *ctx)
 {
 	bzero(ctx->aarch64_neon, 4 * sizeof (zfs_fletcher_aarch64_neon_t));
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_aarch64_neon_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 {

module/zcommon/zfs_fletcher_avx512.c

@@ -35,12 +35,14 @@
 #define	__asm __asm__ __volatile__
 #endif
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_avx512f_init(fletcher_4_ctx_t *ctx)
 {
 	bzero(ctx->avx512, 4 * sizeof (zfs_fletcher_avx512_t));
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_avx512f_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 {

module/zcommon/zfs_fletcher_intel.c

@@ -47,12 +47,14 @@
 #include <sys/strings.h>
 #include <zfs_fletcher.h>
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_avx2_init(fletcher_4_ctx_t *ctx)
 {
 	bzero(ctx->avx, 4 * sizeof (zfs_fletcher_avx_t));
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_avx2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 {

module/zcommon/zfs_fletcher_sse.c

@@ -49,12 +49,14 @@
 #include <sys/strings.h>
 #include <zfs_fletcher.h>
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_sse2_init(fletcher_4_ctx_t *ctx)
 {
 	bzero(ctx->sse, 4 * sizeof (zfs_fletcher_sse_t));
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_sse2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 {

module/zcommon/zfs_fletcher_superscalar.c

@@ -47,12 +47,14 @@
 #include <sys/strings.h>
 #include <zfs_fletcher.h>
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_superscalar_init(fletcher_4_ctx_t *ctx)
 {
 	bzero(ctx->superscalar, 4 * sizeof (zfs_fletcher_superscalar_t));
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_superscalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 {
@@ -68,6 +70,7 @@ fletcher_4_superscalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 	ZIO_SET_CHECKSUM(zcp, A, B, C, D);
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_superscalar_native(fletcher_4_ctx_t *ctx,
     const void *buf, uint64_t size)
@@ -107,6 +110,7 @@ fletcher_4_superscalar_native(fletcher_4_ctx_t *ctx,
 	ctx->superscalar[3].v[1] = d2;
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_superscalar_byteswap(fletcher_4_ctx_t *ctx,
     const void *buf, uint64_t size)

module/zcommon/zfs_fletcher_superscalar4.c

@@ -47,12 +47,14 @@
 #include <sys/strings.h>
 #include <zfs_fletcher.h>
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_superscalar4_init(fletcher_4_ctx_t *ctx)
 {
 	bzero(ctx->superscalar, 4 * sizeof (zfs_fletcher_superscalar_t));
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_superscalar4_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 {
@@ -82,6 +84,7 @@ fletcher_4_superscalar4_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
 	ZIO_SET_CHECKSUM(zcp, A, B, C, D);
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
     const void *buf, uint64_t size)
@@ -147,6 +150,7 @@ fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
 	ctx->superscalar[3].v[3] = d4;
 }
 
+ZFS_NO_SANITIZE_UNDEFINED
 static void
 fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx,
     const void *buf, uint64_t size)

module/zfs/btree.c

@@ -536,7 +536,6 @@ zfs_btree_insert_into_parent(zfs_btree_t *tree, zfs_btree_hdr_t *old_node,
 	ASSERT3P(old_node->bth_parent, ==, new_node->bth_parent);
 	uint64_t size = tree->bt_elem_size;
 	zfs_btree_core_t *parent = old_node->bth_parent;
-	zfs_btree_hdr_t *par_hdr = &parent->btc_hdr;
 
 	/*
 	 * If this is the root node we were splitting, we create a new root
@@ -568,6 +567,7 @@ zfs_btree_insert_into_parent(zfs_btree_t *tree, zfs_btree_hdr_t *old_node,
 	 * Since we have the new separator, binary search for where to put
 	 * new_node.
 	 */
+	zfs_btree_hdr_t *par_hdr = &parent->btc_hdr;
 	zfs_btree_index_t idx;
 	ASSERT(par_hdr->bth_core);
 	VERIFY3P(zfs_btree_find_in_buf(tree, parent->btc_elems,
@@ -1898,7 +1898,8 @@ static uint64_t
 zfs_btree_verify_counts_helper(zfs_btree_t *tree, zfs_btree_hdr_t *hdr)
 {
 	if (!hdr->bth_core) {
-		if (tree->bt_root != hdr && hdr != &tree->bt_bulk->btl_hdr) {
+		if (tree->bt_root != hdr && tree->bt_bulk &&
+		    hdr != &tree->bt_bulk->btl_hdr) {
 			uint64_t capacity = P2ALIGN((BTREE_LEAF_SIZE -
 			    sizeof (zfs_btree_hdr_t)) / tree->bt_elem_size, 2);
 			VERIFY3U(hdr->bth_count, >=, (capacity / 2) - 1);

module/zfs/vdev.c

@@ -4531,8 +4531,8 @@ vdev_stat_update(zio_t *zio, uint64_t psize)
 	vdev_t *vd = zio->io_vd ? zio->io_vd : rvd;
 	vdev_t *pvd;
 	uint64_t txg = zio->io_txg;
-	vdev_stat_t *vs = &vd->vdev_stat;
-	vdev_stat_ex_t *vsx = &vd->vdev_stat_ex;
+	vdev_stat_t *vs = vd ? &vd->vdev_stat : NULL;
+	vdev_stat_ex_t *vsx = vd ? &vd->vdev_stat_ex : NULL;
 	zio_type_t type = zio->io_type;
 	int flags = zio->io_flags;
 

module/zstd/lib/zstd.c

@@ -279,12 +279,9 @@ intptr_t __msan_test_shadow(const volatile void *x, size_t size);
 #endif
 
 /* detects whether we are being compiled under asan */
-#if defined (__has_feature)
-#  if __has_feature(address_sanitizer)
-#    define ADDRESS_SANITIZER 1
-#  endif
-#elif defined(__SANITIZE_ADDRESS__)
+#if defined (ZFS_ASAN_ENABLED)
 #  define ADDRESS_SANITIZER 1
+#  define ZSTD_ASAN_DONT_POISON_WORKSPACE
 #endif
 
 #if defined (ADDRESS_SANITIZER)

module/zstd/zfs_zstd.c

@@ -207,11 +207,7 @@ static struct zstd_pool *zstd_mempool_dctx;
  * and while ASAN does this, KASAN defines that and does not. So to avoid
  * changing the external code, we do this.
  */
-#if defined(__has_feature)
-#if __has_feature(address_sanitizer)
-#define	ADDRESS_SANITIZER 1
-#endif
-#elif defined(__SANITIZE_ADDRESS__)
+#if defined(ZFS_ASAN_ENABLED)
 #define	ADDRESS_SANITIZER 1
 #endif
 #if defined(_KERNEL) && defined(ADDRESS_SANITIZER)