Introduce zfs_refcount_(add|remove)_few().

There are two places where we need to add/remove several references
with semantics of zfs_refcount_(add|remove). But when debug/tracing
is disabled, it is a crime to run multiple atomic_inc() in a loop,
especially under congested pool-wide allocator lock.

Introduced new functions implement the same semantics as the loop,
but without overhead in production builds.

Reviewed-by: Rich Ercolani <rincebrain@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by:	Alexander Motin <mav@FreeBSD.org>
Sponsored by:	iXsystems, Inc.
Closes #14934

include/sys/zfs_refcount.h

@@ -73,13 +73,15 @@ int64_t zfs_refcount_count(zfs_refcount_t *);
 int64_t zfs_refcount_add(zfs_refcount_t *, const void *);
 int64_t zfs_refcount_remove(zfs_refcount_t *, const void *);
 /*
- * Note that (add|remove)_many add/remove one reference with "number" N,
+ * Note that (add|remove)_many adds/removes one reference with "number" N,
- * _not_ make N references with "number" 1, which is what vanilla
+ * _not_ N references with "number" 1, which is what (add|remove)_few does,
- * zfs_refcount_(add|remove) would do if called N times.
+ * or what vanilla zfs_refcount_(add|remove) called N times would do.
  *
  * Attempting to remove a reference with number N when none exists is a
  * panic on debug kernels with reference_tracking enabled.
  */
+void zfs_refcount_add_few(zfs_refcount_t *, uint64_t, const void *);
+void zfs_refcount_remove_few(zfs_refcount_t *, uint64_t, const void *);
 int64_t zfs_refcount_add_many(zfs_refcount_t *, uint64_t, const void *);
 int64_t zfs_refcount_remove_many(zfs_refcount_t *, uint64_t, const void *);
 void zfs_refcount_transfer(zfs_refcount_t *, zfs_refcount_t *);
@@ -108,6 +110,10 @@ typedef struct refcount {
 #define	zfs_refcount_count(rc) atomic_load_64(&(rc)->rc_count)
 #define	zfs_refcount_add(rc, holder) atomic_inc_64_nv(&(rc)->rc_count)
 #define	zfs_refcount_remove(rc, holder) atomic_dec_64_nv(&(rc)->rc_count)
+#define	zfs_refcount_add_few(rc, number, holder) \
+	atomic_add_64(&(rc)->rc_count, number)
+#define	zfs_refcount_remove_few(rc, number, holder) \
+	atomic_add_64(&(rc)->rc_count, -number)
 #define	zfs_refcount_add_many(rc, number, holder) \
 	atomic_add_64_nv(&(rc)->rc_count, number)
 #define	zfs_refcount_remove_many(rc, number, holder) \

module/zfs/dmu_zfetch.c

@@ -520,8 +520,7 @@ dmu_zfetch_run(zstream_t *zs, boolean_t missed, boolean_t have_lock)
 	issued = pf_end - pf_start + ipf_end - ipf_start;
 	if (issued > 1) {
 		/* More references on top of taken in dmu_zfetch_prepare(). */
-		for (int i = 0; i < issued - 1; i++)
+		zfs_refcount_add_few(&zs->zs_refs, issued - 1, NULL);
-			zfs_refcount_add(&zs->zs_refs, NULL);
 	} else if (issued == 0) {
 		/* Some other thread has done our work, so drop the ref. */
 		if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)

module/zfs/metaslab.c

@@ -5650,8 +5650,7 @@ metaslab_class_throttle_reserve(metaslab_class_t *mc, int slots, int allocator,
 		 * We reserve the slots individually so that we can unreserve
 		 * them individually when an I/O completes.
 		 */
-		for (int d = 0; d < slots; d++)
+		zfs_refcount_add_few(&mca->mca_alloc_slots, slots, zio);
-			zfs_refcount_add(&mca->mca_alloc_slots, zio);
 		zio->io_flags |= ZIO_FLAG_IO_ALLOCATING;
 		return (B_TRUE);
 	}
@@ -5665,8 +5664,7 @@ metaslab_class_throttle_unreserve(metaslab_class_t *mc, int slots,
 	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];
 
 	ASSERT(mc->mc_alloc_throttle_enabled);
-	for (int d = 0; d < slots; d++)
+	zfs_refcount_remove_few(&mca->mca_alloc_slots, slots, zio);
-		zfs_refcount_remove(&mca->mca_alloc_slots, zio);
 }
 
 static int

module/zfs/refcount.c

@@ -151,6 +151,15 @@ zfs_refcount_add(zfs_refcount_t *rc, const void *holder)
 	return (zfs_refcount_add_many(rc, 1, holder));
 }
 
+void
+zfs_refcount_add_few(zfs_refcount_t *rc, uint64_t number, const void *holder)
+{
+	if (!rc->rc_tracked)
+		(void) zfs_refcount_add_many(rc, number, holder);
+	else for (; number > 0; number--)
+		(void) zfs_refcount_add(rc, holder);
+}
+
 int64_t
 zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,
     const void *holder)
@@ -204,6 +213,15 @@ zfs_refcount_remove(zfs_refcount_t *rc, const void *holder)
 	return (zfs_refcount_remove_many(rc, 1, holder));
 }
 
+void
+zfs_refcount_remove_few(zfs_refcount_t *rc, uint64_t number, const void *holder)
+{
+	if (!rc->rc_tracked)
+		(void) zfs_refcount_remove_many(rc, number, holder);
+	else for (; number > 0; number--)
+		(void) zfs_refcount_remove(rc, holder);
+}
+
 void
 zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src)
 {