From 4df415aa864690163808147b82d9e5ce2be12f68 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Tue, 29 Nov 2022 12:33:45 -0500 Subject: [PATCH] Switch dnode stats to wmsums I've noticed that some of those counters are used in hot paths like dnode_hold_impl(), and results of this change is visible in profiler. Reviewed-by: Brian Behlendorf Reviewed-by: Richard Yao Reviewed-by: Ryan Moeller Signed-off-by: Alexander Motin Closes #14198 --- include/sys/dnode.h | 35 +++++++++++- module/zfs/dnode.c | 127 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/include/sys/dnode.h b/include/sys/dnode.h index 28d8847a1..9553988e7 100644 --- a/include/sys/dnode.h +++ b/include/sys/dnode.h @@ -36,6 +36,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -587,10 +588,42 @@ typedef struct dnode_stats { kstat_named_t dnode_move_active; } dnode_stats_t; +typedef struct dnode_sums { + wmsum_t dnode_hold_dbuf_hold; + wmsum_t dnode_hold_dbuf_read; + wmsum_t dnode_hold_alloc_hits; + wmsum_t dnode_hold_alloc_misses; + wmsum_t dnode_hold_alloc_interior; + wmsum_t dnode_hold_alloc_lock_retry; + wmsum_t dnode_hold_alloc_lock_misses; + wmsum_t dnode_hold_alloc_type_none; + wmsum_t dnode_hold_free_hits; + wmsum_t dnode_hold_free_misses; + wmsum_t dnode_hold_free_lock_misses; + wmsum_t dnode_hold_free_lock_retry; + wmsum_t dnode_hold_free_refcount; + wmsum_t dnode_hold_free_overflow; + wmsum_t dnode_free_interior_lock_retry; + wmsum_t dnode_allocate; + wmsum_t dnode_reallocate; + wmsum_t dnode_buf_evict; + wmsum_t dnode_alloc_next_chunk; + wmsum_t dnode_alloc_race; + wmsum_t dnode_alloc_next_block; + wmsum_t dnode_move_invalid; + wmsum_t dnode_move_recheck1; + wmsum_t dnode_move_recheck2; + wmsum_t dnode_move_special; + wmsum_t dnode_move_handle; + wmsum_t dnode_move_rwlock; + wmsum_t dnode_move_active; +} dnode_sums_t; + extern dnode_stats_t dnode_stats; +extern dnode_sums_t dnode_sums; #define DNODE_STAT_INCR(stat, val) \ - atomic_add_64(&dnode_stats.stat.value.ui64, (val)); + wmsum_add(&dnode_sums.stat, (val)) #define DNODE_STAT_BUMP(stat) \ DNODE_STAT_INCR(stat, 1); diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index ef27dfd40..1a40e394e 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -71,6 +71,8 @@ dnode_stats_t dnode_stats = { { "dnode_move_active", KSTAT_DATA_UINT64 }, }; +dnode_sums_t dnode_sums; + static kstat_t *dnode_ksp; static kmem_cache_t *dnode_cache; @@ -225,6 +227,72 @@ dnode_dest(void *arg, void *unused) avl_destroy(&dn->dn_dbufs); } +static int +dnode_kstats_update(kstat_t *ksp, int rw) +{ + dnode_stats_t *ds = ksp->ks_data; + + if (rw == KSTAT_WRITE) + return (EACCES); + ds->dnode_hold_dbuf_hold.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_dbuf_hold); + ds->dnode_hold_dbuf_read.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_dbuf_read); + ds->dnode_hold_alloc_hits.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_alloc_hits); + ds->dnode_hold_alloc_misses.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_alloc_misses); + ds->dnode_hold_alloc_interior.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_alloc_interior); + ds->dnode_hold_alloc_lock_retry.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_alloc_lock_retry); + ds->dnode_hold_alloc_lock_misses.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_alloc_lock_misses); + ds->dnode_hold_alloc_type_none.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_alloc_type_none); + ds->dnode_hold_free_hits.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_free_hits); + ds->dnode_hold_free_misses.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_free_misses); + ds->dnode_hold_free_lock_misses.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_free_lock_misses); + ds->dnode_hold_free_lock_retry.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_free_lock_retry); + ds->dnode_hold_free_refcount.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_free_refcount); + ds->dnode_hold_free_overflow.value.ui64 = + wmsum_value(&dnode_sums.dnode_hold_free_overflow); + ds->dnode_free_interior_lock_retry.value.ui64 = + wmsum_value(&dnode_sums.dnode_free_interior_lock_retry); + ds->dnode_allocate.value.ui64 = + wmsum_value(&dnode_sums.dnode_allocate); + ds->dnode_reallocate.value.ui64 = + wmsum_value(&dnode_sums.dnode_reallocate); + ds->dnode_buf_evict.value.ui64 = + wmsum_value(&dnode_sums.dnode_buf_evict); + ds->dnode_alloc_next_chunk.value.ui64 = + wmsum_value(&dnode_sums.dnode_alloc_next_chunk); + ds->dnode_alloc_race.value.ui64 = + wmsum_value(&dnode_sums.dnode_alloc_race); + ds->dnode_alloc_next_block.value.ui64 = + wmsum_value(&dnode_sums.dnode_alloc_next_block); + ds->dnode_move_invalid.value.ui64 = + wmsum_value(&dnode_sums.dnode_move_invalid); + ds->dnode_move_recheck1.value.ui64 = + wmsum_value(&dnode_sums.dnode_move_recheck1); + ds->dnode_move_recheck2.value.ui64 = + wmsum_value(&dnode_sums.dnode_move_recheck2); + ds->dnode_move_special.value.ui64 = + wmsum_value(&dnode_sums.dnode_move_special); + ds->dnode_move_handle.value.ui64 = + wmsum_value(&dnode_sums.dnode_move_handle); + ds->dnode_move_rwlock.value.ui64 = + wmsum_value(&dnode_sums.dnode_move_rwlock); + ds->dnode_move_active.value.ui64 = + wmsum_value(&dnode_sums.dnode_move_active); + return (0); +} + void dnode_init(void) { @@ -233,11 +301,41 @@ dnode_init(void) 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0); kmem_cache_set_move(dnode_cache, dnode_move); + wmsum_init(&dnode_sums.dnode_hold_dbuf_hold, 0); + wmsum_init(&dnode_sums.dnode_hold_dbuf_read, 0); + wmsum_init(&dnode_sums.dnode_hold_alloc_hits, 0); + wmsum_init(&dnode_sums.dnode_hold_alloc_misses, 0); + wmsum_init(&dnode_sums.dnode_hold_alloc_interior, 0); + wmsum_init(&dnode_sums.dnode_hold_alloc_lock_retry, 0); + wmsum_init(&dnode_sums.dnode_hold_alloc_lock_misses, 0); + wmsum_init(&dnode_sums.dnode_hold_alloc_type_none, 0); + wmsum_init(&dnode_sums.dnode_hold_free_hits, 0); + wmsum_init(&dnode_sums.dnode_hold_free_misses, 0); + wmsum_init(&dnode_sums.dnode_hold_free_lock_misses, 0); + wmsum_init(&dnode_sums.dnode_hold_free_lock_retry, 0); + wmsum_init(&dnode_sums.dnode_hold_free_refcount, 0); + wmsum_init(&dnode_sums.dnode_hold_free_overflow, 0); + wmsum_init(&dnode_sums.dnode_free_interior_lock_retry, 0); + wmsum_init(&dnode_sums.dnode_allocate, 0); + wmsum_init(&dnode_sums.dnode_reallocate, 0); + wmsum_init(&dnode_sums.dnode_buf_evict, 0); + wmsum_init(&dnode_sums.dnode_alloc_next_chunk, 0); + wmsum_init(&dnode_sums.dnode_alloc_race, 0); + wmsum_init(&dnode_sums.dnode_alloc_next_block, 0); + wmsum_init(&dnode_sums.dnode_move_invalid, 0); + wmsum_init(&dnode_sums.dnode_move_recheck1, 0); + wmsum_init(&dnode_sums.dnode_move_recheck2, 0); + wmsum_init(&dnode_sums.dnode_move_special, 0); + wmsum_init(&dnode_sums.dnode_move_handle, 0); + wmsum_init(&dnode_sums.dnode_move_rwlock, 0); + wmsum_init(&dnode_sums.dnode_move_active, 0); + dnode_ksp = kstat_create("zfs", 0, "dnodestats", "misc", KSTAT_TYPE_NAMED, sizeof (dnode_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); if (dnode_ksp != NULL) { dnode_ksp->ks_data = &dnode_stats; + dnode_ksp->ks_update = dnode_kstats_update; kstat_install(dnode_ksp); } } @@ -250,6 +348,35 @@ dnode_fini(void) dnode_ksp = NULL; } + wmsum_fini(&dnode_sums.dnode_hold_dbuf_hold); + wmsum_fini(&dnode_sums.dnode_hold_dbuf_read); + wmsum_fini(&dnode_sums.dnode_hold_alloc_hits); + wmsum_fini(&dnode_sums.dnode_hold_alloc_misses); + wmsum_fini(&dnode_sums.dnode_hold_alloc_interior); + wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_retry); + wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_misses); + wmsum_fini(&dnode_sums.dnode_hold_alloc_type_none); + wmsum_fini(&dnode_sums.dnode_hold_free_hits); + wmsum_fini(&dnode_sums.dnode_hold_free_misses); + wmsum_fini(&dnode_sums.dnode_hold_free_lock_misses); + wmsum_fini(&dnode_sums.dnode_hold_free_lock_retry); + wmsum_fini(&dnode_sums.dnode_hold_free_refcount); + wmsum_fini(&dnode_sums.dnode_hold_free_overflow); + wmsum_fini(&dnode_sums.dnode_free_interior_lock_retry); + wmsum_fini(&dnode_sums.dnode_allocate); + wmsum_fini(&dnode_sums.dnode_reallocate); + wmsum_fini(&dnode_sums.dnode_buf_evict); + wmsum_fini(&dnode_sums.dnode_alloc_next_chunk); + wmsum_fini(&dnode_sums.dnode_alloc_race); + wmsum_fini(&dnode_sums.dnode_alloc_next_block); + wmsum_fini(&dnode_sums.dnode_move_invalid); + wmsum_fini(&dnode_sums.dnode_move_recheck1); + wmsum_fini(&dnode_sums.dnode_move_recheck2); + wmsum_fini(&dnode_sums.dnode_move_special); + wmsum_fini(&dnode_sums.dnode_move_handle); + wmsum_fini(&dnode_sums.dnode_move_rwlock); + wmsum_fini(&dnode_sums.dnode_move_active); + kmem_cache_destroy(dnode_cache); dnode_cache = NULL; }