From c0bd9caebeb012d349af85c0a01cddb3d9933dcb Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Mon, 10 Oct 2022 21:35:56 -0300 Subject: [PATCH 1/3] spa_state: fix destructor order --- module/zfs/spa_stats.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/module/zfs/spa_stats.c b/module/zfs/spa_stats.c index 17ed2a620b..0cd3b643e5 100644 --- a/module/zfs/spa_stats.c +++ b/module/zfs/spa_stats.c @@ -855,7 +855,7 @@ spa_guid_init(spa_t *spa) } static void -spa_health_destroy(spa_t *spa) +spa_state_destroy(spa_t *spa) { spa_history_kstat_t *shk = &spa->spa_stats.state; kstat_t *ksp = shk->kstat; @@ -1004,12 +1004,12 @@ void spa_stats_destroy(spa_t *spa) { spa_iostats_destroy(spa); - spa_health_destroy(spa); + spa_guid_destroy(spa); + spa_state_destroy(spa); + spa_mmp_history_destroy(spa); spa_tx_assign_destroy(spa); spa_txg_history_destroy(spa); spa_read_history_destroy(spa); - spa_mmp_history_destroy(spa); - spa_guid_destroy(spa); } ZFS_MODULE_PARAM(zfs, zfs_, read_history, UINT, ZMOD_RW, From bbec9ab76d2f819af8606775f228c57ab6ca95ea Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Mon, 10 Oct 2022 21:37:27 -0300 Subject: [PATCH 2/3] expose free space histograms in /proc example output: ``` [root@nixos:~]# cat /proc/spl/kstat/zfs/vm/fragmentation normal{power="9",pool="vm"} 14 normal{power="10",pool="vm"} 75 normal{power="11",pool="vm"} 16 normal{power="12",pool="vm"} 37 ``` this allows graphing `zdb -MM` without any performance costs # Conflicts: # module/zfs/spa_stats.c --- include/sys/spa.h | 1 + module/zfs/spa_stats.c | 66 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/include/sys/spa.h b/include/sys/spa.h index 08099cd4fa..c1fd2551ea 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -902,6 +902,7 @@ typedef struct spa_stats { spa_history_kstat_t state; /* pool state */ spa_history_kstat_t guid; /* pool guid */ spa_history_kstat_t iostats; + spa_history_kstat_t fragmentation; } spa_stats_t; typedef enum txg_state { diff --git a/module/zfs/spa_stats.c b/module/zfs/spa_stats.c index 0cd3b643e5..846f98214d 100644 --- a/module/zfs/spa_stats.c +++ b/module/zfs/spa_stats.c @@ -24,6 +24,7 @@ #include #include #include +#include /* * Keeps stats on last N reads per spa_t, disabled by default. @@ -302,6 +303,69 @@ spa_txg_history_init(spa_t *spa) offsetof(spa_txg_history_t, sth_node)); } +static void * +spa_frag_addr(kstat_t *ksp, loff_t n) +{ + if (n == 0) + return ksp->ks_private; + return NULL; +} + +static int +spa_frag_data(char *buf, size_t size, void *data) { + spa_t *spa = (spa_t *)data; + size_t offset = 0; + const char *name = spa_name(spa); + + metaslab_class_t *mc = spa_normal_class(spa); + for (int i=0; imc_histogram[i] > 0) { + int res = snprintf(buf + offset, size, "zfs_fragmentation_normal{power=\"%d\",pool=\"%s\"} %llu\n", i, name, (u_longlong_t)mc->mc_histogram[i]); + offset += res; + size -= res; + } + } + + metaslab_class_t *smc = spa_special_class(spa); + for (int i=0; imc_histogram[i] > 0) { + int res = snprintf(buf + offset, size, "zfs_fragmentation_special{power=\"%d\",pool=\"%s\"} %llu\n", i, name, (u_longlong_t)smc->mc_histogram[i]); + offset += res; + size -= res; + } + } + return 0; +} + +static void +spa_fragmentation_init(spa_t *spa) +{ + char *name; + spa_history_kstat_t *shk = &spa->spa_stats.fragmentation; + kstat_t *ksp; + + name = kmem_asprintf("zfs/%s", spa_name(spa)); + ksp = kstat_create(name, 0, "fragmentation", "misc", KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); + + shk->kstat = ksp; + if (ksp) { + ksp->ks_private = spa; + ksp->ks_flags |= KSTAT_FLAG_NO_HEADERS; + kstat_set_raw_ops(ksp, NULL, spa_frag_data, spa_frag_addr); + kstat_install(ksp); + } + kmem_strfree(name); +} + +static void +spa_fragmentation_destroy(spa_t *spa) +{ + spa_history_kstat_t *shk = &spa->spa_stats.fragmentation; + kstat_t *ksp = shk->kstat; + if (ksp) + kstat_delete(ksp); +} + static void spa_txg_history_destroy(spa_t *spa) { @@ -998,11 +1062,13 @@ spa_stats_init(spa_t *spa) spa_state_init(spa); spa_guid_init(spa); spa_iostats_init(spa); + spa_fragmentation_init(spa); } void spa_stats_destroy(spa_t *spa) { + spa_fragmentation_destroy(spa); spa_iostats_destroy(spa); spa_guid_destroy(spa); spa_state_destroy(spa); From 309be3a05cea361eb6f0c4354782312df9ba5347 Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Thu, 20 Oct 2022 05:32:38 -0300 Subject: [PATCH 3/3] de-prometheus the stats, add more metaclasses --- include/sys/spa.h | 2 +- module/zfs/spa_stats.c | 118 +++++++++++++++++++++++++++-------------- 2 files changed, 78 insertions(+), 42 deletions(-) diff --git a/include/sys/spa.h b/include/sys/spa.h index c1fd2551ea..c0b7883f8f 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -902,7 +902,7 @@ typedef struct spa_stats { spa_history_kstat_t state; /* pool state */ spa_history_kstat_t guid; /* pool guid */ spa_history_kstat_t iostats; - spa_history_kstat_t fragmentation; + spa_history_kstat_t fragmentation[5]; } spa_stats_t; typedef enum txg_state { diff --git a/module/zfs/spa_stats.c b/module/zfs/spa_stats.c index 846f98214d..46a797c3cf 100644 --- a/module/zfs/spa_stats.c +++ b/module/zfs/spa_stats.c @@ -303,67 +303,103 @@ spa_txg_history_init(spa_t *spa) offsetof(spa_txg_history_t, sth_node)); } +typedef struct { + spa_t *spa; + int metaclass; +} freespace_histogram_stat_t; + static void * spa_frag_addr(kstat_t *ksp, loff_t n) { - if (n == 0) - return ksp->ks_private; - return NULL; + if (n == 0) { + freespace_histogram_stat_t *stat = ksp->ks_private; + switch (stat->metaclass) { + case 0: + return (spa_normal_class(stat->spa)); + case 1: + return (spa_log_class(stat->spa)); + case 2: + return (spa_embedded_log_class(stat->spa)); + case 3: + return (spa_special_class(stat->spa)); + case 4: + return (spa_dedup_class(stat->spa)); + } + } + return (NULL); } static int -spa_frag_data(char *buf, size_t size, void *data) { - spa_t *spa = (spa_t *)data; - size_t offset = 0; - const char *name = spa_name(spa); +spa_frag_data(char *buf, size_t size, void *data) +{ + metaslab_class_t *mc = data; + size_t offset = 0; + int res; + // without this, it becomes a data-leak, and un-initialized strings + // can be returned if all buckets are 0 + buf[0] = 0; - metaslab_class_t *mc = spa_normal_class(spa); - for (int i=0; imc_histogram[i] > 0) { - int res = snprintf(buf + offset, size, "zfs_fragmentation_normal{power=\"%d\",pool=\"%s\"} %llu\n", i, name, (u_longlong_t)mc->mc_histogram[i]); - offset += res; - size -= res; - } - } + for (int i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) { + if (mc->mc_histogram[i] > 0) { + res = snprintf(buf + offset, size, "%d %llu\n", i, + (u_longlong_t)mc->mc_histogram[i]); + offset += res; + size -= res; + } + } - metaslab_class_t *smc = spa_special_class(spa); - for (int i=0; imc_histogram[i] > 0) { - int res = snprintf(buf + offset, size, "zfs_fragmentation_special{power=\"%d\",pool=\"%s\"} %llu\n", i, name, (u_longlong_t)smc->mc_histogram[i]); - offset += res; - size -= res; - } - } - return 0; + return (0); } static void spa_fragmentation_init(spa_t *spa) { - char *name; - spa_history_kstat_t *shk = &spa->spa_stats.fragmentation; - kstat_t *ksp; + char *dirname; + const char *statnames[] = { + "fragmentation_normal", + "fragmentation_log", + "fragmentation_embedded_log", + "fragmentation_special", + "fragmentation_dedup" + }; + for (int i = 0; i < 5; i++) { + spa_history_kstat_t *shk = &spa->spa_stats.fragmentation[i]; + kstat_t *ksp; - name = kmem_asprintf("zfs/%s", spa_name(spa)); - ksp = kstat_create(name, 0, "fragmentation", "misc", KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); + dirname = kmem_asprintf("zfs/%s", spa_name(spa)); + ksp = kstat_create(dirname, 0, statnames[i], "misc", + KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); + kmem_strfree(dirname); - shk->kstat = ksp; - if (ksp) { - ksp->ks_private = spa; - ksp->ks_flags |= KSTAT_FLAG_NO_HEADERS; - kstat_set_raw_ops(ksp, NULL, spa_frag_data, spa_frag_addr); - kstat_install(ksp); - } - kmem_strfree(name); + shk->kstat = ksp; + if (ksp) { + freespace_histogram_stat_t *stat = kmem_zalloc( + sizeof (freespace_histogram_stat_t), KM_SLEEP); + stat->spa = spa; + stat->metaclass = i; + ksp->ks_private = stat; + ksp->ks_flags |= KSTAT_FLAG_NO_HEADERS; + kstat_set_raw_ops(ksp, NULL, spa_frag_data, + spa_frag_addr); + kstat_install(ksp); + } + } } static void spa_fragmentation_destroy(spa_t *spa) { - spa_history_kstat_t *shk = &spa->spa_stats.fragmentation; - kstat_t *ksp = shk->kstat; - if (ksp) - kstat_delete(ksp); + for (int i = 0; i < 5; i++) { + spa_history_kstat_t *shk = &spa->spa_stats.fragmentation[i]; + kstat_t *ksp = shk->kstat; + if (ksp) { + if (ksp->ks_private) { + kmem_free(ksp->ks_private, + sizeof (freespace_histogram_stat_t)); + } + kstat_delete(ksp); + } + } } static void