Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

remove kmem_cache module parameter KMC_EXPIRE_AGE 

By default, `spl_kmem_cache_expire` is `KMC_EXPIRE_MEM`, meaning that
objects will be removed from kmem cache magazines by
`spl_kmem_cache_reap_now()`.

There is also a module parameter to change this to `KMC_EXPIRE_AGE`,
which establishes a maximum lifetime for objects to stay in the
magazine.  This setting has rarely, if ever, been used, and is not
regularly tested.

This commit removes the code for `KMC_EXPIRE_AGE`, and associated module
parameters.

Additionally, the unused module parameter
`spl_kmem_cache_obj_per_slab_min` is removed.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
Closes #10608
This commit is contained in:
Matthew Ahrens 2020-07-24 09:39:26 -07:00 committed by GitHub
parent 02fced3067
commit 4fbdb10c7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 9 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/os/linux/spl/sys/kmem_cache.h
View File

@ -85,12 +85,8 @@ typedef enum kmem_cbrc {
#define KMC_REAP_CHUNK INT_MAX #define KMC_REAP_CHUNK INT_MAX
#define KMC_DEFAULT_SEEKS 1 #define KMC_DEFAULT_SEEKS 1
#define KMC_EXPIRE_AGE 0x1 /* Due to age */
#define KMC_EXPIRE_MEM 0x2 /* Due to low memory */
#define KMC_RECLAIM_ONCE 0x1 /* Force a single shrinker pass */ #define KMC_RECLAIM_ONCE 0x1 /* Force a single shrinker pass */
extern unsigned int spl_kmem_cache_expire;
extern struct list_head spl_kmem_cache_list; extern struct list_head spl_kmem_cache_list;
extern struct rw_semaphore spl_kmem_cache_sem; extern struct rw_semaphore spl_kmem_cache_sem;
@ -99,9 +95,7 @@ extern struct rw_semaphore spl_kmem_cache_sem;
#define SKS_MAGIC 0x22222222 #define SKS_MAGIC 0x22222222
#define SKC_MAGIC 0x2c2c2c2c #define SKC_MAGIC 0x2c2c2c2c
#define SPL_KMEM_CACHE_DELAY 15 /* Minimum slab release age */
#define SPL_KMEM_CACHE_OBJ_PER_SLAB 8 /* Target objects per slab */ #define SPL_KMEM_CACHE_OBJ_PER_SLAB 8 /* Target objects per slab */
#define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 1 /* Minimum objects per slab */
#define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */ #define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */
#ifdef _LP64 #ifdef _LP64
#define SPL_KMEM_CACHE_MAX_SIZE 32 /* Max slab size in MB */ #define SPL_KMEM_CACHE_MAX_SIZE 32 /* Max slab size in MB */
@ -131,7 +125,6 @@ typedef struct spl_kmem_magazine {
uint32_t skm_size; /* Magazine size */ uint32_t skm_size; /* Magazine size */
uint32_t skm_refill; /* Batch refill size */ uint32_t skm_refill; /* Batch refill size */
struct spl_kmem_cache *skm_cache; /* Owned by cache */ struct spl_kmem_cache *skm_cache; /* Owned by cache */
unsigned long skm_age; /* Last cache access */
unsigned int skm_cpu; /* Owned by cpu */ unsigned int skm_cpu; /* Owned by cpu */
void *skm_objs[0]; /* Object pointers */ void *skm_objs[0]; /* Object pointers */
} spl_kmem_magazine_t; } spl_kmem_magazine_t;
@ -181,7 +174,6 @@ typedef struct spl_kmem_cache {
uint32_t skc_obj_align; /* Object alignment */ uint32_t skc_obj_align; /* Object alignment */
uint32_t skc_slab_objs; /* Objects per slab */ uint32_t skc_slab_objs; /* Objects per slab */
uint32_t skc_slab_size; /* Slab size */ uint32_t skc_slab_size; /* Slab size */
uint32_t skc_delay; /* Slab reclaim interval */
atomic_t skc_ref; /* Ref count callers */ atomic_t skc_ref; /* Ref count callers */
taskqid_t skc_taskqid; /* Slab reclaim task */ taskqid_t skc_taskqid; /* Slab reclaim task */
struct list_head skc_list; /* List of caches linkage */ struct list_head skc_list; /* List of caches linkage */

121
module/os/linux/spl/spl-kmem-cache.c
View File

@ -57,20 +57,7 @@
#define smp_mb__after_atomic(x) smp_mb__after_clear_bit(x) #define smp_mb__after_atomic(x) smp_mb__after_clear_bit(x)
#endif #endif
/*
* Cache expiration was implemented because it was part of the default Solaris
* kmem_cache behavior. The idea is that per-cpu objects which haven't been
* accessed in several seconds should be returned to the cache. On the other
* hand Linux slabs never move objects back to the slabs unless there is
* memory pressure on the system. By default the Linux method is enabled
* because it has been shown to improve responsiveness on low memory systems.
* This policy may be changed by setting KMC_EXPIRE_AGE or KMC_EXPIRE_MEM.
*/
/* BEGIN CSTYLED */ /* BEGIN CSTYLED */
unsigned int spl_kmem_cache_expire = KMC_EXPIRE_MEM;
EXPORT_SYMBOL(spl_kmem_cache_expire);
module_param(spl_kmem_cache_expire, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_expire, "By age (0x1) or low memory (0x2)");
/* /*
* Cache magazines are an optimization designed to minimize the cost of * Cache magazines are an optimization designed to minimize the cost of
@ -106,11 +93,6 @@ unsigned int spl_kmem_cache_obj_per_slab = SPL_KMEM_CACHE_OBJ_PER_SLAB;
module_param(spl_kmem_cache_obj_per_slab, uint, 0644); module_param(spl_kmem_cache_obj_per_slab, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab, "Number of objects per slab"); MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab, "Number of objects per slab");
unsigned int spl_kmem_cache_obj_per_slab_min = SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN;
module_param(spl_kmem_cache_obj_per_slab_min, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab_min,
"Minimal number of objects per slab");
unsigned int spl_kmem_cache_max_size = SPL_KMEM_CACHE_MAX_SIZE; unsigned int spl_kmem_cache_max_size = SPL_KMEM_CACHE_MAX_SIZE;
module_param(spl_kmem_cache_max_size, uint, 0644); module_param(spl_kmem_cache_max_size, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_max_size, "Maximum size of slab in MB"); MODULE_PARM_DESC(spl_kmem_cache_max_size, "Maximum size of slab in MB");
@ -590,104 +572,24 @@ spl_emergency_free(spl_kmem_cache_t *skc, void *obj)
* argument contains the max number of entries to remove from the magazine. * argument contains the max number of entries to remove from the magazine.
*/ */
static void static void
__spl_cache_flush(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flush) spl_cache_flush(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flush)
{ {
int i, count = MIN(flush, skm->skm_avail); spin_lock(&skc->skc_lock);
ASSERT(skc->skc_magic == SKC_MAGIC); ASSERT(skc->skc_magic == SKC_MAGIC);
ASSERT(skm->skm_magic == SKM_MAGIC); ASSERT(skm->skm_magic == SKM_MAGIC);
for (i = 0; i < count; i++) int count = MIN(flush, skm->skm_avail);
for (int i = 0; i < count; i++)
spl_cache_shrink(skc, skm->skm_objs[i]); spl_cache_shrink(skc, skm->skm_objs[i]);
skm->skm_avail -= count; skm->skm_avail -= count;
memmove(skm->skm_objs, &(skm->skm_objs[count]), memmove(skm->skm_objs, &(skm->skm_objs[count]),
sizeof (void *) * skm->skm_avail); sizeof (void *) * skm->skm_avail);
}
static void
spl_cache_flush(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flush)
{
spin_lock(&skc->skc_lock);
__spl_cache_flush(skc, skm, flush);
spin_unlock(&skc->skc_lock); spin_unlock(&skc->skc_lock);
} }
static void
spl_magazine_age(void *data)
{
spl_kmem_cache_t *skc = (spl_kmem_cache_t *)data;
spl_kmem_magazine_t *skm = skc->skc_mag[smp_processor_id()];
ASSERT(skm->skm_magic == SKM_MAGIC);
ASSERT(skm->skm_cpu == smp_processor_id());
ASSERT(irqs_disabled());
/* There are no available objects or they are too young to age out */
if ((skm->skm_avail == 0) ||
time_before(jiffies, skm->skm_age + skc->skc_delay * HZ))
return;
/*
* Because we're executing in interrupt context we may have
* interrupted the holder of this lock. To avoid a potential
* deadlock return if the lock is contended.
*/
if (!spin_trylock(&skc->skc_lock))
return;
__spl_cache_flush(skc, skm, skm->skm_refill);
spin_unlock(&skc->skc_lock);
}
/*
* Called regularly to keep a downward pressure on the cache.
*
* Objects older than skc->skc_delay seconds in the per-cpu magazines will
* be returned to the caches. This is done to prevent idle magazines from
* holding memory which could be better used elsewhere. The delay is
* present to prevent thrashing the magazine.
*
* The newly released objects may result in empty partial slabs. Those
* slabs should be released to the system. Otherwise moving the objects
* out of the magazines is just wasted work.
*/
static void
spl_cache_age(void *data)
{
spl_kmem_cache_t *skc = (spl_kmem_cache_t *)data;
taskqid_t id = 0;
ASSERT(skc->skc_magic == SKC_MAGIC);
/* Dynamically disabled at run time */
if (!(spl_kmem_cache_expire & KMC_EXPIRE_AGE))
return;
atomic_inc(&skc->skc_ref);
if (!(skc->skc_flags & KMC_NOMAGAZINE))
on_each_cpu(spl_magazine_age, skc, 1);
spl_slab_reclaim(skc);
while (!test_bit(KMC_BIT_DESTROY, &skc->skc_flags) && !id) {
id = taskq_dispatch_delay(
spl_kmem_cache_taskq, spl_cache_age, skc, TQ_SLEEP,
ddi_get_lbolt() + skc->skc_delay / 3 * HZ);
/* Destroy issued after dispatch immediately cancel it */
if (test_bit(KMC_BIT_DESTROY, &skc->skc_flags) && id)
taskq_cancel_id(spl_kmem_cache_taskq, id);
}
spin_lock(&skc->skc_lock);
skc->skc_taskqid = id;
spin_unlock(&skc->skc_lock);
atomic_dec(&skc->skc_ref);
}
/* /*
* Size a slab based on the size of each aligned object plus spl_kmem_obj_t. * Size a slab based on the size of each aligned object plus spl_kmem_obj_t.
* When on-slab we want to target spl_kmem_cache_obj_per_slab. However, * When on-slab we want to target spl_kmem_cache_obj_per_slab. However,
@ -789,7 +691,6 @@ spl_magazine_alloc(spl_kmem_cache_t *skc, int cpu)
skm->skm_size = skc->skc_mag_size; skm->skm_size = skc->skc_mag_size;
skm->skm_refill = skc->skc_mag_refill; skm->skm_refill = skc->skc_mag_refill;
skm->skm_cache = skc; skm->skm_cache = skc;
skm->skm_age = jiffies;
skm->skm_cpu = cpu; skm->skm_cpu = cpu;
} }
@ -921,7 +822,6 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
skc->skc_flags = flags; skc->skc_flags = flags;
skc->skc_obj_size = size; skc->skc_obj_size = size;
skc->skc_obj_align = SPL_KMEM_CACHE_ALIGN; skc->skc_obj_align = SPL_KMEM_CACHE_ALIGN;
skc->skc_delay = SPL_KMEM_CACHE_DELAY;
atomic_set(&skc->skc_ref, 0); atomic_set(&skc->skc_ref, 0);
INIT_LIST_HEAD(&skc->skc_list); INIT_LIST_HEAD(&skc->skc_list);
@ -1036,12 +936,6 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
skc->skc_flags |= KMC_NOMAGAZINE; skc->skc_flags |= KMC_NOMAGAZINE;
} }
if (spl_kmem_cache_expire & KMC_EXPIRE_AGE) {
skc->skc_taskqid = taskq_dispatch_delay(spl_kmem_cache_taskq,
spl_cache_age, skc, TQ_SLEEP,
ddi_get_lbolt() + skc->skc_delay / 3 * HZ);
}
down_write(&spl_kmem_cache_sem); down_write(&spl_kmem_cache_sem);
list_add_tail(&skc->skc_list, &spl_kmem_cache_list); list_add_tail(&skc->skc_list, &spl_kmem_cache_list);
up_write(&spl_kmem_cache_sem); up_write(&spl_kmem_cache_sem);
@ -1499,7 +1393,6 @@ restart:
if (likely(skm->skm_avail)) { if (likely(skm->skm_avail)) {
/* Object available in CPU cache, use it */ /* Object available in CPU cache, use it */
obj = skm->skm_objs[--skm->skm_avail]; obj = skm->skm_objs[--skm->skm_avail];
skm->skm_age = jiffies;
} else { } else {
obj = spl_cache_refill(skc, skm, flags); obj = spl_cache_refill(skc, skm, flags);
if ((obj == NULL) && !(flags & KM_NOSLEEP)) if ((obj == NULL) && !(flags & KM_NOSLEEP))
@ -1629,15 +1522,11 @@ spl_kmem_cache_reap_now(spl_kmem_cache_t *skc)
goto out; goto out;
/* Reclaim from the magazine and free all now empty slabs. */ /* Reclaim from the magazine and free all now empty slabs. */
if (spl_kmem_cache_expire & KMC_EXPIRE_MEM) {
spl_kmem_magazine_t *skm;
unsigned long irq_flags; unsigned long irq_flags;
local_irq_save(irq_flags); local_irq_save(irq_flags);
skm = skc->skc_mag[smp_processor_id()]; spl_kmem_magazine_t *skm = skc->skc_mag[smp_processor_id()];
spl_cache_flush(skc, skm, skm->skm_avail); spl_cache_flush(skc, skm, skm->skm_avail);
local_irq_restore(irq_flags); local_irq_restore(irq_flags);
}
spl_slab_reclaim(skc); spl_slab_reclaim(skc);
clear_bit_unlock(KMC_BIT_REAPING, &skc->skc_flags); clear_bit_unlock(KMC_BIT_REAPING, &skc->skc_flags);