Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity
zfs/module/splat/splat-kmem.c

1321 lines
34 KiB
C
Raw Blame History

/*****************************************************************************\
* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
* Copyright (C) 2007 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
* UCRL-CODE-235197
*
* This file is part of the SPL, Solaris Porting Layer.
* For details, see <http://zfsonlinux.org/>.
*
* The SPL is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation; either version 2 of the License, or (at your
* option) any later version.
*
* The SPL is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*
* You should have received a copy of the GNU General Public License along
* with the SPL. If not, see <http://www.gnu.org/licenses/>.
*****************************************************************************
* Solaris Porting LAyer Tests (SPLAT) Kmem Tests.
\*****************************************************************************/
#include <sys/kmem.h>
#include <sys/thread.h>
#include "splat-internal.h"
#define SPLAT_KMEM_NAME "kmem"
#define SPLAT_KMEM_DESC "Kernel Malloc/Slab Tests"
#define SPLAT_KMEM_TEST1_ID 0x0101
#define SPLAT_KMEM_TEST1_NAME "kmem_alloc"
#define SPLAT_KMEM_TEST1_DESC "Memory allocation test (kmem_alloc)"
#define SPLAT_KMEM_TEST2_ID 0x0102
#define SPLAT_KMEM_TEST2_NAME "kmem_zalloc"
#define SPLAT_KMEM_TEST2_DESC "Memory allocation test (kmem_zalloc)"
#define SPLAT_KMEM_TEST3_ID 0x0103
#define SPLAT_KMEM_TEST3_NAME "vmem_alloc"
#define SPLAT_KMEM_TEST3_DESC "Memory allocation test (vmem_alloc)"
#define SPLAT_KMEM_TEST4_ID 0x0104
#define SPLAT_KMEM_TEST4_NAME "vmem_zalloc"
#define SPLAT_KMEM_TEST4_DESC "Memory allocation test (vmem_zalloc)"
#define SPLAT_KMEM_TEST5_ID 0x0105
#define SPLAT_KMEM_TEST5_NAME "slab_small"
#define SPLAT_KMEM_TEST5_DESC "Slab ctor/dtor test (small)"
#define SPLAT_KMEM_TEST6_ID 0x0106
#define SPLAT_KMEM_TEST6_NAME "slab_large"
#define SPLAT_KMEM_TEST6_DESC "Slab ctor/dtor test (large)"
#define SPLAT_KMEM_TEST7_ID 0x0107
#define SPLAT_KMEM_TEST7_NAME "slab_align"
#define SPLAT_KMEM_TEST7_DESC "Slab alignment test"
#define SPLAT_KMEM_TEST8_ID 0x0108
#define SPLAT_KMEM_TEST8_NAME "slab_reap"
#define SPLAT_KMEM_TEST8_DESC "Slab reaping test"
#define SPLAT_KMEM_TEST9_ID 0x0109
#define SPLAT_KMEM_TEST9_NAME "slab_age"
#define SPLAT_KMEM_TEST9_DESC "Slab aging test"
#define SPLAT_KMEM_TEST10_ID 0x010a
#define SPLAT_KMEM_TEST10_NAME "slab_lock"
#define SPLAT_KMEM_TEST10_DESC "Slab locking test"
#if 0
#define SPLAT_KMEM_TEST11_ID 0x010b
#define SPLAT_KMEM_TEST11_NAME "slab_overcommit"
#define SPLAT_KMEM_TEST11_DESC "Slab memory overcommit test"
#endif
#define SPLAT_KMEM_TEST13_ID 0x010d
#define SPLAT_KMEM_TEST13_NAME "slab_reclaim"
#define SPLAT_KMEM_TEST13_DESC "Slab direct memory reclaim test"
#define SPLAT_KMEM_ALLOC_COUNT 10
#define SPLAT_VMEM_ALLOC_COUNT 10
static int
splat_kmem_test1(struct file *file, void *arg)
{
void *ptr[SPLAT_KMEM_ALLOC_COUNT];
int size = PAGE_SIZE;
int i, count, rc = 0;
while ((!rc) && (size <= (PAGE_SIZE * 32))) {
count = 0;
for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++) {
ptr[i] = kmem_alloc(size, KM_SLEEP | KM_NODEBUG);
if (ptr[i])
count++;
}
for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++)
if (ptr[i])
kmem_free(ptr[i], size);
splat_vprint(file, SPLAT_KMEM_TEST1_NAME,
"%d byte allocations, %d/%d successful\n",
size, count, SPLAT_KMEM_ALLOC_COUNT);
if (count != SPLAT_KMEM_ALLOC_COUNT)
rc = -ENOMEM;
size *= 2;
}
return rc;
}
static int
splat_kmem_test2(struct file *file, void *arg)
{
void *ptr[SPLAT_KMEM_ALLOC_COUNT];
int size = PAGE_SIZE;
int i, j, count, rc = 0;
while ((!rc) && (size <= (PAGE_SIZE * 32))) {
count = 0;
for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++) {
ptr[i] = kmem_zalloc(size, KM_SLEEP | KM_NODEBUG);
if (ptr[i])
count++;
}
/* Ensure buffer has been zero filled */
for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++) {
for (j = 0; j < size; j++) {
if (((char *)ptr[i])[j] != '\0') {
splat_vprint(file,SPLAT_KMEM_TEST2_NAME,
"%d-byte allocation was "
"not zeroed\n", size);
rc = -EFAULT;
}
}
}
for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++)
if (ptr[i])
kmem_free(ptr[i], size);
splat_vprint(file, SPLAT_KMEM_TEST2_NAME,
"%d byte allocations, %d/%d successful\n",
size, count, SPLAT_KMEM_ALLOC_COUNT);
if (count != SPLAT_KMEM_ALLOC_COUNT)
rc = -ENOMEM;
size *= 2;
}
return rc;
}
static int
splat_kmem_test3(struct file *file, void *arg)
{
void *ptr[SPLAT_VMEM_ALLOC_COUNT];
int size = PAGE_SIZE;
int i, count, rc = 0;
while ((!rc) && (size <= (PAGE_SIZE * 1024))) {
count = 0;
for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++) {
ptr[i] = vmem_alloc(size, KM_SLEEP);
if (ptr[i])
count++;
}
for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++)
if (ptr[i])
vmem_free(ptr[i], size);
splat_vprint(file, SPLAT_KMEM_TEST3_NAME,
"%d byte allocations, %d/%d successful\n",
size, count, SPLAT_VMEM_ALLOC_COUNT);
if (count != SPLAT_VMEM_ALLOC_COUNT)
rc = -ENOMEM;
size *= 2;
}
return rc;
}
static int
splat_kmem_test4(struct file *file, void *arg)
{
void *ptr[SPLAT_VMEM_ALLOC_COUNT];
int size = PAGE_SIZE;
int i, j, count, rc = 0;
while ((!rc) && (size <= (PAGE_SIZE * 1024))) {
count = 0;
for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++) {
ptr[i] = vmem_zalloc(size, KM_SLEEP);
if (ptr[i])
count++;
}
/* Ensure buffer has been zero filled */
for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++) {
for (j = 0; j < size; j++) {
if (((char *)ptr[i])[j] != '\0') {
splat_vprint(file, SPLAT_KMEM_TEST4_NAME,
"%d-byte allocation was "
"not zeroed\n", size);
rc = -EFAULT;
}
}
}
for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++)
if (ptr[i])
vmem_free(ptr[i], size);
splat_vprint(file, SPLAT_KMEM_TEST4_NAME,
"%d byte allocations, %d/%d successful\n",
size, count, SPLAT_VMEM_ALLOC_COUNT);
if (count != SPLAT_VMEM_ALLOC_COUNT)
rc = -ENOMEM;
size *= 2;
}
return rc;
}
#define SPLAT_KMEM_TEST_MAGIC 0x004488CCUL
#define SPLAT_KMEM_CACHE_NAME "kmem_test"
#define SPLAT_KMEM_OBJ_COUNT 1024
#define SPLAT_KMEM_OBJ_RECLAIM 32 /* objects */
#define SPLAT_KMEM_THREADS 32
#define KCP_FLAG_READY 0x01
typedef struct kmem_cache_data {
unsigned long kcd_magic;
struct list_head kcd_node;
int kcd_flag;
char kcd_buf[0];
} kmem_cache_data_t;
typedef struct kmem_cache_thread {
spinlock_t kct_lock;
int kct_id;
struct list_head kct_list;
} kmem_cache_thread_t;
typedef struct kmem_cache_priv {
unsigned long kcp_magic;
struct file *kcp_file;
kmem_cache_t *kcp_cache;
spinlock_t kcp_lock;
wait_queue_head_t kcp_ctl_waitq;
wait_queue_head_t kcp_thr_waitq;
int kcp_flags;
int kcp_kct_count;
kmem_cache_thread_t *kcp_kct[SPLAT_KMEM_THREADS];
int kcp_size;
int kcp_align;
int kcp_count;
int kcp_alloc;
int kcp_rc;
} kmem_cache_priv_t;
static kmem_cache_priv_t *
splat_kmem_cache_test_kcp_alloc(struct file *file, char *name,
int size, int align, int alloc)
{
kmem_cache_priv_t *kcp;
kcp = kmem_zalloc(sizeof(kmem_cache_priv_t), KM_SLEEP);
if (!kcp)
return NULL;
kcp->kcp_magic = SPLAT_KMEM_TEST_MAGIC;
kcp->kcp_file = file;
kcp->kcp_cache = NULL;
spin_lock_init(&kcp->kcp_lock);
init_waitqueue_head(&kcp->kcp_ctl_waitq);
init_waitqueue_head(&kcp->kcp_thr_waitq);
kcp->kcp_flags = 0;
kcp->kcp_kct_count = -1;
kcp->kcp_size = size;
kcp->kcp_align = align;
kcp->kcp_count = 0;
kcp->kcp_alloc = alloc;
kcp->kcp_rc = 0;
return kcp;
}
static void
splat_kmem_cache_test_kcp_free(kmem_cache_priv_t *kcp)
{
kmem_free(kcp, sizeof(kmem_cache_priv_t));
}
static kmem_cache_thread_t *
splat_kmem_cache_test_kct_alloc(kmem_cache_priv_t *kcp, int id)
{
kmem_cache_thread_t *kct;
ASSERT3S(id, <, SPLAT_KMEM_THREADS);
ASSERT(kcp->kcp_kct[id] == NULL);
kct = kmem_zalloc(sizeof(kmem_cache_thread_t), KM_SLEEP);
if (!kct)
return NULL;
spin_lock_init(&kct->kct_lock);
kct->kct_id = id;
INIT_LIST_HEAD(&kct->kct_list);
spin_lock(&kcp->kcp_lock);
kcp->kcp_kct[id] = kct;
spin_unlock(&kcp->kcp_lock);
return kct;
}
static void
splat_kmem_cache_test_kct_free(kmem_cache_priv_t *kcp,
kmem_cache_thread_t *kct)
{
spin_lock(&kcp->kcp_lock);
kcp->kcp_kct[kct->kct_id] = NULL;
spin_unlock(&kcp->kcp_lock);
kmem_free(kct, sizeof(kmem_cache_thread_t));
}
static void
splat_kmem_cache_test_kcd_free(kmem_cache_priv_t *kcp,
kmem_cache_thread_t *kct)
{
kmem_cache_data_t *kcd;
spin_lock(&kct->kct_lock);
while (!list_empty(&kct->kct_list)) {
kcd = list_entry(kct->kct_list.next,
kmem_cache_data_t, kcd_node);
list_del(&kcd->kcd_node);
spin_unlock(&kct->kct_lock);
kmem_cache_free(kcp->kcp_cache, kcd);
spin_lock(&kct->kct_lock);
}
spin_unlock(&kct->kct_lock);
}
static int
splat_kmem_cache_test_kcd_alloc(kmem_cache_priv_t *kcp,
kmem_cache_thread_t *kct, int count)
{
kmem_cache_data_t *kcd;
int i;
for (i = 0; i < count; i++) {
kcd = kmem_cache_alloc(kcp->kcp_cache, KM_SLEEP);
if (kcd == NULL) {
splat_kmem_cache_test_kcd_free(kcp, kct);
return -ENOMEM;
}
spin_lock(&kct->kct_lock);
list_add_tail(&kcd->kcd_node, &kct->kct_list);
spin_unlock(&kct->kct_lock);
}
return 0;
}
static void
splat_kmem_cache_test_debug(struct file *file, char *name,
kmem_cache_priv_t *kcp)
{
int j;
splat_vprint(file, name, "%s cache objects %d",
kcp->kcp_cache->skc_name, kcp->kcp_count);
if (kcp->kcp_cache->skc_flags & (KMC_KMEM | KMC_VMEM)) {
splat_vprint(file, name, ", slabs %u/%u objs %u/%u",
(unsigned)kcp->kcp_cache->skc_slab_alloc,
(unsigned)kcp->kcp_cache->skc_slab_total,
(unsigned)kcp->kcp_cache->skc_obj_alloc,
(unsigned)kcp->kcp_cache->skc_obj_total);
if (!(kcp->kcp_cache->skc_flags & KMC_NOMAGAZINE)) {
splat_vprint(file, name, "%s", "mags");
for_each_online_cpu(j)
splat_print(file, "%u/%u ",
kcp->kcp_cache->skc_mag[j]->skm_avail,
kcp->kcp_cache->skc_mag[j]->skm_size);
}
}
splat_print(file, "%s\n", "");
}
static int
splat_kmem_cache_test_constructor(void *ptr, void *priv, int flags)
{
kmem_cache_priv_t *kcp = (kmem_cache_priv_t *)priv;
kmem_cache_data_t *kcd = (kmem_cache_data_t *)ptr;
if (kcd && kcp) {
kcd->kcd_magic = kcp->kcp_magic;
INIT_LIST_HEAD(&kcd->kcd_node);
kcd->kcd_flag = 1;
memset(kcd->kcd_buf, 0xaa, kcp->kcp_size - (sizeof *kcd));
kcp->kcp_count++;
}
return 0;
}
static void
splat_kmem_cache_test_destructor(void *ptr, void *priv)
{
kmem_cache_priv_t *kcp = (kmem_cache_priv_t *)priv;
kmem_cache_data_t *kcd = (kmem_cache_data_t *)ptr;
if (kcd && kcp) {
kcd->kcd_magic = 0;
kcd->kcd_flag = 0;
memset(kcd->kcd_buf, 0xbb, kcp->kcp_size - (sizeof *kcd));
kcp->kcp_count--;
}
return;
}
/*
* Generic reclaim function which assumes that all objects may
* be reclaimed at any time. We free a small percentage of the
* objects linked off the kcp or kct[] every time we are called.
*/
static void
splat_kmem_cache_test_reclaim(void *priv)
{
kmem_cache_priv_t *kcp = (kmem_cache_priv_t *)priv;
kmem_cache_thread_t *kct;
kmem_cache_data_t *kcd;
LIST_HEAD(reclaim);
int i, count;
ASSERT(kcp->kcp_magic == SPLAT_KMEM_TEST_MAGIC);
/* For each kct thread reclaim some objects */
spin_lock(&kcp->kcp_lock);
for (i = 0; i < SPLAT_KMEM_THREADS; i++) {
kct = kcp->kcp_kct[i];
if (!kct)
continue;
spin_unlock(&kcp->kcp_lock);
spin_lock(&kct->kct_lock);
count = SPLAT_KMEM_OBJ_RECLAIM;
while (count > 0 && !list_empty(&kct->kct_list)) {
kcd = list_entry(kct->kct_list.next,
kmem_cache_data_t, kcd_node);
list_del(&kcd->kcd_node);
list_add(&kcd->kcd_node, &reclaim);
count--;
}
spin_unlock(&kct->kct_lock);
spin_lock(&kcp->kcp_lock);
}
spin_unlock(&kcp->kcp_lock);
/* Freed outside the spin lock */
while (!list_empty(&reclaim)) {
kcd = list_entry(reclaim.next, kmem_cache_data_t, kcd_node);
list_del(&kcd->kcd_node);
kmem_cache_free(kcp->kcp_cache, kcd);
}
return;
}
static int
splat_kmem_cache_test_threads(kmem_cache_priv_t *kcp, int threads)
{
int rc;
spin_lock(&kcp->kcp_lock);
rc = (kcp->kcp_kct_count == threads);
spin_unlock(&kcp->kcp_lock);
return rc;
}
static int
splat_kmem_cache_test_flags(kmem_cache_priv_t *kcp, int flags)
{
int rc;
spin_lock(&kcp->kcp_lock);
rc = (kcp->kcp_flags & flags);
spin_unlock(&kcp->kcp_lock);
return rc;
}
static void
splat_kmem_cache_test_thread(void *arg)
{
kmem_cache_priv_t *kcp = (kmem_cache_priv_t *)arg;
kmem_cache_thread_t *kct;
int rc = 0, id;
ASSERT(kcp->kcp_magic == SPLAT_KMEM_TEST_MAGIC);
/* Assign thread ids */
spin_lock(&kcp->kcp_lock);
if (kcp->kcp_kct_count == -1)
kcp->kcp_kct_count = 0;
id = kcp->kcp_kct_count;
kcp->kcp_kct_count++;
spin_unlock(&kcp->kcp_lock);
kct = splat_kmem_cache_test_kct_alloc(kcp, id);
if (!kct) {
rc = -ENOMEM;
goto out;
}
/* Wait for all threads to have started and report they are ready */
if (kcp->kcp_kct_count == SPLAT_KMEM_THREADS)
wake_up(&kcp->kcp_ctl_waitq);
wait_event(kcp->kcp_thr_waitq,
splat_kmem_cache_test_flags(kcp, KCP_FLAG_READY));
/* Create and destroy objects */
rc = splat_kmem_cache_test_kcd_alloc(kcp, kct, kcp->kcp_alloc);
splat_kmem_cache_test_kcd_free(kcp, kct);
out:
if (kct)
splat_kmem_cache_test_kct_free(kcp, kct);
spin_lock(&kcp->kcp_lock);
if (!kcp->kcp_rc)
kcp->kcp_rc = rc;
if ((--kcp->kcp_kct_count) == 0)
wake_up(&kcp->kcp_ctl_waitq);
spin_unlock(&kcp->kcp_lock);
thread_exit();
}
static int
splat_kmem_cache_test(struct file *file, void *arg, char *name,
int size, int align, int flags)
{
kmem_cache_priv_t *kcp;
kmem_cache_data_t *kcd = NULL;
int rc = 0, max;
kcp = splat_kmem_cache_test_kcp_alloc(file, name, size, align, 0);
if (!kcp) {
splat_vprint(file, name, "Unable to create '%s'\n", "kcp");
return -ENOMEM;
}
kcp->kcp_cache =
kmem_cache_create(SPLAT_KMEM_CACHE_NAME,
kcp->kcp_size, kcp->kcp_align,
splat_kmem_cache_test_constructor,
splat_kmem_cache_test_destructor,
NULL, kcp, NULL, flags);
if (!kcp->kcp_cache) {
splat_vprint(file, name,
"Unable to create '%s'\n",
SPLAT_KMEM_CACHE_NAME);
rc = -ENOMEM;
goto out_free;
}
kcd = kmem_cache_alloc(kcp->kcp_cache, KM_SLEEP);
if (!kcd) {
splat_vprint(file, name,
"Unable to allocate from '%s'\n",
SPLAT_KMEM_CACHE_NAME);
rc = -EINVAL;
goto out_free;
}
if (!kcd->kcd_flag) {
splat_vprint(file, name,
"Failed to run contructor for '%s'\n",
SPLAT_KMEM_CACHE_NAME);
rc = -EINVAL;
goto out_free;
}
if (kcd->kcd_magic != kcp->kcp_magic) {
splat_vprint(file, name,
"Failed to pass private data to constructor "
"for '%s'\n", SPLAT_KMEM_CACHE_NAME);
rc = -EINVAL;
goto out_free;
}
max = kcp->kcp_count;
kmem_cache_free(kcp->kcp_cache, kcd);
/* Destroy the entire cache which will force destructors to
* run and we can verify one was called for every object */
kmem_cache_destroy(kcp->kcp_cache);
if (kcp->kcp_count) {
splat_vprint(file, name,
"Failed to run destructor on all slab objects "
"for '%s'\n", SPLAT_KMEM_CACHE_NAME);
rc = -EINVAL;
}
splat_kmem_cache_test_kcp_free(kcp);
splat_vprint(file, name,
"Successfully ran ctors/dtors for %d elements in '%s'\n",
max, SPLAT_KMEM_CACHE_NAME);
return rc;
out_free:
if (kcd)
kmem_cache_free(kcp->kcp_cache, kcd);
if (kcp->kcp_cache)
kmem_cache_destroy(kcp->kcp_cache);
splat_kmem_cache_test_kcp_free(kcp);
return rc;
}
static int
splat_kmem_cache_thread_test(struct file *file, void *arg, char *name,
int size, int alloc, int max_time)
{
kmem_cache_priv_t *kcp;
kthread_t *thr;
struct timespec start, stop, delta;
char cache_name[32];
int i, rc = 0;
kcp = splat_kmem_cache_test_kcp_alloc(file, name, size, 0, alloc);
if (!kcp) {
splat_vprint(file, name, "Unable to create '%s'\n", "kcp");
return -ENOMEM;
}
(void)snprintf(cache_name, 32, "%s-%d-%d",
SPLAT_KMEM_CACHE_NAME, size, alloc);
kcp->kcp_cache =
kmem_cache_create(cache_name, kcp->kcp_size, 0,
splat_kmem_cache_test_constructor,
splat_kmem_cache_test_destructor,
splat_kmem_cache_test_reclaim,
kcp, NULL, 0);
if (!kcp->kcp_cache) {
splat_vprint(file, name, "Unable to create '%s'\n", cache_name);
rc = -ENOMEM;
goto out_kcp;
}
getnstimeofday(&start);
for (i = 0; i < SPLAT_KMEM_THREADS; i++) {
thr = thread_create(NULL, 0,
splat_kmem_cache_test_thread,
kcp, 0, &p0, TS_RUN, minclsyspri);
if (thr == NULL) {
rc = -ESRCH;
goto out_cache;
}
}
/* Sleep until all threads have started, then set the ready
* flag and wake them all up for maximum concurrency. */
wait_event(kcp->kcp_ctl_waitq,
splat_kmem_cache_test_threads(kcp, SPLAT_KMEM_THREADS));
spin_lock(&kcp->kcp_lock);
kcp->kcp_flags |= KCP_FLAG_READY;
spin_unlock(&kcp->kcp_lock);
wake_up_all(&kcp->kcp_thr_waitq);
/* Sleep until all thread have finished */
wait_event(kcp->kcp_ctl_waitq, splat_kmem_cache_test_threads(kcp, 0));
getnstimeofday(&stop);
delta = timespec_sub(stop, start);
splat_vprint(file, name,
"%-22s %2ld.%09ld\t"
"%lu/%lu/%lu\t%lu/%lu/%lu\n",
kcp->kcp_cache->skc_name,
delta.tv_sec, delta.tv_nsec,
(unsigned long)kcp->kcp_cache->skc_slab_total,
(unsigned long)kcp->kcp_cache->skc_slab_max,
(unsigned long)(kcp->kcp_alloc *
SPLAT_KMEM_THREADS /
SPL_KMEM_CACHE_OBJ_PER_SLAB),
(unsigned long)kcp->kcp_cache->skc_obj_total,
(unsigned long)kcp->kcp_cache->skc_obj_max,
(unsigned long)(kcp->kcp_alloc *
SPLAT_KMEM_THREADS));
if (delta.tv_sec >= max_time)
rc = -ETIME;
if (!rc && kcp->kcp_rc)
rc = kcp->kcp_rc;
out_cache:
kmem_cache_destroy(kcp->kcp_cache);
out_kcp:
splat_kmem_cache_test_kcp_free(kcp);
return rc;
}
/* Validate small object cache behavior for dynamic/kmem/vmem caches */
static int
splat_kmem_test5(struct file *file, void *arg)
{
char *name = SPLAT_KMEM_TEST5_NAME;
int rc;
/* On slab (default + kmem + vmem) */
rc = splat_kmem_cache_test(file, arg, name, 128, 0, 0);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 128, 0, KMC_KMEM);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 128, 0, KMC_VMEM);
if (rc)
return rc;
/* Off slab (default + kmem + vmem) */
rc = splat_kmem_cache_test(file, arg, name, 128, 0, KMC_OFFSLAB);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 128, 0,
KMC_KMEM | KMC_OFFSLAB);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 128, 0,
KMC_VMEM | KMC_OFFSLAB);
return rc;
}
/*
* Validate large object cache behavior for dynamic/kmem/vmem caches
*/
static int
splat_kmem_test6(struct file *file, void *arg)
{
char *name = SPLAT_KMEM_TEST6_NAME;
int rc;
/* On slab (default + kmem + vmem) */
rc = splat_kmem_cache_test(file, arg, name, 256*1024, 0, 0);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 64*1024, 0, KMC_KMEM);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 1024*1024, 0, KMC_VMEM);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 16*1024*1024, 0, KMC_VMEM);
if (rc)
return rc;
/* Off slab (default + kmem + vmem) */
rc = splat_kmem_cache_test(file, arg, name, 256*1024, 0, KMC_OFFSLAB);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 64*1024, 0,
KMC_KMEM | KMC_OFFSLAB);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 1024*1024, 0,
KMC_VMEM | KMC_OFFSLAB);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 16*1024*1024, 0,
KMC_VMEM | KMC_OFFSLAB);
return rc;
}
/*
* Validate object alignment cache behavior for caches
*/
static int
splat_kmem_test7(struct file *file, void *arg)
{
char *name = SPLAT_KMEM_TEST7_NAME;
int i, rc;
for (i = SPL_KMEM_CACHE_ALIGN; i <= PAGE_SIZE; i *= 2) {
rc = splat_kmem_cache_test(file, arg, name, 157, i, 0);
if (rc)
return rc;
rc = splat_kmem_cache_test(file, arg, name, 157, i,
KMC_OFFSLAB);
if (rc)
return rc;
}
return rc;
}
/*
* Validate kmem_cache_reap() by requesting the slab cache free any objects
* it can. For a few reasons this may not immediately result in more free
* memory even if objects are freed. First off, due to fragmentation we
* may not be able to reclaim any slabs. Secondly, even if we do we fully
* clear some slabs we will not want to immediately reclaim all of them
* because we may contend with cache allocations and thrash. What we want
* to see is the slab size decrease more gradually as it becomes clear they
* will not be needed. This should be achievable in less than a minute.
* If it takes longer than this something has gone wrong.
*/
static int
splat_kmem_test8(struct file *file, void *arg)
{
kmem_cache_priv_t *kcp;
kmem_cache_thread_t *kct;
unsigned int spl_kmem_cache_expire_old;
int i, rc = 0;
/* Enable cache aging just for this test if it is disabled */
spl_kmem_cache_expire_old = spl_kmem_cache_expire;
spl_kmem_cache_expire = KMC_EXPIRE_AGE;
kcp = splat_kmem_cache_test_kcp_alloc(file, SPLAT_KMEM_TEST8_NAME,
256, 0, 0);
if (!kcp) {
splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
"Unable to create '%s'\n", "kcp");
rc = -ENOMEM;
goto out;
}
kcp->kcp_cache =
kmem_cache_create(SPLAT_KMEM_CACHE_NAME, kcp->kcp_size, 0,
splat_kmem_cache_test_constructor,
splat_kmem_cache_test_destructor,
splat_kmem_cache_test_reclaim,
kcp, NULL, 0);
if (!kcp->kcp_cache) {
splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
"Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME);
rc = -ENOMEM;
goto out_kcp;
}
kct = splat_kmem_cache_test_kct_alloc(kcp, 0);
if (!kct) {
splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
"Unable to create '%s'\n", "kct");
rc = -ENOMEM;
goto out_cache;
}
rc = splat_kmem_cache_test_kcd_alloc(kcp, kct, SPLAT_KMEM_OBJ_COUNT);
if (rc) {
splat_vprint(file, SPLAT_KMEM_TEST8_NAME, "Unable to "
"allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME);
goto out_kct;
}
/* Force reclaim every 1/10 a second for 60 seconds. */
for (i = 0; i < 600; i++) {
kmem_cache_reap_now(kcp->kcp_cache);
splat_kmem_cache_test_debug(file, SPLAT_KMEM_TEST8_NAME, kcp);
if (kcp->kcp_count == 0)
break;
set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(HZ / 10);
}
if (kcp->kcp_count == 0) {
splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
"Successfully created %d objects "
"in cache %s and reclaimed them\n",
SPLAT_KMEM_OBJ_COUNT, SPLAT_KMEM_CACHE_NAME);
} else {
splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
"Failed to reclaim %u/%d objects from cache %s\n",
(unsigned)kcp->kcp_count,
SPLAT_KMEM_OBJ_COUNT, SPLAT_KMEM_CACHE_NAME);
rc = -ENOMEM;
}
/* Cleanup our mess (for failure case of time expiring) */
splat_kmem_cache_test_kcd_free(kcp, kct);
out_kct:
splat_kmem_cache_test_kct_free(kcp, kct);
out_cache:
kmem_cache_destroy(kcp->kcp_cache);
out_kcp:
splat_kmem_cache_test_kcp_free(kcp);
out:
spl_kmem_cache_expire = spl_kmem_cache_expire_old;
return rc;
}
/* Test cache aging, we have allocated a large number of objects thus
* creating a large number of slabs and then free'd them all. However,
* since there should be little memory pressure at the moment those
* slabs have not been freed. What we want to see is the slab size
* decrease gradually as it becomes clear they will not be be needed.
* This should be achievable in less than minute. If it takes longer
* than this something has gone wrong.
*/
static int
splat_kmem_test9(struct file *file, void *arg)
{
kmem_cache_priv_t *kcp;
kmem_cache_thread_t *kct;
unsigned int spl_kmem_cache_expire_old;
int i, rc = 0, count = SPLAT_KMEM_OBJ_COUNT * 128;
/* Enable cache aging just for this test if it is disabled */
spl_kmem_cache_expire_old = spl_kmem_cache_expire;
spl_kmem_cache_expire = KMC_EXPIRE_AGE;
kcp = splat_kmem_cache_test_kcp_alloc(file, SPLAT_KMEM_TEST9_NAME,
256, 0, 0);
if (!kcp) {
splat_vprint(file, SPLAT_KMEM_TEST9_NAME,
"Unable to create '%s'\n", "kcp");
rc = -ENOMEM;
goto out;
}
kcp->kcp_cache =
kmem_cache_create(SPLAT_KMEM_CACHE_NAME, kcp->kcp_size, 0,
splat_kmem_cache_test_constructor,
splat_kmem_cache_test_destructor,
NULL, kcp, NULL, 0);
if (!kcp->kcp_cache) {
splat_vprint(file, SPLAT_KMEM_TEST9_NAME,
"Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME);
rc = -ENOMEM;
goto out_kcp;
}
kct = splat_kmem_cache_test_kct_alloc(kcp, 0);
if (!kct) {
splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
"Unable to create '%s'\n", "kct");
rc = -ENOMEM;
goto out_cache;
}
rc = splat_kmem_cache_test_kcd_alloc(kcp, kct, count);
if (rc) {
splat_vprint(file, SPLAT_KMEM_TEST9_NAME, "Unable to "
"allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME);
goto out_kct;
}
splat_kmem_cache_test_kcd_free(kcp, kct);
for (i = 0; i < 60; i++) {
splat_kmem_cache_test_debug(file, SPLAT_KMEM_TEST9_NAME, kcp);
if (kcp->kcp_count == 0)
break;
set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(HZ);
}
if (kcp->kcp_count == 0) {
splat_vprint(file, SPLAT_KMEM_TEST9_NAME,
"Successfully created %d objects "
"in cache %s and reclaimed them\n",
count, SPLAT_KMEM_CACHE_NAME);
} else {
splat_vprint(file, SPLAT_KMEM_TEST9_NAME,
"Failed to reclaim %u/%d objects from cache %s\n",
(unsigned)kcp->kcp_count, count,
SPLAT_KMEM_CACHE_NAME);
rc = -ENOMEM;
}
out_kct:
splat_kmem_cache_test_kct_free(kcp, kct);
out_cache:
kmem_cache_destroy(kcp->kcp_cache);
out_kcp:
splat_kmem_cache_test_kcp_free(kcp);
out:
spl_kmem_cache_expire = spl_kmem_cache_expire_old;
return rc;
}
/*
* This test creates N threads with a shared kmem cache. They then all
* concurrently allocate and free from the cache to stress the locking and
* concurrent cache performance. If any one test takes longer than 5
* seconds to complete it is treated as a failure and may indicate a
* performance regression. On my test system no one test takes more
* than 1 second to complete so a 5x slowdown likely a problem.
*/
static int
splat_kmem_test10(struct file *file, void *arg)
{
uint64_t size, alloc, rc = 0;
for (size = 32; size <= 1024*1024; size *= 2) {
splat_vprint(file, SPLAT_KMEM_TEST10_NAME, "%-22s %s", "name",
"time (sec)\tslabs \tobjs \thash\n");
splat_vprint(file, SPLAT_KMEM_TEST10_NAME, "%-22s %s", "",
" \ttot/max/calc\ttot/max/calc\n");
for (alloc = 1; alloc <= 1024; alloc *= 2) {
/* Skip tests which exceed 1/2 of physical memory. */
if (size * alloc * SPLAT_KMEM_THREADS > physmem / 2)
continue;
rc = splat_kmem_cache_thread_test(file, arg,
SPLAT_KMEM_TEST10_NAME, size, alloc, 5);
if (rc)
break;
}
}
return rc;
}
#if 0
/*
* This test creates N threads with a shared kmem cache which overcommits
* memory by 4x. This makes it impossible for the slab to satify the
* thread requirements without having its reclaim hook run which will
* free objects back for use. This behavior is triggered by the linum VM
* detecting a low memory condition on the node and invoking the shrinkers.
* This should allow all the threads to complete while avoiding deadlock
* and for the most part out of memory events. This is very tough on the
* system so it is possible the test app may get oom'ed. This particular
* test has proven troublesome on 32-bit archs with limited virtual
* address space so it only run on 64-bit systems.
*/
static int
splat_kmem_test11(struct file *file, void *arg)
{
uint64_t size, alloc, rc;
size = 8 * 1024;
alloc = ((4 * physmem * PAGE_SIZE) / size) / SPLAT_KMEM_THREADS;
splat_vprint(file, SPLAT_KMEM_TEST11_NAME, "%-22s %s", "name",
"time (sec)\tslabs \tobjs \thash\n");
splat_vprint(file, SPLAT_KMEM_TEST11_NAME, "%-22s %s", "",
" \ttot/max/calc\ttot/max/calc\n");
rc = splat_kmem_cache_thread_test(file, arg,
SPLAT_KMEM_TEST11_NAME, size, alloc, 60);
return rc;
}
#endif
typedef struct dummy_page {
struct list_head dp_list;
char dp_pad[PAGE_SIZE - sizeof(struct list_head)];
} dummy_page_t;
/*
* This test is designed to verify that direct reclaim is functioning as
* expected. We allocate a large number of objects thus creating a large
* number of slabs. We then apply memory pressure and expect that the
* direct reclaim path can easily recover those slabs. The registered
* reclaim function will free the objects and the slab shrinker will call
* it repeatedly until at least a single slab can be freed.
*
* Note it may not be possible to reclaim every last slab via direct reclaim
* without a failure because the shrinker_rwsem may be contended. For this
* reason, quickly reclaiming 3/4 of the slabs is considered a success.
*
* This should all be possible within 10 seconds. For reference, on a
* system with 2G of memory this test takes roughly 0.2 seconds to run.
* It may take longer on larger memory systems but should still easily
* complete in the alloted 10 seconds.
*/
static int
splat_kmem_test13(struct file *file, void *arg)
{
kmem_cache_priv_t *kcp;
kmem_cache_thread_t *kct;
dummy_page_t *dp;
struct list_head list;
struct timespec start, stop, delta = { 0, 0 };
int size, count, slabs, fails = 0;
int i, rc = 0, max_time = 10;
size = 128 * 1024;
count = ((physmem * PAGE_SIZE) / 4 / size);
kcp = splat_kmem_cache_test_kcp_alloc(file, SPLAT_KMEM_TEST13_NAME,
size, 0, 0);
if (!kcp) {
splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
"Unable to create '%s'\n", "kcp");
rc = -ENOMEM;
goto out;
}
kcp->kcp_cache =
kmem_cache_create(SPLAT_KMEM_CACHE_NAME, kcp->kcp_size, 0,
splat_kmem_cache_test_constructor,
splat_kmem_cache_test_destructor,
splat_kmem_cache_test_reclaim,
kcp, NULL, 0);
if (!kcp->kcp_cache) {
splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
"Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME);
rc = -ENOMEM;
goto out_kcp;
}
kct = splat_kmem_cache_test_kct_alloc(kcp, 0);
if (!kct) {
splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
"Unable to create '%s'\n", "kct");
rc = -ENOMEM;
goto out_cache;
}
rc = splat_kmem_cache_test_kcd_alloc(kcp, kct, count);
if (rc) {
splat_vprint(file, SPLAT_KMEM_TEST13_NAME, "Unable to "
"allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME);
goto out_kct;
}
i = 0;
slabs = kcp->kcp_cache->skc_slab_total;
INIT_LIST_HEAD(&list);
getnstimeofday(&start);
/* Apply memory pressure */
while (kcp->kcp_cache->skc_slab_total > (slabs >> 2)) {
if ((i % 10000) == 0)
splat_kmem_cache_test_debug(
file, SPLAT_KMEM_TEST13_NAME, kcp);
getnstimeofday(&stop);
delta = timespec_sub(stop, start);
if (delta.tv_sec >= max_time) {
splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
"Failed to reclaim 3/4 of cache in %ds, "
"%u/%u slabs remain\n", max_time,
(unsigned)kcp->kcp_cache->skc_slab_total,
slabs);
rc = -ETIME;
break;
}
dp = (dummy_page_t *)__get_free_page(GFP_KERNEL);
if (!dp) {
fails++;
splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
"Failed (%d) to allocate page with %u "
"slabs still in the cache\n", fails,
(unsigned)kcp->kcp_cache->skc_slab_total);
continue;
}
list_add(&dp->dp_list, &list);
i++;
}
if (rc == 0)
splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
"Successfully created %u slabs and with %d alloc "
"failures reclaimed 3/4 of them in %d.%03ds\n",
slabs, fails,
(int)delta.tv_sec, (int)delta.tv_nsec / 1000000);
/* Release memory pressure pages */
while (!list_empty(&list)) {
dp = list_entry(list.next, dummy_page_t, dp_list);
list_del_init(&dp->dp_list);
free_page((unsigned long)dp);
}
/* Release remaining kmem cache objects */
splat_kmem_cache_test_kcd_free(kcp, kct);
out_kct:
splat_kmem_cache_test_kct_free(kcp, kct);
out_cache:
kmem_cache_destroy(kcp->kcp_cache);
out_kcp:
splat_kmem_cache_test_kcp_free(kcp);
out:
return rc;
}
splat_subsystem_t *
splat_kmem_init(void)
{
splat_subsystem_t *sub;
sub = kmalloc(sizeof(*sub), GFP_KERNEL);
if (sub == NULL)
return NULL;
memset(sub, 0, sizeof(*sub));
strncpy(sub->desc.name, SPLAT_KMEM_NAME, SPLAT_NAME_SIZE);
strncpy(sub->desc.desc, SPLAT_KMEM_DESC, SPLAT_DESC_SIZE);
INIT_LIST_HEAD(&sub->subsystem_list);
INIT_LIST_HEAD(&sub->test_list);
spin_lock_init(&sub->test_lock);
sub->desc.id = SPLAT_SUBSYSTEM_KMEM;
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST1_NAME, SPLAT_KMEM_TEST1_DESC,
SPLAT_KMEM_TEST1_ID, splat_kmem_test1);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST2_NAME, SPLAT_KMEM_TEST2_DESC,
SPLAT_KMEM_TEST2_ID, splat_kmem_test2);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST3_NAME, SPLAT_KMEM_TEST3_DESC,
SPLAT_KMEM_TEST3_ID, splat_kmem_test3);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST4_NAME, SPLAT_KMEM_TEST4_DESC,
SPLAT_KMEM_TEST4_ID, splat_kmem_test4);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST5_NAME, SPLAT_KMEM_TEST5_DESC,
SPLAT_KMEM_TEST5_ID, splat_kmem_test5);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST6_NAME, SPLAT_KMEM_TEST6_DESC,
SPLAT_KMEM_TEST6_ID, splat_kmem_test6);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST7_NAME, SPLAT_KMEM_TEST7_DESC,
SPLAT_KMEM_TEST7_ID, splat_kmem_test7);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST8_NAME, SPLAT_KMEM_TEST8_DESC,
SPLAT_KMEM_TEST8_ID, splat_kmem_test8);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST9_NAME, SPLAT_KMEM_TEST9_DESC,
SPLAT_KMEM_TEST9_ID, splat_kmem_test9);
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST10_NAME, SPLAT_KMEM_TEST10_DESC,
SPLAT_KMEM_TEST10_ID, splat_kmem_test10);
#if 0
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST11_NAME, SPLAT_KMEM_TEST11_DESC,
SPLAT_KMEM_TEST11_ID, splat_kmem_test11);
#endif
SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST13_NAME, SPLAT_KMEM_TEST13_DESC,
SPLAT_KMEM_TEST13_ID, splat_kmem_test13);
return sub;
}
void
splat_kmem_fini(splat_subsystem_t *sub)
{
ASSERT(sub);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST13_ID);
#if 0
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST11_ID);
#endif
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST10_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST9_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST8_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST7_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST6_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST5_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST4_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST3_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST2_ID);
SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST1_ID);
kfree(sub);
}
int
splat_kmem_id(void) {
return SPLAT_SUBSYSTEM_KMEM;
}
Reference in New Issue View Git Blame Copy Permalink