Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Fix kmem cstyle issues 

Address all cstyle issues in the kmem, vmem, and kmem_cache source
and headers.  This will done to make it easier to review subsequent
changes which will rework the kmem/vmem implementation.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
This commit is contained in:
Brian Behlendorf 2014-12-08 13:35:51 -05:00
parent e5b9b344c7
commit b34b95635a
6 changed files with 333 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
include/sys/kmem.h
View File

@ -1,4 +1,4 @@
/*****************************************************************************\ /*
* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
* Copyright (C) 2007 The Regents of the University of California. * Copyright (C) 2007 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
@ -20,7 +20,7 @@
* *
* You should have received a copy of the GNU General Public License along * You should have received a copy of the GNU General Public License along
* with the SPL. If not, see <http://www.gnu.org/licenses/>. * with the SPL. If not, see <http://www.gnu.org/licenses/>.
\*****************************************************************************/ */
#ifndef _SPL_KMEM_H #ifndef _SPL_KMEM_H
#define _SPL_KMEM_H #define _SPL_KMEM_H
@ -36,18 +36,18 @@ extern void strfree(char *str);
/* /*
* Memory allocation interfaces * Memory allocation interfaces
*/ */
#define KM_SLEEP GFP_KERNEL /* Can sleep, never fails */ #define KM_SLEEP GFP_KERNEL /* Can sleep, never fails */
#define KM_NOSLEEP GFP_ATOMIC /* Can not sleep, may fail */ #define KM_NOSLEEP GFP_ATOMIC /* Can not sleep, may fail */
#define KM_PUSHPAGE (GFP_NOIO | __GFP_HIGH) /* Use reserved memory */ #define KM_PUSHPAGE (GFP_NOIO | __GFP_HIGH) /* Use reserved memory */
#define KM_NODEBUG __GFP_NOWARN /* Suppress warnings */ #define KM_NODEBUG __GFP_NOWARN /* Suppress warnings */
#define KM_FLAGS __GFP_BITS_MASK #define KM_FLAGS __GFP_BITS_MASK
#define KM_VMFLAGS GFP_LEVEL_MASK #define KM_VMFLAGS GFP_LEVEL_MASK
/* /*
* Used internally, the kernel does not need to support this flag * Used internally, the kernel does not need to support this flag
*/ */
#ifndef __GFP_ZERO #ifndef __GFP_ZERO
# define __GFP_ZERO 0x8000 #define __GFP_ZERO 0x8000
#endif #endif
/* /*
@ -66,7 +66,7 @@ kmalloc_nofail(size_t size, gfp_t flags)
ptr = kmalloc(size, flags); ptr = kmalloc(size, flags);
} while (ptr == NULL && (flags & __GFP_WAIT)); } while (ptr == NULL && (flags & __GFP_WAIT));
return ptr; return (ptr);
} }
static inline void * static inline void *
@ -78,7 +78,7 @@ kzalloc_nofail(size_t size, gfp_t flags)
ptr = kzalloc(size, flags); ptr = kzalloc(size, flags);
} while (ptr == NULL && (flags & __GFP_WAIT)); } while (ptr == NULL && (flags & __GFP_WAIT));
return ptr; return (ptr);
} }
static inline void * static inline void *
@ -90,7 +90,7 @@ kmalloc_node_nofail(size_t size, gfp_t flags, int node)
ptr = kmalloc_node(size, flags, node); ptr = kmalloc_node(size, flags, node);
} while (ptr == NULL && (flags & __GFP_WAIT)); } while (ptr == NULL && (flags & __GFP_WAIT));
return ptr; return (ptr);
} }
#ifdef DEBUG_KMEM #ifdef DEBUG_KMEM
@ -98,29 +98,23 @@ kmalloc_node_nofail(size_t size, gfp_t flags, int node)
/* /*
* Memory accounting functions to be used only when DEBUG_KMEM is set. * Memory accounting functions to be used only when DEBUG_KMEM is set.
*/ */
# ifdef HAVE_ATOMIC64_T #ifdef HAVE_ATOMIC64_T
#define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used)
# define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used) #define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used)
# define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used) #define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used)
# define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used) #define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size)
# define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size)
extern atomic64_t kmem_alloc_used; extern atomic64_t kmem_alloc_used;
extern unsigned long long kmem_alloc_max; extern unsigned long long kmem_alloc_max;
#else /* HAVE_ATOMIC64_T */
# else /* HAVE_ATOMIC64_T */ #define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used)
#define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used)
# define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used) #define kmem_alloc_used_read() atomic_read(&kmem_alloc_used)
# define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used) #define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size)
# define kmem_alloc_used_read() atomic_read(&kmem_alloc_used)
# define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size)
extern atomic_t kmem_alloc_used; extern atomic_t kmem_alloc_used;
extern unsigned long long kmem_alloc_max; extern unsigned long long kmem_alloc_max;
#endif /* HAVE_ATOMIC64_T */
# endif /* HAVE_ATOMIC64_T */ #ifdef DEBUG_KMEM_TRACKING
# ifdef DEBUG_KMEM_TRACKING
/* /*
* DEBUG_KMEM && DEBUG_KMEM_TRACKING * DEBUG_KMEM && DEBUG_KMEM_TRACKING
* *
@ -132,18 +126,18 @@ extern unsigned long long kmem_alloc_max;
* be enabled for debugging. This feature may be enabled by passing * be enabled for debugging. This feature may be enabled by passing
* --enable-debug-kmem-tracking to configure. * --enable-debug-kmem-tracking to configure.
*/ */
# define kmem_alloc(sz, fl) kmem_alloc_track((sz), (fl), \ #define kmem_alloc(sz, fl) kmem_alloc_track((sz), (fl), \
__FUNCTION__, __LINE__, 0, 0) __FUNCTION__, __LINE__, 0, 0)
# define kmem_zalloc(sz, fl) kmem_alloc_track((sz), (fl)|__GFP_ZERO,\ #define kmem_zalloc(sz, fl) kmem_alloc_track((sz), (fl)|__GFP_ZERO,\
__FUNCTION__, __LINE__, 0, 0) __FUNCTION__, __LINE__, 0, 0)
# define kmem_alloc_node(sz, fl, nd) kmem_alloc_track((sz), (fl), \ #define kmem_alloc_node(sz, fl, nd) kmem_alloc_track((sz), (fl), \
__FUNCTION__, __LINE__, 1, nd) __FUNCTION__, __LINE__, 1, nd)
# define kmem_free(ptr, sz) kmem_free_track((ptr), (sz)) #define kmem_free(ptr, sz) kmem_free_track((ptr), (sz))
extern void *kmem_alloc_track(size_t, int, const char *, int, int, int); extern void *kmem_alloc_track(size_t, int, const char *, int, int, int);
extern void kmem_free_track(const void *, size_t); extern void kmem_free_track(const void *, size_t);
# else /* DEBUG_KMEM_TRACKING */ #else /* DEBUG_KMEM_TRACKING */
/* /*
* DEBUG_KMEM && !DEBUG_KMEM_TRACKING * DEBUG_KMEM && !DEBUG_KMEM_TRACKING
* *
@ -153,18 +147,18 @@ extern void kmem_free_track(const void *, size_t);
* will be reported on the console. To disable this basic accounting * will be reported on the console. To disable this basic accounting
* pass the --disable-debug-kmem option to configure. * pass the --disable-debug-kmem option to configure.
*/ */
# define kmem_alloc(sz, fl) kmem_alloc_debug((sz), (fl), \ #define kmem_alloc(sz, fl) kmem_alloc_debug((sz), (fl), \
__FUNCTION__, __LINE__, 0, 0) __FUNCTION__, __LINE__, 0, 0)
# define kmem_zalloc(sz, fl) kmem_alloc_debug((sz), (fl)|__GFP_ZERO,\ #define kmem_zalloc(sz, fl) kmem_alloc_debug((sz), (fl)|__GFP_ZERO,\
__FUNCTION__, __LINE__, 0, 0) __FUNCTION__, __LINE__, 0, 0)
# define kmem_alloc_node(sz, fl, nd) kmem_alloc_debug((sz), (fl), \ #define kmem_alloc_node(sz, fl, nd) kmem_alloc_debug((sz), (fl), \
__FUNCTION__, __LINE__, 1, nd) __FUNCTION__, __LINE__, 1, nd)
# define kmem_free(ptr, sz) kmem_free_debug((ptr), (sz)) #define kmem_free(ptr, sz) kmem_free_debug((ptr), (sz))
extern void *kmem_alloc_debug(size_t, int, const char *, int, int, int); extern void *kmem_alloc_debug(size_t, int, const char *, int, int, int);
extern void kmem_free_debug(const void *, size_t); extern void kmem_free_debug(const void *, size_t);
# endif /* DEBUG_KMEM_TRACKING */ #endif /* DEBUG_KMEM_TRACKING */
#else /* DEBUG_KMEM */ #else /* DEBUG_KMEM */
/* /*
* !DEBUG_KMEM && !DEBUG_KMEM_TRACKING * !DEBUG_KMEM && !DEBUG_KMEM_TRACKING
@ -173,17 +167,17 @@ extern void kmem_free_debug(const void *, size_t);
* minimal memory accounting. To enable basic accounting pass the * minimal memory accounting. To enable basic accounting pass the
* --enable-debug-kmem option to configure. * --enable-debug-kmem option to configure.
*/ */
# define kmem_alloc(sz, fl) kmalloc_nofail((sz), (fl)) #define kmem_alloc(sz, fl) kmalloc_nofail((sz), (fl))
# define kmem_zalloc(sz, fl) kzalloc_nofail((sz), (fl)) #define kmem_zalloc(sz, fl) kzalloc_nofail((sz), (fl))
# define kmem_alloc_node(sz, fl, nd) kmalloc_node_nofail((sz), (fl), (nd)) #define kmem_alloc_node(sz, fl, nd) kmalloc_node_nofail((sz), (fl), (nd))
# define kmem_free(ptr, sz) ((void)(sz), kfree(ptr)) #define kmem_free(ptr, sz) ((void)(sz), kfree(ptr))
#endif /* DEBUG_KMEM */ #endif /* DEBUG_KMEM */
int spl_kmem_init(void); int spl_kmem_init(void);
void spl_kmem_fini(void); void spl_kmem_fini(void);
#define kmem_virt(ptr) (((ptr) >= (void *)VMALLOC_START) && \ #define kmem_virt(ptr) (((ptr) >= (void *)VMALLOC_START) && \
((ptr) < (void *)VMALLOC_END)) ((ptr) < (void *)VMALLOC_END))
#endif /* _SPL_KMEM_H */ #endif /* _SPL_KMEM_H */

117
include/sys/kmem_cache.h
View File

@ -1,4 +1,4 @@
/*****************************************************************************\ /*
* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
* Copyright (C) 2007 The Regents of the University of California. * Copyright (C) 2007 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
@ -20,7 +20,7 @@
* *
* You should have received a copy of the GNU General Public License along * You should have received a copy of the GNU General Public License along
* with the SPL. If not, see <http://www.gnu.org/licenses/>. * with the SPL. If not, see <http://www.gnu.org/licenses/>.
\*****************************************************************************/ */
#ifndef _SPL_KMEM_CACHE_H #ifndef _SPL_KMEM_CACHE_H
#define _SPL_KMEM_CACHE_H #define _SPL_KMEM_CACHE_H
@ -33,7 +33,7 @@
* allocated from the physical or virtal memory address space. The virtual * allocated from the physical or virtal memory address space. The virtual
* slabs allow for good behavior when allocation large objects of identical * slabs allow for good behavior when allocation large objects of identical
* size. This slab implementation also supports both constructors and * size. This slab implementation also supports both constructors and
* destructions which the Linux slab does not. * destructors which the Linux slab does not.
*/ */
enum { enum {
KMC_BIT_NOTOUCH = 0, /* Don't update ages */ KMC_BIT_NOTOUCH = 0, /* Don't update ages */
@ -46,8 +46,8 @@ enum {
KMC_BIT_SLAB = 7, /* Use Linux slab cache */ KMC_BIT_SLAB = 7, /* Use Linux slab cache */
KMC_BIT_OFFSLAB = 8, /* Objects not on slab */ KMC_BIT_OFFSLAB = 8, /* Objects not on slab */
KMC_BIT_NOEMERGENCY = 9, /* Disable emergency objects */ KMC_BIT_NOEMERGENCY = 9, /* Disable emergency objects */
KMC_BIT_DEADLOCKED = 14, /* Deadlock detected */ KMC_BIT_DEADLOCKED = 14, /* Deadlock detected */
KMC_BIT_GROWING = 15, /* Growing in progress */ KMC_BIT_GROWING = 15, /* Growing in progress */
KMC_BIT_REAPING = 16, /* Reaping in progress */ KMC_BIT_REAPING = 16, /* Reaping in progress */
KMC_BIT_DESTROY = 17, /* Destroy in progress */ KMC_BIT_DESTROY = 17, /* Destroy in progress */
KMC_BIT_TOTAL = 18, /* Proc handler helper bit */ KMC_BIT_TOTAL = 18, /* Proc handler helper bit */
@ -64,29 +64,29 @@ typedef enum kmem_cbrc {
KMEM_CBRC_DONT_KNOW = 4, /* Object unknown */ KMEM_CBRC_DONT_KNOW = 4, /* Object unknown */
} kmem_cbrc_t; } kmem_cbrc_t;
#define KMC_NOTOUCH (1 << KMC_BIT_NOTOUCH) #define KMC_NOTOUCH (1 << KMC_BIT_NOTOUCH)
#define KMC_NODEBUG (1 << KMC_BIT_NODEBUG) #define KMC_NODEBUG (1 << KMC_BIT_NODEBUG)
#define KMC_NOMAGAZINE (1 << KMC_BIT_NOMAGAZINE) #define KMC_NOMAGAZINE (1 << KMC_BIT_NOMAGAZINE)
#define KMC_NOHASH (1 << KMC_BIT_NOHASH) #define KMC_NOHASH (1 << KMC_BIT_NOHASH)
#define KMC_QCACHE (1 << KMC_BIT_QCACHE) #define KMC_QCACHE (1 << KMC_BIT_QCACHE)
#define KMC_KMEM (1 << KMC_BIT_KMEM) #define KMC_KMEM (1 << KMC_BIT_KMEM)
#define KMC_VMEM (1 << KMC_BIT_VMEM) #define KMC_VMEM (1 << KMC_BIT_VMEM)
#define KMC_SLAB (1 << KMC_BIT_SLAB) #define KMC_SLAB (1 << KMC_BIT_SLAB)
#define KMC_OFFSLAB (1 << KMC_BIT_OFFSLAB) #define KMC_OFFSLAB (1 << KMC_BIT_OFFSLAB)
#define KMC_NOEMERGENCY (1 << KMC_BIT_NOEMERGENCY) #define KMC_NOEMERGENCY (1 << KMC_BIT_NOEMERGENCY)
#define KMC_DEADLOCKED (1 << KMC_BIT_DEADLOCKED) #define KMC_DEADLOCKED (1 << KMC_BIT_DEADLOCKED)
#define KMC_GROWING (1 << KMC_BIT_GROWING) #define KMC_GROWING (1 << KMC_BIT_GROWING)
#define KMC_REAPING (1 << KMC_BIT_REAPING) #define KMC_REAPING (1 << KMC_BIT_REAPING)
#define KMC_DESTROY (1 << KMC_BIT_DESTROY) #define KMC_DESTROY (1 << KMC_BIT_DESTROY)
#define KMC_TOTAL (1 << KMC_BIT_TOTAL) #define KMC_TOTAL (1 << KMC_BIT_TOTAL)
#define KMC_ALLOC (1 << KMC_BIT_ALLOC) #define KMC_ALLOC (1 << KMC_BIT_ALLOC)
#define KMC_MAX (1 << KMC_BIT_MAX) #define KMC_MAX (1 << KMC_BIT_MAX)
#define KMC_REAP_CHUNK INT_MAX #define KMC_REAP_CHUNK INT_MAX
#define KMC_DEFAULT_SEEKS 1 #define KMC_DEFAULT_SEEKS 1
#define KMC_EXPIRE_AGE 0x1 /* Due to age */ #define KMC_EXPIRE_AGE 0x1 /* Due to age */
#define KMC_EXPIRE_MEM 0x2 /* Due to low memory */ #define KMC_EXPIRE_MEM 0x2 /* Due to low memory */
#define KMC_RECLAIM_ONCE 0x1 /* Force a single shrinker pass */ #define KMC_RECLAIM_ONCE 0x1 /* Force a single shrinker pass */
@ -94,19 +94,19 @@ extern unsigned int spl_kmem_cache_expire;
extern struct list_head spl_kmem_cache_list; extern struct list_head spl_kmem_cache_list;
extern struct rw_semaphore spl_kmem_cache_sem; extern struct rw_semaphore spl_kmem_cache_sem;
#define SKM_MAGIC 0x2e2e2e2e #define SKM_MAGIC 0x2e2e2e2e
#define SKO_MAGIC 0x20202020 #define SKO_MAGIC 0x20202020
#define SKS_MAGIC 0x22222222 #define SKS_MAGIC 0x22222222
#define SKC_MAGIC 0x2c2c2c2c #define SKC_MAGIC 0x2c2c2c2c
#define SPL_KMEM_CACHE_DELAY 15 /* Minimum slab release age */ #define SPL_KMEM_CACHE_DELAY 15 /* Minimum slab release age */
#define SPL_KMEM_CACHE_REAP 0 /* Default reap everything */ #define SPL_KMEM_CACHE_REAP 0 /* Default reap everything */
#define SPL_KMEM_CACHE_OBJ_PER_SLAB 16 /* Target objects per slab */ #define SPL_KMEM_CACHE_OBJ_PER_SLAB 16 /* Target objects per slab */
#define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 1 /* Minimum objects per slab */ #define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 1 /* Minimum objects per slab */
#define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */ #define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */
#define POINTER_IS_VALID(p) 0 /* Unimplemented */ #define POINTER_IS_VALID(p) 0 /* Unimplemented */
#define POINTER_INVALIDATE(pp) /* Unimplemented */ #define POINTER_INVALIDATE(pp) /* Unimplemented */
typedef int (*spl_kmem_ctor_t)(void *, void *, int); typedef int (*spl_kmem_ctor_t)(void *, void *, int);
typedef void (*spl_kmem_dtor_t)(void *, void *); typedef void (*spl_kmem_dtor_t)(void *, void *);
@ -124,14 +124,14 @@ typedef struct spl_kmem_magazine {
} spl_kmem_magazine_t; } spl_kmem_magazine_t;
typedef struct spl_kmem_obj { typedef struct spl_kmem_obj {
uint32_t sko_magic; /* Sanity magic */ uint32_t sko_magic; /* Sanity magic */
void *sko_addr; /* Buffer address */ void *sko_addr; /* Buffer address */
struct spl_kmem_slab *sko_slab; /* Owned by slab */ struct spl_kmem_slab *sko_slab; /* Owned by slab */
struct list_head sko_list; /* Free object list linkage */ struct list_head sko_list; /* Free object list linkage */
} spl_kmem_obj_t; } spl_kmem_obj_t;
typedef struct spl_kmem_slab { typedef struct spl_kmem_slab {
uint32_t sks_magic; /* Sanity magic */ uint32_t sks_magic; /* Sanity magic */
uint32_t sks_objs; /* Objects per slab */ uint32_t sks_objs; /* Objects per slab */
struct spl_kmem_cache *sks_cache; /* Owned by cache */ struct spl_kmem_cache *sks_cache; /* Owned by cache */
struct list_head sks_list; /* Slab list linkage */ struct list_head sks_list; /* Slab list linkage */
@ -174,14 +174,14 @@ typedef struct spl_kmem_cache {
atomic_t skc_ref; /* Ref count callers */ atomic_t skc_ref; /* Ref count callers */
taskqid_t skc_taskqid; /* Slab reclaim task */ taskqid_t skc_taskqid; /* Slab reclaim task */
struct list_head skc_list; /* List of caches linkage */ struct list_head skc_list; /* List of caches linkage */
struct list_head skc_complete_list;/* Completely alloc'ed */ struct list_head skc_complete_list; /* Completely alloc'ed */
struct list_head skc_partial_list; /* Partially alloc'ed */ struct list_head skc_partial_list; /* Partially alloc'ed */
struct rb_root skc_emergency_tree; /* Min sized objects */ struct rb_root skc_emergency_tree; /* Min sized objects */
spinlock_t skc_lock; /* Cache lock */ spinlock_t skc_lock; /* Cache lock */
wait_queue_head_t skc_waitq; /* Allocation waiters */ wait_queue_head_t skc_waitq; /* Allocation waiters */
uint64_t skc_slab_fail; /* Slab alloc failures */ uint64_t skc_slab_fail; /* Slab alloc failures */
uint64_t skc_slab_create;/* Slab creates */ uint64_t skc_slab_create; /* Slab creates */
uint64_t skc_slab_destroy;/* Slab destroys */ uint64_t skc_slab_destroy; /* Slab destroys */
uint64_t skc_slab_total; /* Slab total current */ uint64_t skc_slab_total; /* Slab total current */
uint64_t skc_slab_alloc; /* Slab alloc current */ uint64_t skc_slab_alloc; /* Slab alloc current */
uint64_t skc_slab_max; /* Slab max historic */ uint64_t skc_slab_max; /* Slab max historic */
@ -192,30 +192,31 @@ typedef struct spl_kmem_cache {
uint64_t skc_obj_emergency; /* Obj emergency current */ uint64_t skc_obj_emergency; /* Obj emergency current */
uint64_t skc_obj_emergency_max; /* Obj emergency max */ uint64_t skc_obj_emergency_max; /* Obj emergency max */
} spl_kmem_cache_t; } spl_kmem_cache_t;
#define kmem_cache_t spl_kmem_cache_t #define kmem_cache_t spl_kmem_cache_t
extern spl_kmem_cache_t *spl_kmem_cache_create(char *name, size_t size, extern spl_kmem_cache_t *spl_kmem_cache_create(char *name, size_t size,
size_t align, spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, size_t align, spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor,
spl_kmem_reclaim_t reclaim, void *priv, void *vmp, int flags); spl_kmem_reclaim_t reclaim, void *priv, void *vmp, int flags);
extern void spl_kmem_cache_set_move(spl_kmem_cache_t *, extern void spl_kmem_cache_set_move(spl_kmem_cache_t *,
kmem_cbrc_t (*)(void *, void *, size_t, void *)); kmem_cbrc_t (*)(void *, void *, size_t, void *));
extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc); extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc);
extern void *spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags); extern void *spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags);
extern void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj); extern void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj);
extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc, int count); extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc, int count);
extern void spl_kmem_reap(void); extern void spl_kmem_reap(void);
#define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \ #define kmem_cache_create(name, size, align, ctor, dtor, rclm, priv, vmp, fl) \
spl_kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) spl_kmem_cache_create(name, size, align, ctor, dtor, rclm, priv, vmp, fl)
#define kmem_cache_set_move(skc, move) spl_kmem_cache_set_move(skc, move) #define kmem_cache_set_move(skc, move) spl_kmem_cache_set_move(skc, move)
#define kmem_cache_destroy(skc) spl_kmem_cache_destroy(skc) #define kmem_cache_destroy(skc) spl_kmem_cache_destroy(skc)
#define kmem_cache_alloc(skc, flags) spl_kmem_cache_alloc(skc, flags) #define kmem_cache_alloc(skc, flags) spl_kmem_cache_alloc(skc, flags)
#define kmem_cache_free(skc, obj) spl_kmem_cache_free(skc, obj) #define kmem_cache_free(skc, obj) spl_kmem_cache_free(skc, obj)
#define kmem_cache_reap_now(skc) \ #define kmem_cache_reap_now(skc) \
spl_kmem_cache_reap_now(skc, skc->skc_reap) spl_kmem_cache_reap_now(skc, skc->skc_reap)
#define kmem_reap() spl_kmem_reap() #define kmem_reap() spl_kmem_reap()
#define kmem_virt(ptr) (((ptr) >= (void *)VMALLOC_START) && \ #define kmem_virt(ptr) \
((ptr) < (void *)VMALLOC_END)) (((ptr) >= (void *)VMALLOC_START) && \
((ptr) < (void *)VMALLOC_END))
/* /*
* Allow custom slab allocation flags to be set for KMC_SLAB based caches. * Allow custom slab allocation flags to be set for KMC_SLAB based caches.

68
include/sys/vmem.h
View File

@ -1,4 +1,4 @@
/*****************************************************************************\ /*
* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
* Copyright (C) 2007 The Regents of the University of California. * Copyright (C) 2007 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
@ -20,7 +20,7 @@
* *
* You should have received a copy of the GNU General Public License along * You should have received a copy of the GNU General Public License along
* with the SPL. If not, see <http://www.gnu.org/licenses/>. * with the SPL. If not, see <http://www.gnu.org/licenses/>.
\*****************************************************************************/ */
#ifndef _SPL_VMEM_H #ifndef _SPL_VMEM_H
#define _SPL_VMEM_H #define _SPL_VMEM_H
@ -40,11 +40,11 @@ extern size_t vmem_size(vmem_t *vmp, int typemask);
/* /*
* Memory allocation interfaces * Memory allocation interfaces
*/ */
#define VMEM_ALLOC 0x01 #define VMEM_ALLOC 0x01
#define VMEM_FREE 0x02 #define VMEM_FREE 0x02
#ifndef VMALLOC_TOTAL #ifndef VMALLOC_TOTAL
#define VMALLOC_TOTAL (VMALLOC_END - VMALLOC_START) #define VMALLOC_TOTAL (VMALLOC_END - VMALLOC_START)
#endif #endif
static inline void * static inline void *
@ -78,7 +78,7 @@ vmalloc_nofail(size_t size, gfp_t flags)
} }
} }
return ptr; return (ptr);
} }
static inline void * static inline void *
@ -90,7 +90,7 @@ vzalloc_nofail(size_t size, gfp_t flags)
if (ptr) if (ptr)
memset(ptr, 0, (size)); memset(ptr, 0, (size));
return ptr; return (ptr);
} }
#ifdef DEBUG_KMEM #ifdef DEBUG_KMEM
@ -98,29 +98,29 @@ vzalloc_nofail(size_t size, gfp_t flags)
/* /*
* Memory accounting functions to be used only when DEBUG_KMEM is set. * Memory accounting functions to be used only when DEBUG_KMEM is set.
*/ */
# ifdef HAVE_ATOMIC64_T #ifdef HAVE_ATOMIC64_T
# define vmem_alloc_used_add(size) atomic64_add(size, &vmem_alloc_used) #define vmem_alloc_used_add(size) atomic64_add(size, &vmem_alloc_used)
# define vmem_alloc_used_sub(size) atomic64_sub(size, &vmem_alloc_used) #define vmem_alloc_used_sub(size) atomic64_sub(size, &vmem_alloc_used)
# define vmem_alloc_used_read() atomic64_read(&vmem_alloc_used) #define vmem_alloc_used_read() atomic64_read(&vmem_alloc_used)
# define vmem_alloc_used_set(size) atomic64_set(&vmem_alloc_used, size) #define vmem_alloc_used_set(size) atomic64_set(&vmem_alloc_used, size)
extern atomic64_t vmem_alloc_used; extern atomic64_t vmem_alloc_used;
extern unsigned long long vmem_alloc_max; extern unsigned long long vmem_alloc_max;
# else /* HAVE_ATOMIC64_T */ #else /* HAVE_ATOMIC64_T */
# define vmem_alloc_used_add(size) atomic_add(size, &vmem_alloc_used) #define vmem_alloc_used_add(size) atomic_add(size, &vmem_alloc_used)
# define vmem_alloc_used_sub(size) atomic_sub(size, &vmem_alloc_used) #define vmem_alloc_used_sub(size) atomic_sub(size, &vmem_alloc_used)
# define vmem_alloc_used_read() atomic_read(&vmem_alloc_used) #define vmem_alloc_used_read() atomic_read(&vmem_alloc_used)
# define vmem_alloc_used_set(size) atomic_set(&vmem_alloc_used, size) #define vmem_alloc_used_set(size) atomic_set(&vmem_alloc_used, size)
extern atomic_t vmem_alloc_used; extern atomic_t vmem_alloc_used;
extern unsigned long long vmem_alloc_max; extern unsigned long long vmem_alloc_max;
# endif /* HAVE_ATOMIC64_T */ #endif /* HAVE_ATOMIC64_T */
# ifdef DEBUG_KMEM_TRACKING #ifdef DEBUG_KMEM_TRACKING
/* /*
* DEBUG_KMEM && DEBUG_KMEM_TRACKING * DEBUG_KMEM && DEBUG_KMEM_TRACKING
* *
@ -132,18 +132,18 @@ extern unsigned long long vmem_alloc_max;
* be enabled for debugging. This feature may be enabled by passing * be enabled for debugging. This feature may be enabled by passing
* --enable-debug-kmem-tracking to configure. * --enable-debug-kmem-tracking to configure.
*/ */
# define vmem_alloc(sz, fl) vmem_alloc_track((sz), (fl), \ #define vmem_alloc(sz, fl) vmem_alloc_track((sz), (fl), \
__FUNCTION__, __LINE__) __FUNCTION__, __LINE__)
# define vmem_zalloc(sz, fl) vmem_alloc_track((sz), (fl)|__GFP_ZERO,\ #define vmem_zalloc(sz, fl) vmem_alloc_track((sz), (fl)|__GFP_ZERO,\
__FUNCTION__, __LINE__) __FUNCTION__, __LINE__)
# define vmem_free(ptr, sz) vmem_free_track((ptr), (sz)) #define vmem_free(ptr, sz) vmem_free_track((ptr), (sz))
extern void *kmem_alloc_track(size_t, int, const char *, int, int, int); extern void *kmem_alloc_track(size_t, int, const char *, int, int, int);
extern void kmem_free_track(const void *, size_t); extern void kmem_free_track(const void *, size_t);
extern void *vmem_alloc_track(size_t, int, const char *, int); extern void *vmem_alloc_track(size_t, int, const char *, int);
extern void vmem_free_track(const void *, size_t); extern void vmem_free_track(const void *, size_t);
# else /* DEBUG_KMEM_TRACKING */ #else /* DEBUG_KMEM_TRACKING */
/* /*
* DEBUG_KMEM && !DEBUG_KMEM_TRACKING * DEBUG_KMEM && !DEBUG_KMEM_TRACKING
* *
@ -153,16 +153,16 @@ extern void vmem_free_track(const void *, size_t);
* will be reported on the console. To disable this basic accounting * will be reported on the console. To disable this basic accounting
* pass the --disable-debug-kmem option to configure. * pass the --disable-debug-kmem option to configure.
*/ */
# define vmem_alloc(sz, fl) vmem_alloc_debug((sz), (fl), \ #define vmem_alloc(sz, fl) vmem_alloc_debug((sz), (fl), \
__FUNCTION__, __LINE__) __FUNCTION__, __LINE__)
# define vmem_zalloc(sz, fl) vmem_alloc_debug((sz), (fl)|__GFP_ZERO,\ #define vmem_zalloc(sz, fl) vmem_alloc_debug((sz), (fl)|__GFP_ZERO,\
__FUNCTION__, __LINE__) __FUNCTION__, __LINE__)
# define vmem_free(ptr, sz) vmem_free_debug((ptr), (sz)) #define vmem_free(ptr, sz) vmem_free_debug((ptr), (sz))
extern void *vmem_alloc_debug(size_t, int, const char *, int); extern void *vmem_alloc_debug(size_t, int, const char *, int);
extern void vmem_free_debug(const void *, size_t); extern void vmem_free_debug(const void *, size_t);
# endif /* DEBUG_KMEM_TRACKING */ #endif /* DEBUG_KMEM_TRACKING */
#else /* DEBUG_KMEM */ #else /* DEBUG_KMEM */
/* /*
* !DEBUG_KMEM && !DEBUG_KMEM_TRACKING * !DEBUG_KMEM && !DEBUG_KMEM_TRACKING
@ -171,9 +171,9 @@ extern void vmem_free_debug(const void *, size_t);
* minimal memory accounting. To enable basic accounting pass the * minimal memory accounting. To enable basic accounting pass the
* --enable-debug-kmem option to configure. * --enable-debug-kmem option to configure.
*/ */
# define vmem_alloc(sz, fl) vmalloc_nofail((sz), (fl)) #define vmem_alloc(sz, fl) vmalloc_nofail((sz), (fl))
# define vmem_zalloc(sz, fl) vzalloc_nofail((sz), (fl)) #define vmem_zalloc(sz, fl) vzalloc_nofail((sz), (fl))
# define vmem_free(ptr, sz) ((void)(sz), vfree(ptr)) #define vmem_free(ptr, sz) ((void)(sz), vfree(ptr))
#endif /* DEBUG_KMEM */ #endif /* DEBUG_KMEM */

168
module/spl/spl-kmem-cache.c
View File

@ -1,4 +1,4 @@
/*****************************************************************************\ /*
* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
* Copyright (C) 2007 The Regents of the University of California. * Copyright (C) 2007 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
@ -20,9 +20,7 @@
* *
* You should have received a copy of the GNU General Public License along * You should have received a copy of the GNU General Public License along
* with the SPL. If not, see <http://www.gnu.org/licenses/>. * with the SPL. If not, see <http://www.gnu.org/licenses/>.
***************************************************************************** */
* Solaris Porting Layer (SPL) Kmem Implementation.
\*****************************************************************************/
#include <sys/kmem.h> #include <sys/kmem.h>
#include <sys/kmem_cache.h> #include <sys/kmem_cache.h>
@ -76,7 +74,7 @@ MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab, "Number of objects per slab");
unsigned int spl_kmem_cache_obj_per_slab_min = SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN; unsigned int spl_kmem_cache_obj_per_slab_min = SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN;
module_param(spl_kmem_cache_obj_per_slab_min, uint, 0644); module_param(spl_kmem_cache_obj_per_slab_min, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab_min, MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab_min,
"Minimal number of objects per slab"); "Minimal number of objects per slab");
unsigned int spl_kmem_cache_max_size = 32; unsigned int spl_kmem_cache_max_size = 32;
module_param(spl_kmem_cache_max_size, uint, 0644); module_param(spl_kmem_cache_max_size, uint, 0644);
@ -95,12 +93,12 @@ unsigned int spl_kmem_cache_slab_limit = 0;
#endif #endif
module_param(spl_kmem_cache_slab_limit, uint, 0644); module_param(spl_kmem_cache_slab_limit, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_slab_limit, MODULE_PARM_DESC(spl_kmem_cache_slab_limit,
"Objects less than N bytes use the Linux slab"); "Objects less than N bytes use the Linux slab");
unsigned int spl_kmem_cache_kmem_limit = (PAGE_SIZE / 4); unsigned int spl_kmem_cache_kmem_limit = (PAGE_SIZE / 4);
module_param(spl_kmem_cache_kmem_limit, uint, 0644); module_param(spl_kmem_cache_kmem_limit, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_kmem_limit, MODULE_PARM_DESC(spl_kmem_cache_kmem_limit,
"Objects less than N bytes use the kmalloc"); "Objects less than N bytes use the kmalloc");
/* /*
* Slab allocation interfaces * Slab allocation interfaces
@ -114,7 +112,7 @@ MODULE_PARM_DESC(spl_kmem_cache_kmem_limit,
* breaker for the SPL which contains particularly expensive * breaker for the SPL which contains particularly expensive
* initializers for mutex's, condition variables, etc. We also * initializers for mutex's, condition variables, etc. We also
* require a minimal level of cleanup for these data types unlike * require a minimal level of cleanup for these data types unlike
* many Linux data type which do need to be explicitly destroyed. * many Linux data types which do need to be explicitly destroyed.
* *
* 2) Virtual address space backed slab. Callers of the Solaris slab * 2) Virtual address space backed slab. Callers of the Solaris slab
* expect it to work well for both small are very large allocations. * expect it to work well for both small are very large allocations.
@ -135,7 +133,7 @@ MODULE_PARM_DESC(spl_kmem_cache_kmem_limit,
* *
* XXX: Improve the partial slab list by carefully maintaining a * XXX: Improve the partial slab list by carefully maintaining a
* strict ordering of fullest to emptiest slabs based on * strict ordering of fullest to emptiest slabs based on
* the slab reference count. This guarantees the when freeing * the slab reference count. This guarantees that when freeing
* slabs back to the system we need only linearly traverse the * slabs back to the system we need only linearly traverse the
* last N slabs in the list to discover all the freeable slabs. * last N slabs in the list to discover all the freeable slabs.
* *
@ -149,7 +147,7 @@ MODULE_PARM_DESC(spl_kmem_cache_kmem_limit,
struct list_head spl_kmem_cache_list; /* List of caches */ struct list_head spl_kmem_cache_list; /* List of caches */
struct rw_semaphore spl_kmem_cache_sem; /* Cache list lock */ struct rw_semaphore spl_kmem_cache_sem; /* Cache list lock */
taskq_t *spl_kmem_cache_taskq; /* Task queue for ageing / reclaim */ taskq_t *spl_kmem_cache_taskq; /* Task queue for ageing / reclaim */
static void spl_cache_shrink(spl_kmem_cache_t *skc, void *obj); static void spl_cache_shrink(spl_kmem_cache_t *skc, void *obj);
@ -173,7 +171,7 @@ kv_alloc(spl_kmem_cache_t *skc, int size, int flags)
/* Resulting allocated memory will be page aligned */ /* Resulting allocated memory will be page aligned */
ASSERT(IS_P2ALIGNED(ptr, PAGE_SIZE)); ASSERT(IS_P2ALIGNED(ptr, PAGE_SIZE));
return ptr; return (ptr);
} }
static void static void
@ -204,8 +202,8 @@ kv_free(spl_kmem_cache_t *skc, void *ptr, int size)
static inline uint32_t static inline uint32_t
spl_sks_size(spl_kmem_cache_t *skc) spl_sks_size(spl_kmem_cache_t *skc)
{ {
return P2ROUNDUP_TYPED(sizeof(spl_kmem_slab_t), return (P2ROUNDUP_TYPED(sizeof (spl_kmem_slab_t),
skc->skc_obj_align, uint32_t); skc->skc_obj_align, uint32_t));
} }
/* /*
@ -216,8 +214,8 @@ spl_obj_size(spl_kmem_cache_t *skc)
{ {
uint32_t align = skc->skc_obj_align; uint32_t align = skc->skc_obj_align;
return P2ROUNDUP_TYPED(skc->skc_obj_size, align, uint32_t) + return (P2ROUNDUP_TYPED(skc->skc_obj_size, align, uint32_t) +
P2ROUNDUP_TYPED(sizeof(spl_kmem_obj_t), align, uint32_t); P2ROUNDUP_TYPED(sizeof (spl_kmem_obj_t), align, uint32_t));
} }
/* /*
@ -226,8 +224,8 @@ spl_obj_size(spl_kmem_cache_t *skc)
static inline spl_kmem_obj_t * static inline spl_kmem_obj_t *
spl_sko_from_obj(spl_kmem_cache_t *skc, void *obj) spl_sko_from_obj(spl_kmem_cache_t *skc, void *obj)
{ {
return obj + P2ROUNDUP_TYPED(skc->skc_obj_size, return (obj + P2ROUNDUP_TYPED(skc->skc_obj_size,
skc->skc_obj_align, uint32_t); skc->skc_obj_align, uint32_t));
} }
/* /*
@ -237,7 +235,7 @@ spl_sko_from_obj(spl_kmem_cache_t *skc, void *obj)
static inline uint32_t static inline uint32_t
spl_offslab_size(spl_kmem_cache_t *skc) spl_offslab_size(spl_kmem_cache_t *skc)
{ {
return 1UL << (fls64(spl_obj_size(skc)) + 1); return (1UL << (fls64(spl_obj_size(skc)) + 1));
} }
/* /*
@ -320,8 +318,8 @@ spl_slab_alloc(spl_kmem_cache_t *skc, int flags)
out: out:
if (rc) { if (rc) {
if (skc->skc_flags & KMC_OFFSLAB) if (skc->skc_flags & KMC_OFFSLAB)
list_for_each_entry_safe(sko, n, &sks->sks_free_list, list_for_each_entry_safe(sko,
sko_list) n, &sks->sks_free_list, sko_list)
kv_free(skc, sko->sko_addr, offslab_size); kv_free(skc, sko->sko_addr, offslab_size);
kv_free(skc, base, skc->skc_slab_size); kv_free(skc, base, skc->skc_slab_size);
@ -338,7 +336,7 @@ out:
*/ */
static void static void
spl_slab_free(spl_kmem_slab_t *sks, spl_slab_free(spl_kmem_slab_t *sks,
struct list_head *sks_list, struct list_head *sko_list) struct list_head *sks_list, struct list_head *sko_list)
{ {
spl_kmem_cache_t *skc; spl_kmem_cache_t *skc;
@ -363,7 +361,7 @@ spl_slab_free(spl_kmem_slab_t *sks,
} }
/* /*
* Traverses all the partial slabs attached to a cache and free those * Traverse all the partial slabs attached to a cache and free those
* which which are currently empty, and have not been touched for * which which are currently empty, and have not been touched for
* skc_delay seconds to avoid thrashing. The count argument is * skc_delay seconds to avoid thrashing. The count argument is
* passed to optionally cap the number of slabs reclaimed, a count * passed to optionally cap the number of slabs reclaimed, a count
@ -387,7 +385,8 @@ spl_slab_reclaim(spl_kmem_cache_t *skc, int count, int flag)
* however when flag is set the delay will not be used. * however when flag is set the delay will not be used.
*/ */
spin_lock(&skc->skc_lock); spin_lock(&skc->skc_lock);
list_for_each_entry_safe_reverse(sks,m,&skc->skc_partial_list,sks_list){ list_for_each_entry_safe_reverse(sks, m,
&skc->skc_partial_list, sks_list) {
/* /*
* All empty slabs are at the end of skc->skc_partial_list, * All empty slabs are at the end of skc->skc_partial_list,
* therefore once a non-empty slab is found we can stop * therefore once a non-empty slab is found we can stop
@ -397,7 +396,8 @@ spl_slab_reclaim(spl_kmem_cache_t *skc, int count, int flag)
if ((sks->sks_ref > 0) || (count && i >= count)) if ((sks->sks_ref > 0) || (count && i >= count))
break; break;
if (time_after(jiffies,sks->sks_age+skc->skc_delay*HZ)||flag) { if (time_after(jiffies, sks->sks_age + skc->skc_delay * HZ) ||
flag) {
spl_slab_free(sks, &sks_list, &sko_list); spl_slab_free(sks, &sks_list, &sko_list);
i++; i++;
} }
@ -443,10 +443,10 @@ spl_emergency_search(struct rb_root *root, void *obj)
else if (address > (unsigned long)ske->ske_obj) else if (address > (unsigned long)ske->ske_obj)
node = node->rb_right; node = node->rb_right;
else else
return ske; return (ske);
} }
return NULL; return (NULL);
} }
static int static int
@ -465,13 +465,13 @@ spl_emergency_insert(struct rb_root *root, spl_kmem_emergency_t *ske)
else if (address > (unsigned long)ske_tmp->ske_obj) else if (address > (unsigned long)ske_tmp->ske_obj)
new = &((*new)->rb_right); new = &((*new)->rb_right);
else else
return 0; return (0);
} }
rb_link_node(&ske->ske_node, parent, new); rb_link_node(&ske->ske_node, parent, new);
rb_insert_color(&ske->ske_node, root); rb_insert_color(&ske->ske_node, root);
return 1; return (1);
} }
/* /*
@ -490,7 +490,7 @@ spl_emergency_alloc(spl_kmem_cache_t *skc, int flags, void **obj)
if (!empty) if (!empty)
return (-EEXIST); return (-EEXIST);
ske = kmalloc(sizeof(*ske), flags); ske = kmalloc(sizeof (*ske), flags);
if (ske == NULL) if (ske == NULL)
return (-ENOMEM); return (-ENOMEM);
@ -565,7 +565,7 @@ __spl_cache_flush(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flush)
skm->skm_avail -= count; skm->skm_avail -= count;
memmove(skm->skm_objs, &(skm->skm_objs[count]), memmove(skm->skm_objs, &(skm->skm_objs[count]),
sizeof(void *) * skm->skm_avail); sizeof (void *) * skm->skm_avail);
} }
static void static void
@ -666,7 +666,7 @@ spl_slab_size(spl_kmem_cache_t *skc, uint32_t *objs, uint32_t *size)
if (skc->skc_flags & KMC_OFFSLAB) { if (skc->skc_flags & KMC_OFFSLAB) {
*objs = spl_kmem_cache_obj_per_slab; *objs = spl_kmem_cache_obj_per_slab;
*size = P2ROUNDUP(sizeof(spl_kmem_slab_t), PAGE_SIZE); *size = P2ROUNDUP(sizeof (spl_kmem_slab_t), PAGE_SIZE);
return (0); return (0);
} else { } else {
sks_size = spl_sks_size(skc); sks_size = spl_sks_size(skc);
@ -731,8 +731,8 @@ static spl_kmem_magazine_t *
spl_magazine_alloc(spl_kmem_cache_t *skc, int cpu) spl_magazine_alloc(spl_kmem_cache_t *skc, int cpu)
{ {
spl_kmem_magazine_t *skm; spl_kmem_magazine_t *skm;
int size = sizeof(spl_kmem_magazine_t) + int size = sizeof (spl_kmem_magazine_t) +
sizeof(void *) * skc->skc_mag_size; sizeof (void *) * skc->skc_mag_size;
skm = kmem_alloc_node(size, KM_SLEEP, cpu_to_node(cpu)); skm = kmem_alloc_node(size, KM_SLEEP, cpu_to_node(cpu));
if (skm) { if (skm) {
@ -754,8 +754,8 @@ spl_magazine_alloc(spl_kmem_cache_t *skc, int cpu)
static void static void
spl_magazine_free(spl_kmem_magazine_t *skm) spl_magazine_free(spl_kmem_magazine_t *skm)
{ {
int size = sizeof(spl_kmem_magazine_t) + int size = sizeof (spl_kmem_magazine_t) +
sizeof(void *) * skm->skm_size; sizeof (void *) * skm->skm_size;
ASSERT(skm->skm_magic == SKM_MAGIC); ASSERT(skm->skm_magic == SKM_MAGIC);
ASSERT(skm->skm_avail == 0); ASSERT(skm->skm_avail == 0);
@ -802,11 +802,11 @@ spl_magazine_destroy(spl_kmem_cache_t *skc)
if (skc->skc_flags & KMC_NOMAGAZINE) if (skc->skc_flags & KMC_NOMAGAZINE)
return; return;
for_each_online_cpu(i) { for_each_online_cpu(i) {
skm = skc->skc_mag[i]; skm = skc->skc_mag[i];
spl_cache_flush(skc, skm, skm->skm_avail); spl_cache_flush(skc, skm, skm->skm_avail);
spl_magazine_free(skm); spl_magazine_free(skm);
} }
} }
/* /*
@ -832,12 +832,10 @@ spl_magazine_destroy(spl_kmem_cache_t *skc)
*/ */
spl_kmem_cache_t * spl_kmem_cache_t *
spl_kmem_cache_create(char *name, size_t size, size_t align, spl_kmem_cache_create(char *name, size_t size, size_t align,
spl_kmem_ctor_t ctor, spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, spl_kmem_reclaim_t reclaim,
spl_kmem_dtor_t dtor, void *priv, void *vmp, int flags)
spl_kmem_reclaim_t reclaim,
void *priv, void *vmp, int flags)
{ {
spl_kmem_cache_t *skc; spl_kmem_cache_t *skc;
int rc; int rc;
/* /*
@ -851,13 +849,13 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
might_sleep(); might_sleep();
/* /*
* Allocate memory for a new cache an initialize it. Unfortunately, * Allocate memory for a new cache and initialize it. Unfortunately,
* this usually ends up being a large allocation of ~32k because * this usually ends up being a large allocation of ~32k because
* we need to allocate enough memory for the worst case number of * we need to allocate enough memory for the worst case number of
* cpus in the magazine, skc_mag[NR_CPUS]. Because of this we * cpus in the magazine, skc_mag[NR_CPUS]. Because of this we
* explicitly pass KM_NODEBUG to suppress the kmem warning * explicitly pass KM_NODEBUG to suppress the kmem warning
*/ */
skc = kmem_zalloc(sizeof(*skc), KM_SLEEP| KM_NODEBUG); skc = kmem_zalloc(sizeof (*skc), KM_SLEEP| KM_NODEBUG);
if (skc == NULL) if (skc == NULL)
return (NULL); return (NULL);
@ -865,7 +863,7 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
skc->skc_name_size = strlen(name) + 1; skc->skc_name_size = strlen(name) + 1;
skc->skc_name = (char *)kmem_alloc(skc->skc_name_size, KM_SLEEP); skc->skc_name = (char *)kmem_alloc(skc->skc_name_size, KM_SLEEP);
if (skc->skc_name == NULL) { if (skc->skc_name == NULL) {
kmem_free(skc, sizeof(*skc)); kmem_free(skc, sizeof (*skc));
return (NULL); return (NULL);
} }
strncpy(skc->skc_name, name, skc->skc_name_size); strncpy(skc->skc_name, name, skc->skc_name_size);
@ -923,7 +921,7 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
* Objects smaller than spl_kmem_cache_slab_limit can * Objects smaller than spl_kmem_cache_slab_limit can
* use the Linux slab for better space-efficiency. By * use the Linux slab for better space-efficiency. By
* default this functionality is disabled until its * default this functionality is disabled until its
* performance characters are fully understood. * performance characteristics are fully understood.
*/ */
if (spl_kmem_cache_slab_limit && if (spl_kmem_cache_slab_limit &&
size <= (size_t)spl_kmem_cache_slab_limit) size <= (size_t)spl_kmem_cache_slab_limit)
@ -980,20 +978,20 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
return (skc); return (skc);
out: out:
kmem_free(skc->skc_name, skc->skc_name_size); kmem_free(skc->skc_name, skc->skc_name_size);
kmem_free(skc, sizeof(*skc)); kmem_free(skc, sizeof (*skc));
return (NULL); return (NULL);
} }
EXPORT_SYMBOL(spl_kmem_cache_create); EXPORT_SYMBOL(spl_kmem_cache_create);
/* /*
* Register a move callback to for cache defragmentation. * Register a move callback for cache defragmentation.
* XXX: Unimplemented but harmless to stub out for now. * XXX: Unimplemented but harmless to stub out for now.
*/ */
void void
spl_kmem_cache_set_move(spl_kmem_cache_t *skc, spl_kmem_cache_set_move(spl_kmem_cache_t *skc,
kmem_cbrc_t (move)(void *, void *, size_t, void *)) kmem_cbrc_t (move)(void *, void *, size_t, void *))
{ {
ASSERT(move != NULL); ASSERT(move != NULL);
} }
EXPORT_SYMBOL(spl_kmem_cache_set_move); EXPORT_SYMBOL(spl_kmem_cache_set_move);
@ -1022,9 +1020,11 @@ spl_kmem_cache_destroy(spl_kmem_cache_t *skc)
taskq_cancel_id(spl_kmem_cache_taskq, id); taskq_cancel_id(spl_kmem_cache_taskq, id);
/* Wait until all current callers complete, this is mainly /*
* Wait until all current callers complete, this is mainly
* to catch the case where a low memory situation triggers a * to catch the case where a low memory situation triggers a
* cache reaping action which races with this destroy. */ * cache reaping action which races with this destroy.
*/
wait_event(wq, atomic_read(&skc->skc_ref) == 0); wait_event(wq, atomic_read(&skc->skc_ref) == 0);
if (skc->skc_flags & (KMC_KMEM | KMC_VMEM)) { if (skc->skc_flags & (KMC_KMEM | KMC_VMEM)) {
@ -1037,8 +1037,10 @@ spl_kmem_cache_destroy(spl_kmem_cache_t *skc)
spin_lock(&skc->skc_lock); spin_lock(&skc->skc_lock);
/* Validate there are no objects in use and free all the /*
* spl_kmem_slab_t, spl_kmem_obj_t, and object buffers. */ * Validate there are no objects in use and free all the
* spl_kmem_slab_t, spl_kmem_obj_t, and object buffers.
*/
ASSERT3U(skc->skc_slab_alloc, ==, 0); ASSERT3U(skc->skc_slab_alloc, ==, 0);
ASSERT3U(skc->skc_obj_alloc, ==, 0); ASSERT3U(skc->skc_obj_alloc, ==, 0);
ASSERT3U(skc->skc_slab_total, ==, 0); ASSERT3U(skc->skc_slab_total, ==, 0);
@ -1049,7 +1051,7 @@ spl_kmem_cache_destroy(spl_kmem_cache_t *skc)
kmem_free(skc->skc_name, skc->skc_name_size); kmem_free(skc->skc_name, skc->skc_name_size);
spin_unlock(&skc->skc_lock); spin_unlock(&skc->skc_lock);
kmem_free(skc, sizeof(*skc)); kmem_free(skc, sizeof (*skc));
} }
EXPORT_SYMBOL(spl_kmem_cache_destroy); EXPORT_SYMBOL(spl_kmem_cache_destroy);
@ -1089,7 +1091,7 @@ spl_cache_obj(spl_kmem_cache_t *skc, spl_kmem_slab_t *sks)
skc->skc_slab_max = skc->skc_slab_alloc; skc->skc_slab_max = skc->skc_slab_alloc;
} }
return sko->sko_addr; return (sko->sko_addr);
} }
/* /*
@ -1127,7 +1129,7 @@ spl_cache_grow_work(void *data)
static int static int
spl_cache_grow_wait(spl_kmem_cache_t *skc) spl_cache_grow_wait(spl_kmem_cache_t *skc)
{ {
return !test_bit(KMC_BIT_GROWING, &skc->skc_flags); return (!test_bit(KMC_BIT_GROWING, &skc->skc_flags));
} }
/* /*
@ -1164,7 +1166,7 @@ spl_cache_grow(spl_kmem_cache_t *skc, int flags, void **obj)
if (test_and_set_bit(KMC_BIT_GROWING, &skc->skc_flags) == 0) { if (test_and_set_bit(KMC_BIT_GROWING, &skc->skc_flags) == 0) {
spl_kmem_alloc_t *ska; spl_kmem_alloc_t *ska;
ska = kmalloc(sizeof(*ska), flags); ska = kmalloc(sizeof (*ska), flags);
if (ska == NULL) { if (ska == NULL) {
clear_bit(KMC_BIT_GROWING, &skc->skc_flags); clear_bit(KMC_BIT_GROWING, &skc->skc_flags);
wake_up_all(&skc->skc_waitq); wake_up_all(&skc->skc_waitq);
@ -1192,7 +1194,7 @@ spl_cache_grow(spl_kmem_cache_t *skc, int flags, void **obj)
rc = spl_emergency_alloc(skc, flags, obj); rc = spl_emergency_alloc(skc, flags, obj);
} else { } else {
remaining = wait_event_timeout(skc->skc_waitq, remaining = wait_event_timeout(skc->skc_waitq,
spl_cache_grow_wait(skc), HZ); spl_cache_grow_wait(skc), HZ);
if (!remaining && test_bit(KMC_BIT_VMEM, &skc->skc_flags)) { if (!remaining && test_bit(KMC_BIT_VMEM, &skc->skc_flags)) {
spin_lock(&skc->skc_lock); spin_lock(&skc->skc_lock);
@ -1249,9 +1251,11 @@ spl_cache_refill(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flags)
if (skm != skc->skc_mag[smp_processor_id()]) if (skm != skc->skc_mag[smp_processor_id()])
goto out; goto out;
/* Potentially rescheduled to the same CPU but /*
* Potentially rescheduled to the same CPU but
* allocations may have occurred from this CPU while * allocations may have occurred from this CPU while
* we were sleeping so recalculate max refill. */ * we were sleeping so recalculate max refill.
*/
refill = MIN(refill, skm->skm_size - skm->skm_avail); refill = MIN(refill, skm->skm_size - skm->skm_avail);
spin_lock(&skc->skc_lock); spin_lock(&skc->skc_lock);
@ -1260,17 +1264,21 @@ spl_cache_refill(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flags)
/* Grab the next available slab */ /* Grab the next available slab */
sks = list_entry((&skc->skc_partial_list)->next, sks = list_entry((&skc->skc_partial_list)->next,
spl_kmem_slab_t, sks_list); spl_kmem_slab_t, sks_list);
ASSERT(sks->sks_magic == SKS_MAGIC); ASSERT(sks->sks_magic == SKS_MAGIC);
ASSERT(sks->sks_ref < sks->sks_objs); ASSERT(sks->sks_ref < sks->sks_objs);
ASSERT(!list_empty(&sks->sks_free_list)); ASSERT(!list_empty(&sks->sks_free_list));
/* Consume as many objects as needed to refill the requested /*
* cache. We must also be careful not to overfill it. */ * Consume as many objects as needed to refill the requested
while (sks->sks_ref < sks->sks_objs && refill-- > 0 && ++count) { * cache. We must also be careful not to overfill it.
*/
while (sks->sks_ref < sks->sks_objs && refill-- > 0 &&
++count) {
ASSERT(skm->skm_avail < skm->skm_size); ASSERT(skm->skm_avail < skm->skm_size);
ASSERT(count < skm->skm_size); ASSERT(count < skm->skm_size);
skm->skm_objs[skm->skm_avail++]=spl_cache_obj(skc,sks); skm->skm_objs[skm->skm_avail++] =
spl_cache_obj(skc, sks);
} }
/* Move slab to skc_complete_list when full */ /* Move slab to skc_complete_list when full */
@ -1308,16 +1316,20 @@ spl_cache_shrink(spl_kmem_cache_t *skc, void *obj)
sks->sks_ref--; sks->sks_ref--;
skc->skc_obj_alloc--; skc->skc_obj_alloc--;
/* Move slab to skc_partial_list when no longer full. Slabs /*
* Move slab to skc_partial_list when no longer full. Slabs
* are added to the head to keep the partial list is quasi-full * are added to the head to keep the partial list is quasi-full
* sorted order. Fuller at the head, emptier at the tail. */ * sorted order. Fuller at the head, emptier at the tail.
*/
if (sks->sks_ref == (sks->sks_objs - 1)) { if (sks->sks_ref == (sks->sks_objs - 1)) {
list_del(&sks->sks_list); list_del(&sks->sks_list);
list_add(&sks->sks_list, &skc->skc_partial_list); list_add(&sks->sks_list, &skc->skc_partial_list);
} }
/* Move empty slabs to the end of the partial list so /*
* they can be easily found and freed during reclamation. */ * Move empty slabs to the end of the partial list so
* they can be easily found and freed during reclamation.
*/
if (sks->sks_ref == 0) { if (sks->sks_ref == 0) {
list_del(&sks->sks_list); list_del(&sks->sks_list);
list_add_tail(&sks->sks_list, &skc->skc_partial_list); list_add_tail(&sks->sks_list, &skc->skc_partial_list);
@ -1359,10 +1371,12 @@ spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags)
local_irq_disable(); local_irq_disable();
restart: restart:
/* Safe to update per-cpu structure without lock, but /*
* Safe to update per-cpu structure without lock, but
* in the restart case we must be careful to reacquire * in the restart case we must be careful to reacquire
* the local magazine since this may have changed * the local magazine since this may have changed
* when we need to grow the cache. */ * when we need to grow the cache.
*/
skm = skc->skc_mag[smp_processor_id()]; skm = skc->skc_mag[smp_processor_id()];
ASSERT(skm->skm_magic == SKM_MAGIC); ASSERT(skm->skm_magic == SKM_MAGIC);
@ -1438,10 +1452,12 @@ spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj)
local_irq_save(flags); local_irq_save(flags);
/* Safe to update per-cpu structure without lock, but /*
* Safe to update per-cpu structure without lock, but
* no remote memory allocation tracking is being performed * no remote memory allocation tracking is being performed
* it is entirely possible to allocate an object from one * it is entirely possible to allocate an object from one
* CPU cache and return it to another. */ * CPU cache and return it to another.
*/
skm = skc->skc_mag[smp_processor_id()]; skm = skc->skc_mag[smp_processor_id()];
ASSERT(skm->skm_magic == SKM_MAGIC); ASSERT(skm->skm_magic == SKM_MAGIC);
@ -1495,12 +1511,12 @@ __spl_kmem_cache_generic_shrinker(struct shrinker *shrink,
#ifdef HAVE_SPLIT_SHRINKER_CALLBACK #ifdef HAVE_SPLIT_SHRINKER_CALLBACK
uint64_t oldalloc = skc->skc_obj_alloc; uint64_t oldalloc = skc->skc_obj_alloc;
spl_kmem_cache_reap_now(skc, spl_kmem_cache_reap_now(skc,
MAX(sc->nr_to_scan >> fls64(skc->skc_slab_objs), 1)); MAX(sc->nr_to_scan>>fls64(skc->skc_slab_objs), 1));
if (oldalloc > skc->skc_obj_alloc) if (oldalloc > skc->skc_obj_alloc)
alloc += oldalloc - skc->skc_obj_alloc; alloc += oldalloc - skc->skc_obj_alloc;
#else #else
spl_kmem_cache_reap_now(skc, spl_kmem_cache_reap_now(skc,
MAX(sc->nr_to_scan >> fls64(skc->skc_slab_objs), 1)); MAX(sc->nr_to_scan>>fls64(skc->skc_slab_objs), 1));
alloc += skc->skc_obj_alloc; alloc += skc->skc_obj_alloc;
#endif /* HAVE_SPLIT_SHRINKER_CALLBACK */ #endif /* HAVE_SPLIT_SHRINKER_CALLBACK */
} else { } else {
@ -1581,7 +1597,7 @@ spl_kmem_cache_reap_now(spl_kmem_cache_t *skc, int count)
spin_lock(&skc->skc_lock); spin_lock(&skc->skc_lock);
do_reclaim = do_reclaim =
(skc->skc_slab_total > 0) && (skc->skc_slab_total > 0) &&
((skc->skc_slab_total - skc->skc_slab_alloc) == 0) && ((skc->skc_slab_total-skc->skc_slab_alloc) == 0) &&
(skc->skc_obj_alloc < objects); (skc->skc_obj_alloc < objects);
objects = skc->skc_obj_alloc; objects = skc->skc_obj_alloc;

104
module/spl/spl-kmem.c
View File

@ -1,4 +1,4 @@
/*****************************************************************************\ /*
* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
* Copyright (C) 2007 The Regents of the University of California. * Copyright (C) 2007 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
@ -20,9 +20,7 @@
* *
* You should have received a copy of the GNU General Public License along * You should have received a copy of the GNU General Public License along
* with the SPL. If not, see <http://www.gnu.org/licenses/>. * with the SPL. If not, see <http://www.gnu.org/licenses/>.
***************************************************************************** */
* Solaris Porting Layer (SPL) Kmem Implementation.
\*****************************************************************************/
#include <sys/debug.h> #include <sys/debug.h>
#include <sys/kmem.h> #include <sys/kmem.h>
@ -31,7 +29,7 @@
int int
kmem_debugging(void) kmem_debugging(void)
{ {
return 0; return (0);
} }
EXPORT_SYMBOL(kmem_debugging); EXPORT_SYMBOL(kmem_debugging);
@ -47,7 +45,7 @@ kmem_vasprintf(const char *fmt, va_list ap)
va_end(aq); va_end(aq);
} while (ptr == NULL); } while (ptr == NULL);
return ptr; return (ptr);
} }
EXPORT_SYMBOL(kmem_vasprintf); EXPORT_SYMBOL(kmem_vasprintf);
@ -63,7 +61,7 @@ kmem_asprintf(const char *fmt, ...)
va_end(ap); va_end(ap);
} while (ptr == NULL); } while (ptr == NULL);
return ptr; return (ptr);
} }
EXPORT_SYMBOL(kmem_asprintf); EXPORT_SYMBOL(kmem_asprintf);
@ -78,13 +76,13 @@ __strdup(const char *str, int flags)
if (ptr) if (ptr)
memcpy(ptr, str, n + 1); memcpy(ptr, str, n + 1);
return ptr; return (ptr);
} }
char * char *
strdup(const char *str) strdup(const char *str)
{ {
return __strdup(str, KM_SLEEP); return (__strdup(str, KM_SLEEP));
} }
EXPORT_SYMBOL(strdup); EXPORT_SYMBOL(strdup);
@ -104,18 +102,19 @@ EXPORT_SYMBOL(strfree);
#ifdef DEBUG_KMEM #ifdef DEBUG_KMEM
/* Shim layer memory accounting */ /* Shim layer memory accounting */
# ifdef HAVE_ATOMIC64_T #ifdef HAVE_ATOMIC64_T
atomic64_t kmem_alloc_used = ATOMIC64_INIT(0); atomic64_t kmem_alloc_used = ATOMIC64_INIT(0);
unsigned long long kmem_alloc_max = 0; unsigned long long kmem_alloc_max = 0;
# else /* HAVE_ATOMIC64_T */ #else /* HAVE_ATOMIC64_T */
atomic_t kmem_alloc_used = ATOMIC_INIT(0); atomic_t kmem_alloc_used = ATOMIC_INIT(0);
unsigned long long kmem_alloc_max = 0; unsigned long long kmem_alloc_max = 0;
# endif /* HAVE_ATOMIC64_T */ #endif /* HAVE_ATOMIC64_T */
EXPORT_SYMBOL(kmem_alloc_used); EXPORT_SYMBOL(kmem_alloc_used);
EXPORT_SYMBOL(kmem_alloc_max); EXPORT_SYMBOL(kmem_alloc_max);
/* When DEBUG_KMEM_TRACKING is enabled not only will total bytes be tracked /*
* When DEBUG_KMEM_TRACKING is enabled not only will total bytes be tracked
* but also the location of every alloc and free. When the SPL module is * but also the location of every alloc and free. When the SPL module is
* unloaded a list of all leaked addresses and where they were allocated * unloaded a list of all leaked addresses and where they were allocated
* will be dumped to the console. Enabling this feature has a significant * will be dumped to the console. Enabling this feature has a significant
@ -126,18 +125,18 @@ EXPORT_SYMBOL(kmem_alloc_max);
* debugging enabled for anything other than debugging we need to minimize * debugging enabled for anything other than debugging we need to minimize
* the contention by moving to a lock per xmem_table entry model. * the contention by moving to a lock per xmem_table entry model.
*/ */
# ifdef DEBUG_KMEM_TRACKING #ifdef DEBUG_KMEM_TRACKING
# define KMEM_HASH_BITS 10 #define KMEM_HASH_BITS 10
# define KMEM_TABLE_SIZE (1 << KMEM_HASH_BITS) #define KMEM_TABLE_SIZE (1 << KMEM_HASH_BITS)
typedef struct kmem_debug { typedef struct kmem_debug {
struct hlist_node kd_hlist; /* Hash node linkage */ struct hlist_node kd_hlist; /* Hash node linkage */
struct list_head kd_list; /* List of all allocations */ struct list_head kd_list; /* List of all allocations */
void *kd_addr; /* Allocation pointer */ void *kd_addr; /* Allocation pointer */
size_t kd_size; /* Allocation size */ size_t kd_size; /* Allocation size */
const char *kd_func; /* Allocation function */ const char *kd_func; /* Allocation function */
int kd_line; /* Allocation line */ int kd_line; /* Allocation line */
} kmem_debug_t; } kmem_debug_t;
spinlock_t kmem_lock; spinlock_t kmem_lock;
@ -149,7 +148,8 @@ EXPORT_SYMBOL(kmem_table);
EXPORT_SYMBOL(kmem_list); EXPORT_SYMBOL(kmem_list);
static kmem_debug_t * static kmem_debug_t *
kmem_del_init(spinlock_t *lock, struct hlist_head *table, int bits, const void *addr) kmem_del_init(spinlock_t *lock, struct hlist_head *table,
int bits, const void *addr)
{ {
struct hlist_head *head; struct hlist_head *head;
struct hlist_node *node; struct hlist_node *node;
@ -165,7 +165,7 @@ kmem_del_init(spinlock_t *lock, struct hlist_head *table, int bits, const void *
hlist_del_init(&p->kd_hlist); hlist_del_init(&p->kd_hlist);
list_del_init(&p->kd_list); list_del_init(&p->kd_list);
spin_unlock_irqrestore(lock, flags); spin_unlock_irqrestore(lock, flags);
return p; return (p);
} }
} }
@ -183,12 +183,12 @@ kmem_alloc_track(size_t size, int flags, const char *func, int line,
unsigned long irq_flags; unsigned long irq_flags;
/* Function may be called with KM_NOSLEEP so failure is possible */ /* Function may be called with KM_NOSLEEP so failure is possible */
dptr = (kmem_debug_t *) kmalloc_nofail(sizeof(kmem_debug_t), dptr = (kmem_debug_t *) kmalloc_nofail(sizeof (kmem_debug_t),
flags & ~__GFP_ZERO); flags & ~__GFP_ZERO);
if (unlikely(dptr == NULL)) { if (unlikely(dptr == NULL)) {
printk(KERN_WARNING "debug kmem_alloc(%ld, 0x%x) at %s:%d " printk(KERN_WARNING "debug kmem_alloc(%ld, 0x%x) at %s:%d "
"failed (%lld/%llu)\n", sizeof(kmem_debug_t), flags, "failed (%lld/%llu)\n", sizeof (kmem_debug_t), flags,
func, line, kmem_alloc_used_read(), kmem_alloc_max); func, line, kmem_alloc_used_read(), kmem_alloc_max);
} else { } else {
/* /*
@ -280,7 +280,7 @@ kmem_free_track(const void *ptr, size_t size)
kmem_alloc_used_sub(size); kmem_alloc_used_sub(size);
kfree(dptr->kd_func); kfree(dptr->kd_func);
memset((void *)dptr, 0x5a, sizeof(kmem_debug_t)); memset((void *)dptr, 0x5a, sizeof (kmem_debug_t));
kfree(dptr); kfree(dptr);
memset((void *)ptr, 0x5a, size); memset((void *)ptr, 0x5a, size);
@ -288,7 +288,7 @@ kmem_free_track(const void *ptr, size_t size)
} }
EXPORT_SYMBOL(kmem_free_track); EXPORT_SYMBOL(kmem_free_track);
# else /* DEBUG_KMEM_TRACKING */ #else /* DEBUG_KMEM_TRACKING */
void * void *
kmem_alloc_debug(size_t size, int flags, const char *func, int line, kmem_alloc_debug(size_t size, int flags, const char *func, int line,
@ -342,7 +342,7 @@ kmem_free_debug(const void *ptr, size_t size)
} }
EXPORT_SYMBOL(kmem_free_debug); EXPORT_SYMBOL(kmem_free_debug);
# endif /* DEBUG_KMEM_TRACKING */ #endif /* DEBUG_KMEM_TRACKING */
#endif /* DEBUG_KMEM */ #endif /* DEBUG_KMEM */
#if defined(DEBUG_KMEM) && defined(DEBUG_KMEM_TRACKING) #if defined(DEBUG_KMEM) && defined(DEBUG_KMEM_TRACKING)
@ -355,15 +355,19 @@ spl_sprintf_addr(kmem_debug_t *kd, char *str, int len, int min)
ASSERT(str != NULL && len >= 17); ASSERT(str != NULL && len >= 17);
memset(str, 0, len); memset(str, 0, len);
/* Check for a fully printable string, and while we are at /*
* it place the printable characters in the passed buffer. */ * Check for a fully printable string, and while we are at
* it place the printable characters in the passed buffer.
*/
for (i = 0; i < size; i++) { for (i = 0; i < size; i++) {
str[i] = ((char *)(kd->kd_addr))[i]; str[i] = ((char *)(kd->kd_addr))[i];
if (isprint(str[i])) { if (isprint(str[i])) {
continue; continue;
} else { } else {
/* Minimum number of printable characters found /*
* to make it worthwhile to print this as ascii. */ * Minimum number of printable characters found
* to make it worthwhile to print this as ascii.
*/
if (i > min) if (i > min)
break; break;
@ -374,17 +378,17 @@ spl_sprintf_addr(kmem_debug_t *kd, char *str, int len, int min)
if (!flag) { if (!flag) {
sprintf(str, "%02x%02x%02x%02x%02x%02x%02x%02x", sprintf(str, "%02x%02x%02x%02x%02x%02x%02x%02x",
*((uint8_t *)kd->kd_addr), *((uint8_t *)kd->kd_addr),
*((uint8_t *)kd->kd_addr + 2), *((uint8_t *)kd->kd_addr + 2),
*((uint8_t *)kd->kd_addr + 4), *((uint8_t *)kd->kd_addr + 4),
*((uint8_t *)kd->kd_addr + 6), *((uint8_t *)kd->kd_addr + 6),
*((uint8_t *)kd->kd_addr + 8), *((uint8_t *)kd->kd_addr + 8),
*((uint8_t *)kd->kd_addr + 10), *((uint8_t *)kd->kd_addr + 10),
*((uint8_t *)kd->kd_addr + 12), *((uint8_t *)kd->kd_addr + 12),
*((uint8_t *)kd->kd_addr + 14)); *((uint8_t *)kd->kd_addr + 14));
} }
return str; return (str);
} }
static int static int
@ -411,18 +415,18 @@ spl_kmem_fini_tracking(struct list_head *list, spinlock_t *lock)
spin_lock_irqsave(lock, flags); spin_lock_irqsave(lock, flags);
if (!list_empty(list)) if (!list_empty(list))
printk(KERN_WARNING "%-16s %-5s %-16s %s:%s\n", "address", printk(KERN_WARNING "%-16s %-5s %-16s %s:%s\n", "address",
"size", "data", "func", "line"); "size", "data", "func", "line");
list_for_each_entry(kd, list, kd_list) list_for_each_entry(kd, list, kd_list)
printk(KERN_WARNING "%p %-5d %-16s %s:%d\n", kd->kd_addr, printk(KERN_WARNING "%p %-5d %-16s %s:%d\n", kd->kd_addr,
(int)kd->kd_size, spl_sprintf_addr(kd, str, 17, 8), (int)kd->kd_size, spl_sprintf_addr(kd, str, 17, 8),
kd->kd_func, kd->kd_line); kd->kd_func, kd->kd_line);
spin_unlock_irqrestore(lock, flags); spin_unlock_irqrestore(lock, flags);
} }
#else /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */ #else /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */
#define spl_kmem_init_tracking(list, lock, size) #define spl_kmem_init_tracking(list, lock, size)
#define spl_kmem_fini_tracking(list, lock) #define spl_kmem_fini_tracking(list, lock)
#endif /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */ #endif /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */
int int
@ -442,10 +446,12 @@ void
spl_kmem_fini(void) spl_kmem_fini(void)
{ {
#ifdef DEBUG_KMEM #ifdef DEBUG_KMEM
/* Display all unreclaimed memory addresses, including the /*
* Display all unreclaimed memory addresses, including the
* allocation size and the first few bytes of what's located * allocation size and the first few bytes of what's located
* at that address to aid in debugging. Performance is not * at that address to aid in debugging. Performance is not
* a serious concern here since it is module unload time. */ * a serious concern here since it is module unload time.
*/
if (kmem_alloc_used_read() != 0) if (kmem_alloc_used_read() != 0)
printk(KERN_WARNING "kmem leaked %ld/%llu bytes\n", printk(KERN_WARNING "kmem leaked %ld/%llu bytes\n",
kmem_alloc_used_read(), kmem_alloc_max); kmem_alloc_used_read(), kmem_alloc_max);

89
module/spl/spl-vmem.c
View File

@ -1,4 +1,4 @@
/*****************************************************************************\ /*
* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
* Copyright (C) 2007 The Regents of the University of California. * Copyright (C) 2007 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
@ -20,9 +20,7 @@
* *
* You should have received a copy of the GNU General Public License along * You should have received a copy of the GNU General Public License along
* with the SPL. If not, see <http://www.gnu.org/licenses/>. * with the SPL. If not, see <http://www.gnu.org/licenses/>.
***************************************************************************** */
* Solaris Porting Layer (SPL) Kmem Implementation.
\*****************************************************************************/
#include <sys/debug.h> #include <sys/debug.h>
#include <sys/vmem.h> #include <sys/vmem.h>
@ -57,18 +55,19 @@ EXPORT_SYMBOL(vmem_size);
#ifdef DEBUG_KMEM #ifdef DEBUG_KMEM
/* Shim layer memory accounting */ /* Shim layer memory accounting */
# ifdef HAVE_ATOMIC64_T #ifdef HAVE_ATOMIC64_T
atomic64_t vmem_alloc_used = ATOMIC64_INIT(0); atomic64_t vmem_alloc_used = ATOMIC64_INIT(0);
unsigned long long vmem_alloc_max = 0; unsigned long long vmem_alloc_max = 0;
# else /* HAVE_ATOMIC64_T */ #else /* HAVE_ATOMIC64_T */
atomic_t vmem_alloc_used = ATOMIC_INIT(0); atomic_t vmem_alloc_used = ATOMIC_INIT(0);
unsigned long long vmem_alloc_max = 0; unsigned long long vmem_alloc_max = 0;
# endif /* HAVE_ATOMIC64_T */ #endif /* HAVE_ATOMIC64_T */
EXPORT_SYMBOL(vmem_alloc_used); EXPORT_SYMBOL(vmem_alloc_used);
EXPORT_SYMBOL(vmem_alloc_max); EXPORT_SYMBOL(vmem_alloc_max);
/* When DEBUG_KMEM_TRACKING is enabled not only will total bytes be tracked /*
* When DEBUG_KMEM_TRACKING is enabled not only will total bytes be tracked
* but also the location of every alloc and free. When the SPL module is * but also the location of every alloc and free. When the SPL module is
* unloaded a list of all leaked addresses and where they were allocated * unloaded a list of all leaked addresses and where they were allocated
* will be dumped to the console. Enabling this feature has a significant * will be dumped to the console. Enabling this feature has a significant
@ -79,18 +78,18 @@ EXPORT_SYMBOL(vmem_alloc_max);
* debugging enabled for anything other than debugging we need to minimize * debugging enabled for anything other than debugging we need to minimize
* the contention by moving to a lock per xmem_table entry model. * the contention by moving to a lock per xmem_table entry model.
*/ */
# ifdef DEBUG_KMEM_TRACKING #ifdef DEBUG_KMEM_TRACKING
# define VMEM_HASH_BITS 10 #define VMEM_HASH_BITS 10
# define VMEM_TABLE_SIZE (1 << VMEM_HASH_BITS) #define VMEM_TABLE_SIZE (1 << VMEM_HASH_BITS)
typedef struct kmem_debug { typedef struct kmem_debug {
struct hlist_node kd_hlist; /* Hash node linkage */ struct hlist_node kd_hlist; /* Hash node linkage */
struct list_head kd_list; /* List of all allocations */ struct list_head kd_list; /* List of all allocations */
void *kd_addr; /* Allocation pointer */ void *kd_addr; /* Allocation pointer */
size_t kd_size; /* Allocation size */ size_t kd_size; /* Allocation size */
const char *kd_func; /* Allocation function */ const char *kd_func; /* Allocation function */
int kd_line; /* Allocation line */ int kd_line; /* Allocation line */
} kmem_debug_t; } kmem_debug_t;
spinlock_t vmem_lock; spinlock_t vmem_lock;
@ -111,12 +110,12 @@ vmem_alloc_track(size_t size, int flags, const char *func, int line)
ASSERT(flags & KM_SLEEP); ASSERT(flags & KM_SLEEP);
/* Function may be called with KM_NOSLEEP so failure is possible */ /* Function may be called with KM_NOSLEEP so failure is possible */
dptr = (kmem_debug_t *) kmalloc_nofail(sizeof(kmem_debug_t), dptr = (kmem_debug_t *) kmalloc_nofail(sizeof (kmem_debug_t),
flags & ~__GFP_ZERO); flags & ~__GFP_ZERO);
if (unlikely(dptr == NULL)) { if (unlikely(dptr == NULL)) {
printk(KERN_WARNING "debug vmem_alloc(%ld, 0x%x) " printk(KERN_WARNING "debug vmem_alloc(%ld, 0x%x) "
"at %s:%d failed (%lld/%llu)\n", "at %s:%d failed (%lld/%llu)\n",
sizeof(kmem_debug_t), flags, func, line, sizeof (kmem_debug_t), flags, func, line,
vmem_alloc_used_read(), vmem_alloc_max); vmem_alloc_used_read(), vmem_alloc_max);
} else { } else {
/* /*
@ -194,7 +193,7 @@ vmem_free_track(const void *ptr, size_t size)
vmem_alloc_used_sub(size); vmem_alloc_used_sub(size);
kfree(dptr->kd_func); kfree(dptr->kd_func);
memset((void *)dptr, 0x5a, sizeof(kmem_debug_t)); memset((void *)dptr, 0x5a, sizeof (kmem_debug_t));
kfree(dptr); kfree(dptr);
memset((void *)ptr, 0x5a, size); memset((void *)ptr, 0x5a, size);
@ -202,7 +201,7 @@ vmem_free_track(const void *ptr, size_t size)
} }
EXPORT_SYMBOL(vmem_free_track); EXPORT_SYMBOL(vmem_free_track);
# else /* DEBUG_KMEM_TRACKING */ #else /* DEBUG_KMEM_TRACKING */
void * void *
vmem_alloc_debug(size_t size, int flags, const char *func, int line) vmem_alloc_debug(size_t size, int flags, const char *func, int line)
@ -242,7 +241,7 @@ vmem_free_debug(const void *ptr, size_t size)
} }
EXPORT_SYMBOL(vmem_free_debug); EXPORT_SYMBOL(vmem_free_debug);
# endif /* DEBUG_KMEM_TRACKING */ #endif /* DEBUG_KMEM_TRACKING */
#endif /* DEBUG_KMEM */ #endif /* DEBUG_KMEM */
#if defined(DEBUG_KMEM) && defined(DEBUG_KMEM_TRACKING) #if defined(DEBUG_KMEM) && defined(DEBUG_KMEM_TRACKING)
@ -255,15 +254,19 @@ spl_sprintf_addr(kmem_debug_t *kd, char *str, int len, int min)
ASSERT(str != NULL && len >= 17); ASSERT(str != NULL && len >= 17);
memset(str, 0, len); memset(str, 0, len);
/* Check for a fully printable string, and while we are at /*
* it place the printable characters in the passed buffer. */ * Check for a fully printable string, and while we are at
* it place the printable characters in the passed buffer.
*/
for (i = 0; i < size; i++) { for (i = 0; i < size; i++) {
str[i] = ((char *)(kd->kd_addr))[i]; str[i] = ((char *)(kd->kd_addr))[i];
if (isprint(str[i])) { if (isprint(str[i])) {
continue; continue;
} else { } else {
/* Minimum number of printable characters found /*
* to make it worthwhile to print this as ascii. */ * Minimum number of printable characters found
* to make it worthwhile to print this as ascii.
*/
if (i > min) if (i > min)
break; break;
@ -274,17 +277,17 @@ spl_sprintf_addr(kmem_debug_t *kd, char *str, int len, int min)
if (!flag) { if (!flag) {
sprintf(str, "%02x%02x%02x%02x%02x%02x%02x%02x", sprintf(str, "%02x%02x%02x%02x%02x%02x%02x%02x",
*((uint8_t *)kd->kd_addr), *((uint8_t *)kd->kd_addr),
*((uint8_t *)kd->kd_addr + 2), *((uint8_t *)kd->kd_addr + 2),
*((uint8_t *)kd->kd_addr + 4), *((uint8_t *)kd->kd_addr + 4),
*((uint8_t *)kd->kd_addr + 6), *((uint8_t *)kd->kd_addr + 6),
*((uint8_t *)kd->kd_addr + 8), *((uint8_t *)kd->kd_addr + 8),
*((uint8_t *)kd->kd_addr + 10), *((uint8_t *)kd->kd_addr + 10),
*((uint8_t *)kd->kd_addr + 12), *((uint8_t *)kd->kd_addr + 12),
*((uint8_t *)kd->kd_addr + 14)); *((uint8_t *)kd->kd_addr + 14));
} }
return str; return (str);
} }
static int static int
@ -311,18 +314,18 @@ spl_kmem_fini_tracking(struct list_head *list, spinlock_t *lock)
spin_lock_irqsave(lock, flags); spin_lock_irqsave(lock, flags);
if (!list_empty(list)) if (!list_empty(list))
printk(KERN_WARNING "%-16s %-5s %-16s %s:%s\n", "address", printk(KERN_WARNING "%-16s %-5s %-16s %s:%s\n", "address",
"size", "data", "func", "line"); "size", "data", "func", "line");
list_for_each_entry(kd, list, kd_list) list_for_each_entry(kd, list, kd_list)
printk(KERN_WARNING "%p %-5d %-16s %s:%d\n", kd->kd_addr, printk(KERN_WARNING "%p %-5d %-16s %s:%d\n", kd->kd_addr,
(int)kd->kd_size, spl_sprintf_addr(kd, str, 17, 8), (int)kd->kd_size, spl_sprintf_addr(kd, str, 17, 8),
kd->kd_func, kd->kd_line); kd->kd_func, kd->kd_line);
spin_unlock_irqrestore(lock, flags); spin_unlock_irqrestore(lock, flags);
} }
#else /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */ #else /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */
#define spl_kmem_init_tracking(list, lock, size) #define spl_kmem_init_tracking(list, lock, size)
#define spl_kmem_fini_tracking(list, lock) #define spl_kmem_fini_tracking(list, lock)
#endif /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */ #endif /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */
int int
@ -342,10 +345,12 @@ void
spl_vmem_fini(void) spl_vmem_fini(void)
{ {
#ifdef DEBUG_KMEM #ifdef DEBUG_KMEM
/* Display all unreclaimed memory addresses, including the /*
* Display all unreclaimed memory addresses, including the
* allocation size and the first few bytes of what's located * allocation size and the first few bytes of what's located
* at that address to aid in debugging. Performance is not * at that address to aid in debugging. Performance is not
* a serious concern here since it is module unload time. */ * a serious concern here since it is module unload time.
*/
if (vmem_alloc_used_read() != 0) if (vmem_alloc_used_read() != 0)
printk(KERN_WARNING "vmem leaked %ld/%llu bytes\n", printk(KERN_WARNING "vmem leaked %ld/%llu bytes\n",
vmem_alloc_used_read(), vmem_alloc_max); vmem_alloc_used_read(), vmem_alloc_max);