/* * This file is part of the SPL: Solaris Porting Layer. * * Copyright (c) 2008 Lawrence Livermore National Security, LLC. * Produced at Lawrence Livermore National Laboratory * Written by: * Brian Behlendorf , * Herb Wartens , * Jim Garlick * UCRL-CODE-235197 * * This is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * for more details. * * You should have received a copy of the GNU General Public License along * with this program; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include #ifdef DEBUG_SUBSYSTEM #undef DEBUG_SUBSYSTEM #endif #define DEBUG_SUBSYSTEM S_KMEM /* * Memory allocation interfaces and debugging for basic kmem_* * and vmem_* style memory allocation. When DEBUG_KMEM is enable * all allocations will be tracked when they are allocated and * freed. When the SPL module is unload a list of all leaked * addresses and where they were allocated will be dumped to the * console. Enabling this feature has a significant impant on * performance but it makes finding memory leaks staight forward. */ #ifdef DEBUG_KMEM /* Shim layer memory accounting */ atomic64_t kmem_alloc_used; unsigned long kmem_alloc_max = 0; atomic64_t vmem_alloc_used; unsigned long vmem_alloc_max = 0; int kmem_warning_flag = 1; atomic64_t kmem_cache_alloc_failed; spinlock_t kmem_lock; struct hlist_head kmem_table[KMEM_TABLE_SIZE]; struct list_head kmem_list; spinlock_t vmem_lock; struct hlist_head vmem_table[VMEM_TABLE_SIZE]; struct list_head vmem_list; EXPORT_SYMBOL(kmem_alloc_used); EXPORT_SYMBOL(kmem_alloc_max); EXPORT_SYMBOL(vmem_alloc_used); EXPORT_SYMBOL(vmem_alloc_max); EXPORT_SYMBOL(kmem_warning_flag); EXPORT_SYMBOL(kmem_lock); EXPORT_SYMBOL(kmem_table); EXPORT_SYMBOL(kmem_list); EXPORT_SYMBOL(vmem_lock); EXPORT_SYMBOL(vmem_table); EXPORT_SYMBOL(vmem_list); int kmem_set_warning(int flag) { return (kmem_warning_flag = !!flag); } #else int kmem_set_warning(int flag) { return 0; } #endif EXPORT_SYMBOL(kmem_set_warning); /* * Slab allocation interfaces * * While the Linux slab implementation was inspired by the Solaris * implemenation I cannot use it to emulate the Solaris APIs. I * require two features which are not provided by the Linux slab. * * 1) Constructors AND destructors. Recent versions of the Linux * kernel have removed support for destructors. This is a deal * breaker for the SPL which contains particularly expensive * initializers for mutex's, condition variables, etc. We also * require a minimal level of cleaner for these data types unlike * may Linux data type which do need to be explicitly destroyed. * * 2) Virtual address backed slab. Callers of the Solaris slab * expect it to work well for both small are very large allocations. * Because of memory fragmentation the Linux slab which is backed * by kmalloc'ed memory performs very badly when confronted with * large numbers of large allocations. Basing the slab on the * virtual address space removes the need for contigeous pages * and greatly improve performance for large allocations. * * For these reasons, the SPL has its own slab implementation with * the needed features. It is not as highly optimized as either the * Solaris or Linux slabs, but it should get me most of what is * needed until it can be optimized or obsoleted by another approach. * * One serious concern I do have about this method is the relatively * small virtual address space on 32bit arches. This will seriously * constrain the size of the slab caches and their performance. * * XXX: Refactor the below code in to smaller functions. This works * for a first pass but each function is doing to much. * * XXX: Implement SPL proc interface to export full per cache stats. * * XXX: Implement work requests to keep an eye on each cache and * shrink them via slab_reclaim() when they are wasting lots * of space. Currently this process is driven by the reapers. * * XXX: Implement proper small cache object support by embedding * the spl_kmem_slab_t, spl_kmem_obj_t's, and objects in the * allocated for a particular slab. * * XXX: Implement a resizable used object hash. Currently the hash * is statically sized for thousands of objects but it should * grow based on observed worst case slab depth. * * XXX: Improve the partial slab list by carefully maintaining a * strict ordering of fullest to emptiest slabs based on * the slab reference count. This gaurentees the when freeing * slabs back to the system we need only linearly traverse the * last N slabs in the list to discover all the freeable slabs. * * XXX: NUMA awareness for optionally allocating memory close to a * particular core. This can be adventageous if you know the slab * object will be short lived and primarily accessed from one core. * * XXX: Slab coloring may also yield performance improvements and would * be desirable to implement. */ /* Ensure the __kmem_cache_create/__kmem_cache_destroy macros are * removed here to prevent a recursive substitution, we want to call * the native linux version. */ #undef kmem_cache_t #undef kmem_cache_create #undef kmem_cache_destroy #undef kmem_cache_alloc #undef kmem_cache_free static struct list_head spl_kmem_cache_list; /* List of caches */ static struct rw_semaphore spl_kmem_cache_sem; /* Cache list lock */ static kmem_cache_t *spl_slab_cache; /* Cache for slab structs */ static kmem_cache_t *spl_obj_cache; /* Cache for obj structs */ #ifdef HAVE_SET_SHRINKER static struct shrinker *spl_kmem_cache_shrinker; #else static int kmem_cache_generic_shrinker(int nr_to_scan, unsigned int gfp_mask); static struct shrinker spl_kmem_cache_shrinker = { .shrink = kmem_cache_generic_shrinker, .seeks = KMC_DEFAULT_SEEKS, }; #endif static spl_kmem_slab_t * slab_alloc(spl_kmem_cache_t *skc, int flags) { spl_kmem_slab_t *sks; spl_kmem_obj_t *sko, *n; int i; ENTRY; sks = kmem_cache_alloc(spl_slab_cache, flags); if (sks == NULL) RETURN(sks); sks->sks_magic = SKS_MAGIC; sks->sks_objs = SPL_KMEM_CACHE_OBJ_PER_SLAB; sks->sks_age = jiffies; sks->sks_cache = skc; INIT_LIST_HEAD(&sks->sks_list); INIT_LIST_HEAD(&sks->sks_free_list); atomic_set(&sks->sks_ref, 0); for (i = 0; i < sks->sks_objs; i++) { sko = kmem_cache_alloc(spl_obj_cache, flags); if (sko == NULL) { out_alloc: /* Unable to fully construct slab, objects, * and object data buffers unwind everything. */ list_for_each_entry_safe(sko, n, &sks->sks_free_list, sko_list) { ASSERT(sko->sko_magic == SKO_MAGIC); vmem_free(sko->sko_addr, skc->skc_obj_size); list_del(&sko->sko_list); kmem_cache_free(spl_obj_cache, sko); } kmem_cache_free(spl_slab_cache, sks); GOTO(out, sks = NULL); } sko->sko_addr = vmem_alloc(skc->skc_obj_size, flags); if (sko->sko_addr == NULL) { kmem_cache_free(spl_obj_cache, sko); GOTO(out_alloc, sks = NULL); } sko->sko_magic = SKO_MAGIC; sko->sko_flags = 0; sko->sko_slab = sks; INIT_LIST_HEAD(&sko->sko_list); INIT_HLIST_NODE(&sko->sko_hlist); list_add(&sko->sko_list, &sks->sks_free_list); } out: RETURN(sks); } /* Removes slab from complete or partial list, so it must * be called with the 'skc->skc_lock' held. * */ static void slab_free(spl_kmem_slab_t *sks) { spl_kmem_cache_t *skc; spl_kmem_obj_t *sko, *n; int i = 0; ENTRY; ASSERT(sks->sks_magic == SKS_MAGIC); ASSERT(atomic_read(&sks->sks_ref) == 0); skc = sks->sks_cache; skc->skc_obj_total -= sks->sks_objs; skc->skc_slab_total--; //#ifdef CONFIG_RWSEM_GENERIC_SPINLOCK ASSERT(spin_is_locked(&skc->skc_lock)); //#endif list_for_each_entry_safe(sko, n, &sks->sks_free_list, sko_list) { ASSERT(sko->sko_magic == SKO_MAGIC); /* Run destructors for being freed */ if (skc->skc_dtor) skc->skc_dtor(sko->sko_addr, skc->skc_private); vmem_free(sko->sko_addr, skc->skc_obj_size); list_del(&sko->sko_list); kmem_cache_free(spl_obj_cache, sko); i++; } ASSERT(sks->sks_objs == i); list_del(&sks->sks_list); kmem_cache_free(spl_slab_cache, sks); EXIT; } static int __slab_reclaim(spl_kmem_cache_t *skc) { spl_kmem_slab_t *sks, *m; int rc = 0; ENTRY; //#ifdef CONFIG_RWSEM_GENERIC_SPINLOCK ASSERT(spin_is_locked(&skc->skc_lock)); //#endif /* * Free empty slabs which have not been touched in skc_delay * seconds. This delay time is important to avoid thrashing. * Empty slabs will be at the end of the skc_partial_list. */ list_for_each_entry_safe_reverse(sks, m, &skc->skc_partial_list, sks_list) { if (atomic_read(&sks->sks_ref) > 0) break; if (time_after(jiffies, sks->sks_age + skc->skc_delay * HZ)) { slab_free(sks); rc++; } } /* Returns number of slabs reclaimed */ RETURN(rc); } static int slab_reclaim(spl_kmem_cache_t *skc) { int rc; ENTRY; spin_lock(&skc->skc_lock); rc = __slab_reclaim(skc); spin_unlock(&skc->skc_lock); RETURN(rc); } spl_kmem_cache_t * spl_kmem_cache_create(char *name, size_t size, size_t align, spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, spl_kmem_reclaim_t reclaim, void *priv, void *vmp, int flags) { spl_kmem_cache_t *skc; int i, kmem_flags = KM_SLEEP; ENTRY; /* We may be called when there is a non-zero preempt_count or * interrupts are disabled is which case we must not sleep. */ if (current_thread_info()->preempt_count || irqs_disabled()) kmem_flags = KM_NOSLEEP; /* Allocate new cache memory and initialize. */ skc = (spl_kmem_cache_t *)kmem_alloc(sizeof(*skc), kmem_flags); if (skc == NULL) RETURN(NULL); skc->skc_magic = SKC_MAGIC; skc->skc_name_size = strlen(name) + 1; skc->skc_name = (char *)kmem_alloc(skc->skc_name_size, kmem_flags); if (skc->skc_name == NULL) { kmem_free(skc, sizeof(*skc)); RETURN(NULL); } strncpy(skc->skc_name, name, skc->skc_name_size); skc->skc_ctor = ctor; skc->skc_dtor = dtor; skc->skc_reclaim = reclaim; skc->skc_private = priv; skc->skc_vmp = vmp; skc->skc_flags = flags; skc->skc_obj_size = size; skc->skc_chunk_size = 0; /* XXX: Needed only when implementing */ skc->skc_slab_size = 0; /* small slab object optimizations */ skc->skc_max_chunks = 0; /* which are yet supported. */ skc->skc_delay = SPL_KMEM_CACHE_DELAY; skc->skc_hash_bits = SPL_KMEM_CACHE_HASH_BITS; skc->skc_hash_size = SPL_KMEM_CACHE_HASH_SIZE; skc->skc_hash_elts = SPL_KMEM_CACHE_HASH_ELTS; skc->skc_hash = (struct hlist_head *) kmem_alloc(skc->skc_hash_size, kmem_flags); if (skc->skc_hash == NULL) { kmem_free(skc->skc_name, skc->skc_name_size); kmem_free(skc, sizeof(*skc)); } for (i = 0; i < skc->skc_hash_elts; i++) INIT_HLIST_HEAD(&skc->skc_hash[i]); INIT_LIST_HEAD(&skc->skc_list); INIT_LIST_HEAD(&skc->skc_complete_list); INIT_LIST_HEAD(&skc->skc_partial_list); spin_lock_init(&skc->skc_lock); skc->skc_slab_fail = 0; skc->skc_slab_create = 0; skc->skc_slab_destroy = 0; skc->skc_slab_total = 0; skc->skc_slab_alloc = 0; skc->skc_slab_max = 0; skc->skc_obj_total = 0; skc->skc_obj_alloc = 0; skc->skc_obj_max = 0; skc->skc_hash_depth = 0; skc->skc_hash_max = 0; down_write(&spl_kmem_cache_sem); list_add_tail(&skc->skc_list, &spl_kmem_cache_list); up_write(&spl_kmem_cache_sem); RETURN(skc); } EXPORT_SYMBOL(spl_kmem_cache_create); /* The caller must ensure there are no racing calls to * spl_kmem_cache_alloc() for this spl_kmem_cache_t when * it is being destroyed. */ void spl_kmem_cache_destroy(spl_kmem_cache_t *skc) { spl_kmem_slab_t *sks, *m; ENTRY; down_write(&spl_kmem_cache_sem); list_del_init(&skc->skc_list); up_write(&spl_kmem_cache_sem); spin_lock(&skc->skc_lock); /* Validate there are no objects in use and free all the * spl_kmem_slab_t, spl_kmem_obj_t, and object buffers. */ ASSERT(list_empty(&skc->skc_complete_list)); list_for_each_entry_safe(sks, m, &skc->skc_partial_list, sks_list) slab_free(sks); kmem_free(skc->skc_hash, skc->skc_hash_size); kmem_free(skc->skc_name, skc->skc_name_size); kmem_free(skc, sizeof(*skc)); spin_unlock(&skc->skc_lock); EXIT; } EXPORT_SYMBOL(spl_kmem_cache_destroy); /* The kernel provided hash_ptr() function behaves exceptionally badly * when all the addresses are page aligned which is likely the case * here. To avoid this issue shift off the low order non-random bits. */ static unsigned long spl_hash_ptr(void *ptr, unsigned int bits) { return hash_long((unsigned long)ptr >> PAGE_SHIFT, bits); } #ifndef list_first_entry #define list_first_entry(ptr, type, member) \ list_entry((ptr)->next, type, member) #endif void * spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags) { spl_kmem_slab_t *sks; spl_kmem_obj_t *sko; void *obj; unsigned long key; ENTRY; spin_lock(&skc->skc_lock); restart: /* Check for available objects from the partial slabs */ if (!list_empty(&skc->skc_partial_list)) { sks = list_first_entry(&skc->skc_partial_list, spl_kmem_slab_t, sks_list); ASSERT(sks->sks_magic == SKS_MAGIC); ASSERT(atomic_read(&sks->sks_ref) < sks->sks_objs); ASSERT(!list_empty(&sks->sks_free_list)); sko = list_first_entry(&sks->sks_free_list, spl_kmem_obj_t, sko_list); ASSERT(sko->sko_magic == SKO_MAGIC); ASSERT(sko->sko_addr != NULL); /* Remove from sks_free_list, add to used hash */ list_del_init(&sko->sko_list); key = spl_hash_ptr(sko->sko_addr, skc->skc_hash_bits); hlist_add_head(&sko->sko_hlist, &skc->skc_hash[key]); sks->sks_age = jiffies; atomic_inc(&sks->sks_ref); skc->skc_obj_alloc++; if (skc->skc_obj_alloc > skc->skc_obj_max) skc->skc_obj_max = skc->skc_obj_alloc; if (atomic_read(&sks->sks_ref) == 1) { skc->skc_slab_alloc++; if (skc->skc_slab_alloc > skc->skc_slab_max) skc->skc_slab_max = skc->skc_slab_alloc; } /* Move slab to skc_complete_list when full */ if (atomic_read(&sks->sks_ref) == sks->sks_objs) { list_del(&sks->sks_list); list_add(&sks->sks_list, &skc->skc_complete_list); } GOTO(out_lock, obj = sko->sko_addr); } spin_unlock(&skc->skc_lock); /* No available objects create a new slab. Since this is an * expensive operation we do it without holding the semaphore * and only briefly aquire it when we link in the fully * allocated and constructed slab. */ /* Under Solaris if the KM_SLEEP flag is passed we may never * fail, so sleep as long as needed. Additionally, since we are * using vmem_alloc() KM_NOSLEEP is not an option and we must * fail. Shifting to allocating our own pages and mapping the * virtual address space may allow us to bypass this issue. */ if (!flags) flags |= KM_SLEEP; if (flags & KM_SLEEP) flags |= __GFP_NOFAIL; else GOTO(out, obj = NULL); sks = slab_alloc(skc, flags); if (sks == NULL) GOTO(out, obj = NULL); /* Run all the constructors now that the slab is fully allocated */ list_for_each_entry(sko, &sks->sks_free_list, sko_list) { ASSERT(sko->sko_magic == SKO_MAGIC); if (skc->skc_ctor) skc->skc_ctor(sko->sko_addr, skc->skc_private, flags); } /* Link the newly created slab in to the skc_partial_list, * and retry the allocation which will now succeed. */ spin_lock(&skc->skc_lock); skc->skc_slab_total++; skc->skc_obj_total += sks->sks_objs; list_add_tail(&sks->sks_list, &skc->skc_partial_list); GOTO(restart, obj = NULL); out_lock: spin_unlock(&skc->skc_lock); out: RETURN(obj); } EXPORT_SYMBOL(spl_kmem_cache_alloc); void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj) { struct hlist_node *node; spl_kmem_slab_t *sks = NULL; spl_kmem_obj_t *sko = NULL; unsigned long key = spl_hash_ptr(obj, skc->skc_hash_bits); int i = 0; ENTRY; spin_lock(&skc->skc_lock); hlist_for_each_entry(sko, node, &skc->skc_hash[key], sko_hlist) { if (unlikely((++i) > skc->skc_hash_depth)) skc->skc_hash_depth = i; if (sko->sko_addr == obj) { ASSERT(sko->sko_magic == SKO_MAGIC); sks = sko->sko_slab; break; } } ASSERT(sko != NULL); /* Obj must be in hash */ ASSERT(sks != NULL); /* Obj must reference slab */ ASSERT(sks->sks_cache == skc); hlist_del_init(&sko->sko_hlist); list_add(&sko->sko_list, &sks->sks_free_list); sks->sks_age = jiffies; atomic_dec(&sks->sks_ref); skc->skc_obj_alloc--; /* Move slab to skc_partial_list when no longer full. Slabs * are added to the kead to keep the partial list is quasi * full sorted order. Fuller at the head, emptier at the tail. */ if (atomic_read(&sks->sks_ref) == (sks->sks_objs - 1)) { list_del(&sks->sks_list); list_add(&sks->sks_list, &skc->skc_partial_list); } /* Move emply slabs to the end of the partial list so * they can be easily found and freed during reclamation. */ if (atomic_read(&sks->sks_ref) == 0) { list_del(&sks->sks_list); list_add_tail(&sks->sks_list, &skc->skc_partial_list); skc->skc_slab_alloc--; } __slab_reclaim(skc); spin_unlock(&skc->skc_lock); } EXPORT_SYMBOL(spl_kmem_cache_free); static int kmem_cache_generic_shrinker(int nr_to_scan, unsigned int gfp_mask) { spl_kmem_cache_t *skc; /* Under linux a shrinker is not tightly coupled with a slab * cache. In fact linux always systematically trys calling all * registered shrinker callbacks until its target reclamation level * is reached. Because of this we only register one shrinker * function in the shim layer for all slab caches. And we always * attempt to shrink all caches when this generic shrinker is called. */ down_read(&spl_kmem_cache_sem); list_for_each_entry(skc, &spl_kmem_cache_list, skc_list) spl_kmem_cache_reap_now(skc); up_read(&spl_kmem_cache_sem); /* XXX: Under linux we should return the remaining number of * entries in the cache. We should do this as well. */ return 1; } void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc) { ENTRY; ASSERT(skc && skc->skc_magic == SKC_MAGIC); if (skc->skc_reclaim) skc->skc_reclaim(skc->skc_private); slab_reclaim(skc); EXIT; } EXPORT_SYMBOL(spl_kmem_cache_reap_now); void spl_kmem_reap(void) { kmem_cache_generic_shrinker(KMC_REAP_CHUNK, GFP_KERNEL); } EXPORT_SYMBOL(spl_kmem_reap); int spl_kmem_init(void) { int rc = 0; ENTRY; init_rwsem(&spl_kmem_cache_sem); INIT_LIST_HEAD(&spl_kmem_cache_list); spl_slab_cache = NULL; spl_obj_cache = NULL; spl_slab_cache = __kmem_cache_create("spl_slab_cache", sizeof(spl_kmem_slab_t), 0, 0, NULL, NULL); if (spl_slab_cache == NULL) GOTO(out_cache, rc = -ENOMEM); spl_obj_cache = __kmem_cache_create("spl_obj_cache", sizeof(spl_kmem_obj_t), 0, 0, NULL, NULL); if (spl_obj_cache == NULL) GOTO(out_cache, rc = -ENOMEM); #ifdef HAVE_SET_SHRINKER spl_kmem_cache_shrinker = set_shrinker(KMC_DEFAULT_SEEKS, kmem_cache_generic_shrinker); if (spl_kmem_cache_shrinker == NULL) GOTO(out_cache, rc = -ENOMEM); #else register_shrinker(&spl_kmem_cache_shrinker); #endif #ifdef DEBUG_KMEM { int i; atomic64_set(&kmem_alloc_used, 0); atomic64_set(&vmem_alloc_used, 0); atomic64_set(&kmem_cache_alloc_failed, 0); spin_lock_init(&kmem_lock); INIT_LIST_HEAD(&kmem_list); for (i = 0; i < KMEM_TABLE_SIZE; i++) INIT_HLIST_HEAD(&kmem_table[i]); spin_lock_init(&vmem_lock); INIT_LIST_HEAD(&vmem_list); for (i = 0; i < VMEM_TABLE_SIZE; i++) INIT_HLIST_HEAD(&vmem_table[i]); } #endif RETURN(rc); out_cache: if (spl_obj_cache) (void)kmem_cache_destroy(spl_obj_cache); if (spl_slab_cache) (void)kmem_cache_destroy(spl_slab_cache); RETURN(rc); } #ifdef DEBUG_KMEM static char * sprintf_addr(kmem_debug_t *kd, char *str, int len, int min) { int size = ((len - 1) < kd->kd_size) ? (len - 1) : kd->kd_size; int i, flag = 1; ASSERT(str != NULL && len >= 17); memset(str, 0, len); /* Check for a fully printable string, and while we are at * it place the printable characters in the passed buffer. */ for (i = 0; i < size; i++) { str[i] = ((char *)(kd->kd_addr))[i]; if (isprint(str[i])) { continue; } else { /* Minimum number of printable characters found * to make it worthwhile to print this as ascii. */ if (i > min) break; flag = 0; break; } } if (!flag) { sprintf(str, "%02x%02x%02x%02x%02x%02x%02x%02x", *((uint8_t *)kd->kd_addr), *((uint8_t *)kd->kd_addr + 2), *((uint8_t *)kd->kd_addr + 4), *((uint8_t *)kd->kd_addr + 6), *((uint8_t *)kd->kd_addr + 8), *((uint8_t *)kd->kd_addr + 10), *((uint8_t *)kd->kd_addr + 12), *((uint8_t *)kd->kd_addr + 14)); } return str; } #endif /* DEBUG_KMEM */ void spl_kmem_fini(void) { #ifdef DEBUG_KMEM unsigned long flags; kmem_debug_t *kd; char str[17]; /* Display all unreclaimed memory addresses, including the * allocation size and the first few bytes of what's located * at that address to aid in debugging. Performance is not * a serious concern here since it is module unload time. */ if (atomic64_read(&kmem_alloc_used) != 0) CWARN("kmem leaked %ld/%ld bytes\n", atomic_read(&kmem_alloc_used), kmem_alloc_max); spin_lock_irqsave(&kmem_lock, flags); if (!list_empty(&kmem_list)) CDEBUG(D_WARNING, "%-16s %-5s %-16s %s:%s\n", "address", "size", "data", "func", "line"); list_for_each_entry(kd, &kmem_list, kd_list) CDEBUG(D_WARNING, "%p %-5d %-16s %s:%d\n", kd->kd_addr, kd->kd_size, sprintf_addr(kd, str, 17, 8), kd->kd_func, kd->kd_line); spin_unlock_irqrestore(&kmem_lock, flags); if (atomic64_read(&vmem_alloc_used) != 0) CWARN("vmem leaked %ld/%ld bytes\n", atomic_read(&vmem_alloc_used), vmem_alloc_max); spin_lock_irqsave(&vmem_lock, flags); if (!list_empty(&vmem_list)) CDEBUG(D_WARNING, "%-16s %-5s %-16s %s:%s\n", "address", "size", "data", "func", "line"); list_for_each_entry(kd, &vmem_list, kd_list) CDEBUG(D_WARNING, "%p %-5d %-16s %s:%d\n", kd->kd_addr, kd->kd_size, sprintf_addr(kd, str, 17, 8), kd->kd_func, kd->kd_line); spin_unlock_irqrestore(&vmem_lock, flags); #endif ENTRY; #ifdef HAVE_SET_SHRINKER remove_shrinker(spl_kmem_cache_shrinker); #else unregister_shrinker(&spl_kmem_cache_shrinker); #endif (void)kmem_cache_destroy(spl_obj_cache); (void)kmem_cache_destroy(spl_slab_cache); EXIT; }