/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2018 Joyent, Inc. */ #ifndef _SYS_KMEM_IMPL_H #define _SYS_KMEM_IMPL_H #include <sys/kmem.h> #include <sys/vmem.h> #include <sys/thread.h> #include <sys/t_lock.h> #include <sys/time.h> #include <sys/kstat.h> #include <sys/cpuvar.h> #include <sys/systm.h> #include <vm/page.h> #include <sys/avl.h> #include <sys/list.h> #ifdef __cplusplus extern "C" { #endif /* * kernel memory allocator: implementation-private data structures * * Lock order: * 1. cache_lock * 2. cc_lock in order by CPU ID * 3. cache_depot_lock * * Do not call kmem_cache_alloc() or taskq_dispatch() while holding any of the * above locks. */ #define KMF_AUDIT 0x00000001 /* transaction auditing */ #define KMF_DEADBEEF 0x00000002 /* deadbeef checking */ #define KMF_REDZONE 0x00000004 /* redzone checking */ #define KMF_CONTENTS 0x00000008 /* freed-buffer content logging */ #define KMF_STICKY 0x00000010 /* if set, override /etc/system */ #define KMF_NOMAGAZINE 0x00000020 /* disable per-cpu magazines */ #define KMF_FIREWALL 0x00000040 /* put all bufs before unmapped pages */ #define KMF_LITE 0x00000100 /* lightweight debugging */ #define KMF_HASH 0x00000200 /* cache has hash table */ #define KMF_RANDOMIZE 0x00000400 /* randomize other kmem_flags */ #define KMF_DUMPDIVERT 0x00001000 /* use alternate memory at dump time */ #define KMF_DUMPUNSAFE 0x00002000 /* flag caches used at dump time */ #define KMF_PREFILL 0x00004000 /* Prefill the slab when created. */ #define KMF_BUFTAG (KMF_DEADBEEF | KMF_REDZONE) #define KMF_TOUCH (KMF_BUFTAG | KMF_LITE | KMF_CONTENTS) #define KMF_RANDOM (KMF_TOUCH | KMF_AUDIT | KMF_NOMAGAZINE) #define KMF_DEBUG (KMF_RANDOM | KMF_FIREWALL) #define KMEM_STACK_DEPTH 15 #define KMEM_FREE_PATTERN 0xdeadbeefdeadbeefULL #define KMEM_UNINITIALIZED_PATTERN 0xbaddcafebaddcafeULL #define KMEM_REDZONE_PATTERN 0xfeedfacefeedfaceULL #define KMEM_REDZONE_BYTE 0xbb /* * Redzone size encodings for kmem_alloc() / kmem_free(). We encode the * allocation size, rather than storing it directly, so that kmem_free() * can distinguish frees of the wrong size from redzone violations. * * A size of zero is never valid. */ #define KMEM_SIZE_ENCODE(x) (251 * (x) + 1) #define KMEM_SIZE_DECODE(x) ((x) / 251) #define KMEM_SIZE_VALID(x) ((x) % 251 == 1 && (x) != 1) #define KMEM_ALIGN 8 /* min guaranteed alignment */ #define KMEM_ALIGN_SHIFT 3 /* log2(KMEM_ALIGN) */ #define KMEM_VOID_FRACTION 8 /* never waste more than 1/8 of slab */ #define KMEM_SLAB_IS_PARTIAL(sp) \ ((sp)->slab_refcnt > 0 && (sp)->slab_refcnt < (sp)->slab_chunks) #define KMEM_SLAB_IS_ALL_USED(sp) \ ((sp)->slab_refcnt == (sp)->slab_chunks) /* * The bufctl (buffer control) structure keeps some minimal information * about each buffer: its address, its slab, and its current linkage, * which is either on the slab's freelist (if the buffer is free), or * on the cache's buf-to-bufctl hash table (if the buffer is allocated). * In the case of non-hashed, or "raw", caches (the common case), only * the freelist linkage is necessary: the buffer address is at a fixed * offset from the bufctl address, and the slab is at the end of the page. * * NOTE: bc_next must be the first field; raw buffers have linkage only. */ typedef struct kmem_bufctl { struct kmem_bufctl *bc_next; /* next bufctl struct */ void *bc_addr; /* address of buffer */ struct kmem_slab *bc_slab; /* controlling slab */ } kmem_bufctl_t; /* * The KMF_AUDIT version of the bufctl structure. The beginning of this * structure must be identical to the normal bufctl structure so that * pointers are interchangeable. */ typedef struct kmem_bufctl_audit { struct kmem_bufctl *bc_next; /* next bufctl struct */ void *bc_addr; /* address of buffer */ struct kmem_slab *bc_slab; /* controlling slab */ kmem_cache_t *bc_cache; /* controlling cache */ hrtime_t bc_timestamp; /* transaction time */ kthread_t *bc_thread; /* thread doing transaction */ struct kmem_bufctl *bc_lastlog; /* last log entry */ void *bc_contents; /* contents at last free */ int bc_depth; /* stack depth */ pc_t bc_stack[KMEM_STACK_DEPTH]; /* pc stack */ } kmem_bufctl_audit_t; /* * A kmem_buftag structure is appended to each buffer whenever any of the * KMF_BUFTAG flags (KMF_DEADBEEF, KMF_REDZONE, KMF_VERIFY) are set. */ typedef struct kmem_buftag { uint64_t bt_redzone; /* 64-bit redzone pattern */ kmem_bufctl_t *bt_bufctl; /* bufctl */ intptr_t bt_bxstat; /* bufctl ^ (alloc/free) */ } kmem_buftag_t; /* * A variant of the kmem_buftag structure used for KMF_LITE caches. * Previous callers are stored in reverse chronological order. (i.e. most * recent first) */ typedef struct kmem_buftag_lite { kmem_buftag_t bt_buftag; /* a normal buftag */ pc_t bt_history[1]; /* zero or more callers */ } kmem_buftag_lite_t; #define KMEM_BUFTAG_LITE_SIZE(f) \ (offsetof(kmem_buftag_lite_t, bt_history[f])) #define KMEM_BUFTAG(cp, buf) \ ((kmem_buftag_t *)((char *)(buf) + (cp)->cache_buftag)) #define KMEM_BUFCTL(cp, buf) \ ((kmem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl)) #define KMEM_BUF(cp, bcp) \ ((void *)((char *)(bcp) - (cp)->cache_bufctl)) #define KMEM_SLAB(cp, buf) \ ((kmem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1) /* * Test for using alternate memory at dump time. */ #define KMEM_DUMP(cp) ((cp)->cache_flags & KMF_DUMPDIVERT) #define KMEM_DUMPCC(ccp) ((ccp)->cc_flags & KMF_DUMPDIVERT) /* * The "CPU" macro loads a cpu_t that refers to the cpu that the current * thread is running on at the time the macro is executed. A context switch * may occur immediately after loading this data structure, leaving this * thread pointing at the cpu_t for the previous cpu. This is not a problem; * we'd just end up checking the previous cpu's per-cpu cache, and then check * the other layers of the kmem cache if need be. * * It's not even a problem if the old cpu gets DR'ed out during the context * switch. The cpu-remove DR operation bzero()s the cpu_t, but doesn't free * it. So the cpu_t's cpu_cache_offset would read as 0, causing us to use * cpu 0's per-cpu cache. * * So, there is no need to disable kernel preemption while using the CPU macro * below since if we have been context switched, there will not be any * correctness problem, just a momentary use of a different per-cpu cache. */ #define KMEM_CPU_CACHE(cp) \ ((kmem_cpu_cache_t *)((char *)(&cp->cache_cpu) + CPU->cpu_cache_offset)) #define KMEM_MAGAZINE_VALID(cp, mp) \ (((kmem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \ (cp)->cache_magtype->mt_cache) #define KMEM_SLAB_OFFSET(sp, buf) \ ((size_t)((uintptr_t)(buf) - (uintptr_t)((sp)->slab_base))) #define KMEM_SLAB_MEMBER(sp, buf) \ (KMEM_SLAB_OFFSET(sp, buf) < (sp)->slab_cache->cache_slabsize) #define KMEM_BUFTAG_ALLOC 0xa110c8edUL #define KMEM_BUFTAG_FREE 0xf4eef4eeUL /* slab_later_count thresholds */ #define KMEM_DISBELIEF 3 /* slab_flags */ #define KMEM_SLAB_NOMOVE 0x1 #define KMEM_SLAB_MOVE_PENDING 0x2 typedef struct kmem_slab { struct kmem_cache *slab_cache; /* controlling cache */ void *slab_base; /* base of allocated memory */ avl_node_t slab_link; /* slab linkage */ struct kmem_bufctl *slab_head; /* first free buffer */ long slab_refcnt; /* outstanding allocations */ long slab_chunks; /* chunks (bufs) in this slab */ uint32_t slab_stuck_offset; /* unmoved buffer offset */ uint16_t slab_later_count; /* cf KMEM_CBRC_LATER */ uint16_t slab_flags; /* bits to mark the slab */ } kmem_slab_t; #define KMEM_HASH_INITIAL 64 #define KMEM_HASH(cp, buf) \ ((cp)->cache_hash_table + \ (((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask)) typedef struct kmem_magazine { void *mag_next; void *mag_round[1]; /* one or more rounds */ } kmem_magazine_t; /* * The magazine types for fast per-cpu allocation */ typedef struct kmem_magtype { short mt_magsize; /* magazine size (number of rounds) */ int mt_align; /* magazine alignment */ size_t mt_minbuf; /* all smaller buffers qualify */ size_t mt_maxbuf; /* no larger buffers qualify */ kmem_cache_t *mt_cache; /* magazine cache */ } kmem_magtype_t; #define KMEM_CPU_CACHE_SIZE 64 /* must be power of 2 */ #define KMEM_CPU_PAD (KMEM_CPU_CACHE_SIZE - sizeof (kmutex_t) - \ 2 * sizeof (uint64_t) - 2 * sizeof (void *) - sizeof (int) - \ 5 * sizeof (short)) #define KMEM_CACHE_SIZE(ncpus) \ ((size_t)(&((kmem_cache_t *)0)->cache_cpu[ncpus])) /* Offset from kmem_cache->cache_cpu for per cpu caches */ #define KMEM_CPU_CACHE_OFFSET(cpuid) \ ((size_t)(&((kmem_cache_t *)0)->cache_cpu[cpuid]) - \ (size_t)(&((kmem_cache_t *)0)->cache_cpu)) typedef struct kmem_cpu_cache { kmutex_t cc_lock; /* protects this cpu's local cache */ uint64_t cc_alloc; /* allocations from this cpu */ uint64_t cc_free; /* frees to this cpu */ kmem_magazine_t *cc_loaded; /* the currently loaded magazine */ kmem_magazine_t *cc_ploaded; /* the previously loaded magazine */ int cc_flags; /* CPU-local copy of cache_flags */ short cc_rounds; /* number of objects in loaded mag */ short cc_prounds; /* number of objects in previous mag */ short cc_magsize; /* number of rounds in a full mag */ short cc_dump_rounds; /* dump time copy of cc_rounds */ short cc_dump_prounds; /* dump time copy of cc_prounds */ char cc_pad[KMEM_CPU_PAD]; /* for nice alignment */ } kmem_cpu_cache_t; /* * The magazine lists used in the depot. */ typedef struct kmem_maglist { kmem_magazine_t *ml_list; /* magazine list */ long ml_total; /* number of magazines */ long ml_min; /* min since last update */ long ml_reaplimit; /* max reapable magazines */ uint64_t ml_alloc; /* allocations from this list */ } kmem_maglist_t; typedef struct kmem_defrag { /* * Statistics */ uint64_t kmd_callbacks; /* move callbacks */ uint64_t kmd_yes; /* KMEM_CBRC_YES responses */ uint64_t kmd_no; /* NO responses */ uint64_t kmd_later; /* LATER responses */ uint64_t kmd_dont_need; /* DONT_NEED responses */ uint64_t kmd_dont_know; /* DONT_KNOW responses */ uint64_t kmd_slabs_freed; /* slabs freed by moves */ uint64_t kmd_defrags; /* kmem_cache_defrag() */ uint64_t kmd_scans; /* kmem_cache_scan() */ /* * Consolidator fields */ avl_tree_t kmd_moves_pending; /* buffer moves pending */ list_t kmd_deadlist; /* deferred slab frees */ size_t kmd_deadcount; /* # of slabs in kmd_deadlist */ uint8_t kmd_reclaim_numer; /* slab usage threshold */ uint8_t kmd_pad1; /* compiler padding */ uint16_t kmd_consolidate; /* triggers consolidator */ uint32_t kmd_pad2; /* compiler padding */ size_t kmd_slabs_sought; /* reclaimable slabs sought */ size_t kmd_slabs_found; /* reclaimable slabs found */ size_t kmd_tries; /* nth scan interval counter */ /* * Fields used to ASSERT that the client does not kmem_cache_free() * objects passed to the move callback. */ void *kmd_from_buf; /* object to move */ void *kmd_to_buf; /* move destination */ kthread_t *kmd_thread; /* thread calling move */ } kmem_defrag_t; typedef struct kmem_dump { void *kd_freelist; /* heap during crash dump */ uint_t kd_alloc_fails; /* # of allocation failures */ uint_t kd_unsafe; /* cache was used, but unsafe */ } kmem_dump_t; #define KMEM_CACHE_NAMELEN 31 struct kmem_cache { /* * Statistics */ uint64_t cache_slab_create; /* slab creates */ uint64_t cache_slab_destroy; /* slab destroys */ uint64_t cache_slab_alloc; /* slab layer allocations */ uint64_t cache_slab_free; /* slab layer frees */ uint64_t cache_alloc_fail; /* total failed allocations */ uint64_t cache_buftotal; /* total buffers */ uint64_t cache_bufmax; /* max buffers ever */ uint64_t cache_bufslab; /* buffers free in slab layer */ uint64_t cache_reap; /* cache reaps */ uint64_t cache_rescale; /* hash table rescales */ uint64_t cache_lookup_depth; /* hash lookup depth */ uint64_t cache_depot_contention; /* mutex contention count */ uint64_t cache_depot_contention_prev; /* previous snapshot */ /* * Cache properties */ char cache_name[KMEM_CACHE_NAMELEN + 1]; size_t cache_bufsize; /* object size */ size_t cache_align; /* object alignment */ int (*cache_constructor)(void *, void *, int); void (*cache_destructor)(void *, void *); void (*cache_reclaim)(void *); kmem_cbrc_t (*cache_move)(void *, void *, size_t, void *); void *cache_private; /* opaque arg to callbacks */ vmem_t *cache_arena; /* vmem source for slabs */ int cache_cflags; /* cache creation flags */ int cache_flags; /* various cache state info */ uint32_t cache_mtbf; /* induced alloc failure rate */ uint32_t cache_pad1; /* compiler padding */ kstat_t *cache_kstat; /* exported statistics */ list_node_t cache_link; /* cache linkage */ /* * Slab layer */ kmutex_t cache_lock; /* protects slab layer */ size_t cache_chunksize; /* buf + alignment [+ debug] */ size_t cache_slabsize; /* size of a slab */ size_t cache_maxchunks; /* max buffers per slab */ size_t cache_bufctl; /* buf-to-bufctl distance */ size_t cache_buftag; /* buf-to-buftag distance */ size_t cache_verify; /* bytes to verify */ size_t cache_contents; /* bytes of saved content */ size_t cache_color; /* next slab color */ size_t cache_mincolor; /* maximum slab color */ size_t cache_maxcolor; /* maximum slab color */ size_t cache_hash_shift; /* get to interesting bits */ size_t cache_hash_mask; /* hash table mask */ list_t cache_complete_slabs; /* completely allocated slabs */ size_t cache_complete_slab_count; avl_tree_t cache_partial_slabs; /* partial slab freelist */ size_t cache_partial_binshift; /* for AVL sort bins */ kmem_cache_t *cache_bufctl_cache; /* source of bufctls */ kmem_bufctl_t **cache_hash_table; /* hash table base */ kmem_defrag_t *cache_defrag; /* slab consolidator fields */ /* * Depot layer */ kmutex_t cache_depot_lock; /* protects depot */ kmem_magtype_t *cache_magtype; /* magazine type */ kmem_maglist_t cache_full; /* full magazines */ kmem_maglist_t cache_empty; /* empty magazines */ kmem_dump_t cache_dump; /* used during crash dump */ /* * Per-CPU layer */ kmem_cpu_cache_t cache_cpu[1]; /* max_ncpus actual elements */ }; typedef struct kmem_cpu_log_header { kmutex_t clh_lock; char *clh_current; size_t clh_avail; int clh_chunk; int clh_hits; char clh_pad[64 - sizeof (kmutex_t) - sizeof (char *) - sizeof (size_t) - 2 * sizeof (int)]; } kmem_cpu_log_header_t; typedef struct kmem_log_header { kmutex_t lh_lock; char *lh_base; int *lh_free; size_t lh_chunksize; int lh_nchunks; int lh_head; int lh_tail; int lh_hits; kmem_cpu_log_header_t lh_cpu[1]; /* ncpus actually allocated */ } kmem_log_header_t; /* kmem_move kmm_flags */ #define KMM_DESPERATE 0x1 #define KMM_NOTIFY 0x2 #define KMM_DEBUG 0x4 typedef struct kmem_move { kmem_slab_t *kmm_from_slab; void *kmm_from_buf; void *kmm_to_buf; avl_node_t kmm_entry; int kmm_flags; } kmem_move_t; /* * In order to consolidate partial slabs, it must be possible for the cache to * have partial slabs. */ #define KMEM_IS_MOVABLE(cp) \ (((cp)->cache_chunksize * 2) <= (cp)->cache_slabsize) #ifdef __cplusplus } #endif #endif /* _SYS_KMEM_IMPL_H */
