/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Kernel Physical Mapping (kpm) segment driver (segkpm). * * This driver delivers along with the hat_kpm* interfaces an alternative * mechanism for kernel mappings within the 64-bit Solaris operating system, * which allows the mapping of all physical memory into the kernel address * space at once. This is feasible in 64 bit kernels, e.g. for Ultrasparc II * and beyond processors, since the available VA range is much larger than * possible physical memory. Momentarily all physical memory is supported, * that is represented by the list of memory segments (memsegs). * * Segkpm mappings have also very low overhead and large pages are used * (when possible) to minimize the TLB and TSB footprint. It is also * extentable for other than Sparc architectures (e.g. AMD64). Main * advantage is the avoidance of the TLB-shootdown X-calls, which are * normally needed when a kernel (global) mapping has to be removed. * * First example of a kernel facility that uses the segkpm mapping scheme * is seg_map, where it is used as an alternative to hat_memload(). * See also hat layer for more information about the hat_kpm* routines. * The kpm facilty can be turned off at boot time (e.g. /etc/system). */ #include <sys/types.h> #include <sys/param.h> #include <sys/sysmacros.h> #include <sys/systm.h> #include <sys/vnode.h> #include <sys/cmn_err.h> #include <sys/debug.h> #include <sys/thread.h> #include <sys/cpuvar.h> #include <sys/bitmap.h> #include <sys/atomic.h> #include <sys/lgrp.h> #include <vm/seg_kmem.h> #include <vm/seg_kpm.h> #include <vm/hat.h> #include <vm/as.h> #include <vm/seg.h> #include <vm/page.h> /* * Global kpm controls. * See also platform and mmu specific controls. * * kpm_enable -- global on/off switch for segkpm. * . Set by default on 64bit platforms that have kpm support. * . Will be disabled from platform layer if not supported. * . Can be disabled via /etc/system. * * kpm_smallpages -- use only regular/system pagesize for kpm mappings. * . Can be useful for critical debugging of kpm clients. * . Set to zero by default for platforms that support kpm large pages. * The use of kpm large pages reduces the footprint of kpm meta data * and has all the other advantages of using large pages (e.g TLB * miss reduction). * . Set by default for platforms that don't support kpm large pages or * where large pages cannot be used for other reasons (e.g. there are * only few full associative TLB entries available for large pages). * * segmap_kpm -- separate on/off switch for segmap using segkpm: * . Set by default. * . Will be disabled when kpm_enable is zero. * . Will be disabled when MAXBSIZE != PAGESIZE. * . Can be disabled via /etc/system. * */ int kpm_enable = 1; int kpm_smallpages = 0; int segmap_kpm = 1; /* * Private seg op routines. */ faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw); static void segkpm_dump(struct seg *); static void segkpm_badop(void); static int segkpm_notsup(void); static int segkpm_capable(struct seg *, segcapability_t); #define SEGKPM_BADOP(t) (t(*)())segkpm_badop #define SEGKPM_NOTSUP (int(*)())segkpm_notsup static struct seg_ops segkpm_ops = { SEGKPM_BADOP(int), /* dup */ SEGKPM_BADOP(int), /* unmap */ SEGKPM_BADOP(void), /* free */ segkpm_fault, SEGKPM_BADOP(int), /* faulta */ SEGKPM_BADOP(int), /* setprot */ SEGKPM_BADOP(int), /* checkprot */ SEGKPM_BADOP(int), /* kluster */ SEGKPM_BADOP(size_t), /* swapout */ SEGKPM_BADOP(int), /* sync */ SEGKPM_BADOP(size_t), /* incore */ SEGKPM_BADOP(int), /* lockop */ SEGKPM_BADOP(int), /* getprot */ SEGKPM_BADOP(u_offset_t), /* getoffset */ SEGKPM_BADOP(int), /* gettype */ SEGKPM_BADOP(int), /* getvp */ SEGKPM_BADOP(int), /* advise */ segkpm_dump, /* dump */ SEGKPM_NOTSUP, /* pagelock */ SEGKPM_BADOP(int), /* setpgsz */ SEGKPM_BADOP(int), /* getmemid */ SEGKPM_BADOP(lgrp_mem_policy_info_t *), /* getpolicy */ segkpm_capable, /* capable */ seg_inherit_notsup /* inherit */ }; /* * kpm_pgsz and kpm_pgshft are set by platform layer. */ size_t kpm_pgsz; /* kpm page size */ uint_t kpm_pgshft; /* kpm page shift */ u_offset_t kpm_pgoff; /* kpm page offset mask */ uint_t kpmp2pshft; /* kpm page to page shift */ pgcnt_t kpmpnpgs; /* how many pages per kpm page */ #ifdef SEGKPM_SUPPORT int segkpm_create(struct seg *seg, void *argsp) { struct segkpm_data *skd; struct segkpm_crargs *b = (struct segkpm_crargs *)argsp; ushort_t *p; int i, j; ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock)); ASSERT(btokpmp(seg->s_size) >= 1 && kpmpageoff((uintptr_t)seg->s_base) == 0 && kpmpageoff((uintptr_t)seg->s_base + seg->s_size) == 0); skd = kmem_zalloc(sizeof (struct segkpm_data), KM_SLEEP); seg->s_data = (void *)skd; seg->s_ops = &segkpm_ops; skd->skd_prot = b->prot; /* * (1) Segkpm virtual addresses are based on physical adresses. * From this and in opposite to other segment drivers it is * often required to allocate a page first to be able to * calculate the final segkpm virtual address. * (2) Page allocation is done by calling page_create_va(), * one important input argument is a virtual address (also * expressed by the "va" in the function name). This function * is highly optimized to select the right page for an optimal * processor and platform support (e.g. virtual addressed * caches (VAC), physical addressed caches, NUMA). * * Because of (1) the approach is to generate a faked virtual * address for calling page_create_va(). In order to exploit * the abilities of (2), especially to utilize the cache * hierarchy (3) and to avoid VAC alias conflicts (4) the * selection has to be done carefully. For each virtual color * a separate counter is provided (4). The count values are * used for the utilization of all cache lines (3) and are * corresponding to the cache bins. */ skd->skd_nvcolors = b->nvcolors; p = skd->skd_va_select = kmem_zalloc(NCPU * b->nvcolors * sizeof (ushort_t), KM_SLEEP); for (i = 0; i < NCPU; i++) for (j = 0; j < b->nvcolors; j++, p++) *p = j; return (0); } /* * This routine is called via a machine specific fault handling * routine. */ /* ARGSUSED */ faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw) { ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); switch (type) { case F_INVAL: return (hat_kpm_fault(hat, addr)); case F_SOFTLOCK: case F_SOFTUNLOCK: return (0); default: return (FC_NOSUPPORT); } /*NOTREACHED*/ } #define addr_to_vcolor(addr, vcolors) \ ((int)(((uintptr_t)(addr) & ((vcolors << PAGESHIFT) - 1)) >> PAGESHIFT)) /* * Create a virtual address that can be used for invocations of * page_create_va. Goal is to utilize the cache hierarchy (round * robin bins) and to select the right color for virtual indexed * caches. It isn't exact since we also increment the bin counter * when the caller uses VOP_GETPAGE and gets a hit in the page * cache, but we keep the bins turning for cache distribution * (see also segkpm_create block comment). */ caddr_t segkpm_create_va(u_offset_t off) { int vcolor; ushort_t *p; struct segkpm_data *skd = (struct segkpm_data *)segkpm->s_data; int nvcolors = skd->skd_nvcolors; caddr_t va; vcolor = (nvcolors > 1) ? addr_to_vcolor(off, nvcolors) : 0; p = &skd->skd_va_select[(CPU->cpu_id * nvcolors) + vcolor]; va = (caddr_t)ptob(*p); atomic_add_16(p, nvcolors); return (va); } /* * Unload mapping if the instance has an active kpm mapping. */ void segkpm_mapout_validkpme(struct kpme *kpme) { caddr_t vaddr; page_t *pp; retry: if ((pp = kpme->kpe_page) == NULL) { return; } if (page_lock(pp, SE_SHARED, (kmutex_t *)NULL, P_RECLAIM) == 0) goto retry; /* * Check if segkpm mapping is not unloaded in the meantime */ if (kpme->kpe_page == NULL) { page_unlock(pp); return; } vaddr = hat_kpm_page2va(pp, 1); hat_kpm_mapout(pp, kpme, vaddr); page_unlock(pp); } static void segkpm_badop() { panic("segkpm_badop"); } #else /* SEGKPM_SUPPORT */ /* segkpm stubs */ /*ARGSUSED*/ int segkpm_create(struct seg *seg, void *argsp) { return (0); } /* ARGSUSED */ faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw) { return ((faultcode_t)0); } /* ARGSUSED */ caddr_t segkpm_create_va(u_offset_t off) { return (NULL); } /* ARGSUSED */ void segkpm_mapout_validkpme(struct kpme *kpme) {} static void segkpm_badop() {} #endif /* SEGKPM_SUPPORT */ static int segkpm_notsup() { return (ENOTSUP); } /* * segkpm pages are not dumped, so we just return */ /*ARGSUSED*/ static void segkpm_dump(struct seg *seg) {} /* * We claim to have no special capabilities. */ /*ARGSUSED*/ static int segkpm_capable(struct seg *seg, segcapability_t capability) { return (0); }
