/*- * Copyright (c) 2011 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Matt Thomas of 3am Software Foundry. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include <sys/cdefs.h> __KERNEL_RCSID(0, "$NetBSD: pmap_synci.c,v 1.5 2020/04/13 08:05:22 skrll Exp $"); #define __PMAP_PRIVATE #include "opt_multiprocessor.h" #include <sys/param.h> #include <sys/atomic.h> #include <sys/cpu.h> #include <sys/mutex.h> #include <sys/systm.h> #include <uvm/uvm.h> #if defined(MULTIPROCESSOR) u_int pmap_tlb_synci_page_mask; u_int pmap_tlb_synci_map_mask; void pmap_tlb_syncicache_ast(struct cpu_info *ci) { struct pmap_tlb_info * const ti = cpu_tlb_info(ci); KASSERT(kpreempt_disabled()); uint32_t page_bitmap = atomic_swap_32(&ti->ti_synci_page_bitmap, 0); #if 0 printf("%s: need to sync %#x\n", __func__, page_bitmap); #endif ti->ti_evcnt_synci_asts.ev_count++; /* * If every bit is set in the bitmap, sync the entire icache. */ if (page_bitmap == pmap_tlb_synci_map_mask) { pmap_md_icache_sync_all(); ti->ti_evcnt_synci_all.ev_count++; ti->ti_evcnt_synci_pages.ev_count += pmap_tlb_synci_page_mask+1; return; } /* * Loop through the bitmap clearing each set of indices for each page. */ for (vaddr_t va = 0; page_bitmap != 0; page_bitmap >>= 1, va += PAGE_SIZE) { if (page_bitmap & 1) { /* * Each bit set represents a page index to be synced. */ pmap_md_icache_sync_range_index(va, PAGE_SIZE); ti->ti_evcnt_synci_pages.ev_count++; } } } void pmap_tlb_syncicache(vaddr_t va, const kcpuset_t *page_onproc) { KASSERT(kpreempt_disabled()); /* * We don't sync the icache here but let ast do it for us just before * returning to userspace. We do this because we don't really know * on which CPU we will return to userspace and if we synch the icache * now it might not be on the CPU we need it on. In addition, others * threads might sync the icache before we get to return to userland * so there's no reason for us to do it. * * Each TLB/cache keeps a synci sequence number which gets advanced * each time that TLB/cache performs a pmap_md_sync_icache_all. When * we return to userland, we check the pmap's corresponding synci * sequence number for that TLB/cache. If they match, it means that * no one has yet synched the icache so we much do it ourselves. If * they don't match someone has already synced the icache for us. * * There is a small chance that the generation numbers will wrap and * then become equal but that's a one in 4 billion cache and will * just cause an extra sync of the icache. */ struct cpu_info * const ci = curcpu(); kcpuset_t *onproc; kcpuset_create(&onproc, true); const uint32_t page_mask = 1L << ((va >> PGSHIFT) & pmap_tlb_synci_page_mask); for (size_t i = 0; i < pmap_ntlbs; i++) { struct pmap_tlb_info * const ti = pmap_tlbs[i]; TLBINFO_LOCK(ti); for (;;) { uint32_t old_page_bitmap = ti->ti_synci_page_bitmap; if (old_page_bitmap & page_mask) { ti->ti_evcnt_synci_duplicate.ev_count++; break; } uint32_t orig_page_bitmap = atomic_cas_32( &ti->ti_synci_page_bitmap, old_page_bitmap, old_page_bitmap | page_mask); if (orig_page_bitmap == old_page_bitmap) { if (old_page_bitmap == 0) { kcpuset_merge(onproc, ti->ti_kcpuset); } else { ti->ti_evcnt_synci_deferred.ev_count++; } ti->ti_evcnt_synci_desired.ev_count++; break; } } #if 0 printf("%s: %s: %x to %x on cpus %#x\n", __func__, ti->ti_name, page_mask, ti->ti_synci_page_bitmap, onproc & page_onproc & ti->ti_cpu_mask); #endif TLBINFO_UNLOCK(ti); } kcpuset_intersect(onproc, page_onproc); if (__predict_false(!kcpuset_iszero(onproc))) { /* * If the cpu need to sync this page, tell the current lwp * to sync the icache before it returns to userspace. */ if (kcpuset_isset(onproc, cpu_index(ci))) { if (ci->ci_flags & CPUF_USERPMAP) { curlwp->l_md.md_astpending = 1; /* force call to ast() */ ci->ci_evcnt_synci_onproc_rqst.ev_count++; } else { ci->ci_evcnt_synci_deferred_rqst.ev_count++; } kcpuset_clear(onproc, cpu_index(ci)); } /* * For each cpu that is affect, send an IPI telling * that CPU that the current thread needs to sync its icache. * We might cause some spurious icache syncs but that's not * going to break anything. */ for (cpuid_t n = kcpuset_ffs(onproc); n-- > 0; n = kcpuset_ffs(onproc)) { kcpuset_clear(onproc, n); cpu_send_ipi(cpu_lookup(n), IPI_SYNCICACHE); } } kcpuset_destroy(onproc); } void pmap_tlb_syncicache_wanted(struct cpu_info *ci) { struct pmap_tlb_info * const ti = cpu_tlb_info(ci); KASSERT(cpu_intr_p()); TLBINFO_LOCK(ti); /* * We might have been notified because another CPU changed an exec * page and now needs us to sync the icache so tell the current lwp * to do the next time it returns to userland (which should be very * soon). */ if (ti->ti_synci_page_bitmap && (ci->ci_flags & CPUF_USERPMAP)) { curlwp->l_md.md_astpending = 1; /* force call to ast() */ ci->ci_evcnt_synci_ipi_rqst.ev_count++; } TLBINFO_UNLOCK(ti); } #endif /* MULTIPROCESSOR */
