/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2019 Joyent, Inc. */ /* * To understand the present state of interrupt handling on i86pc, we must * first consider the history of interrupt controllers and our way of handling * interrupts. * * History of Interrupt Controllers on i86pc * ----------------------------------------- * * Intel 8259 and 8259A * * The first interrupt controller that attained widespread use on i86pc was * the Intel 8259(A) Programmable Interrupt Controller that first saw use with * the 8086. It took up to 8 interrupt sources and combined them into one * output wire. Up to 8 8259s could be slaved together providing up to 64 IRQs. * With the switch to the 8259A, level mode interrupts became possible. For a * long time on i86pc the 8259A was the only way to handle interrupts and it * had its own set of quirks. The 8259A and its corresponding interval timer * the 8254 are programmed using outb and inb instructions. * * Intel Advanced Programmable Interrupt Controller (APIC) * * Starting around the time of the introduction of the P6 family * microarchitecture (i686) Intel introduced a new interrupt controller. * Instead of having the series of slaved 8259A devices, Intel opted to outfit * each processor with a Local APIC (lapic) and to outfit the system with at * least one, but potentially more, I/O APICs (ioapic). The lapics and ioapics * initially communicated over a dedicated bus, but this has since been * replaced. Each physical core and even hyperthread currently contains its * own local apic, which is not shared. There are a few exceptions for * hyperthreads, but that does not usually concern us. * * Instead of talking directly to 8259 for status, sending End Of Interrupt * (EOI), etc. a microprocessor now communicates directly to the lapic. This * also allows for each microprocessor to be able to have independent controls. * The programming method is different from the 8259. Consumers map the lapic * registers into uncacheable memory to read and manipulate the state. * * The number of addressable interrupt vectors was increased to 256. However * vectors 0-31 are reserved for the processor exception handling, leaving the * remaining vectors for general use. In addition to hardware generated * interrupts, the lapic provides a way for generating inter-processor * interrupts (IPI) which are the basis for CPU cross calls and CPU pokes. * * AMD ended up implementing the Intel APIC architecture in lieu of their work * with Cyrix. * * Intel x2apic * * The x2apic is an extension to the lapic which started showing up around the * same time as the Sandy Bridge chipsets. It provides a new programming mode * as well as new features. The goal of the x2apic is to solve a few problems * with the previous generation of lapic and the x2apic is backwards compatible * with the previous programming and model. The only downsides to using the * backwards compatibility is that you are not able to take advantage of the new * x2apic features. * * o The APIC ID is increased from an 8-bit value to a 32-bit value. This * increases the maximum number of addressable physical processors beyond * 256. This new ID is assembled in a similar manner as the information that * is obtainable by the extended cpuid topology leaves. * * o A new means of generating IPIs was introduced. * * o Instead of memory mapping the registers, the x2apic only allows for * programming it through a series of wrmsrs. This has important semantic * side effects. Recall that the registers were previously all mapped to * uncachable memory which meant that all operations to the local apic were * serializing instructions. With the switch to using wrmsrs this has been * relaxed and these operations can no longer be assumed to be serializing * instructions. * * Note for the rest of this we are only going to concern ourselves with the * apic and x2apic which practically all of i86pc has been using now for * quite some time. * * Interrupt Priority Levels * ------------------------- * * On i86pc systems there are a total of fifteen interrupt priority levels * (ipls) which range from 1-15. Level 0 is for normal processing and * non-interrupt processing. To manipulate these values the family of spl * functions (which date back to UNIX on the PDP-11) are used. Specifically, * splr() to raise the priority level and splx() to lower it. One should not * generally call setspl() directly. * * Both i86pc and the supported SPARC platforms honor the same conventions for * the meaning behind these IPLs. The most important IPL is the platform's * LOCK_LEVEL (0xa on i86pc). If a thread is above LOCK_LEVEL it _must_ not * sleep on any synchronization object. The only allowed synchronization * primitive is a mutex that has been specifically initialized to be a spin * lock (see mutex_init(9F)). Another important level is DISP_LEVEL (0xb on * i86pc). You must be at DISP_LEVEL if you want to control the dispatcher. * The XC_HI_PIL is the highest level (0xf) and is used during cross-calls. * * Each interrupt that is registered in the system fires at a specific IPL. * Generally most interrupts fire below LOCK_LEVEL. * * PSM Drivers * ----------- * * We currently have three sets of PSM (platform specific module) drivers * available. uppc, pcplusmp, and apix. uppc (uni-processor PC) is the original * driver that interacts with the 8259A and 8254. In general, it is not used * anymore given the prevalence of the apic. * * The system prefers to use the apix driver over the pcplusmp driver. The apix * driver requires HW support for an x2apic. If there is no x2apic HW, apix * will not be used. In general we prefer using the apix driver over the * pcplusmp driver because it gives us much more flexibility with respect to * interrupts. In the apix driver each local apic has its own independent set * of interrupts, whereas the pcplusmp driver only has a single global set of * interrupts. This is why pcplusmp only supports a finite number of interrupts * per IPL -- generally 16, often less. The apix driver supports using either * the x2apic or the local apic programing modes. The programming mode does not * change the number of interrupts available, just the number of processors * that we can address. For the apix driver, the x2apic mode is enabled if the * system supports interrupt re-mapping, otherwise the module manages the * x2apic in local mode. * * When there is no x2apic present, we default back to the pcplusmp PSM driver. * In general, this is not problematic unless you have more than 256 * processors in the machine or you do not have enough interrupts available. * * Controlling Interrupt Generation on i86pc * ----------------------------------------- * * There are two different ways to manipulate which interrupts will be * generated on i86pc. Each offers different degrees of control. * * The first is through the flags register (eflags and rflags on i386 and amd64 * respectively). The IF bit determines whether or not interrupts are enabled * or disabled. This is manipulated in one of several ways. The most common way * is through the cli and sti instructions. These clear the IF flag and set it, * respectively, for the current processor. The other common way is through the * use of the intr_clear and intr_restore functions. * * Assuming interrupts are not blocked by the IF flag, then the second form is * through the Processor-Priority Register (PPR). The PPR is used to determine * whether or not a pending interrupt should be delivered. If the ipl of the * new interrupt is higher than the current value in the PPR, then the lapic * will either deliver it immediately (if interrupts are not in progress) or it * will deliver it once the current interrupt processing has issued an EOI. The * highest unmasked interrupt will be the one delivered. * * The PPR register is based upon the max of the following two registers in the * lapic, the TPR register (also known as CR8 on amd64) that can be used to * mask interrupt levels, and the current vector. Because the pcplusmp module * always sets TPR appropriately early in the do_interrupt path, we can usually * just think that the PPR is the TPR. The pcplusmp module also issues an EOI * once it has set the TPR, so higher priority interrupts can come in while * we're servicing a lower priority interrupt. * * Handling Interrupts * ------------------- * * Interrupts can be broken down into three categories based on priority and * source: * * o High level interrupts * o Low level hardware interrupts * o Low level software interrupts * * High Level Interrupts * * High level interrupts encompasses both hardware-sourced and software-sourced * interrupts. Examples of high level hardware interrupts include the serial * console. High level software-sourced interrupts are still delivered through * the local apic through IPIs. This is primarily cross calls. * * When a high level interrupt comes in, we will raise the SPL and then pin the * current lwp to the processor. We will use its lwp, but our own interrupt * stack and process the high level interrupt in-situ. These handlers are * designed to be very short in nature and cannot go to sleep, only block on a * spin lock. If the interrupt has a lot of work to do, it must generate a * low-priority software interrupt that will be processed later. * * Low level hardware interrupts * * Low level hardware interrupts start off like their high-level cousins. The * current CPU contains a number of kernel threads (kthread_t) that can be used * to process low level interrupts. These are shared between both low level * hardware and software interrupts. Note that while we run with our * kthread_t, we borrow the pinned threads lwp_t until such a time as we hit a * synchronization object. If we hit one and need to sleep, then the scheduler * will instead create the rest of what we need. * * Low level software interrupts * * Low level software interrupts are handled in a similar way as hardware * interrupts, but the notification vector is different. Each CPU has a bitmask * of pending software interrupts. We can notify a CPU to process software * interrupts through a specific trap vector as well as through several * checks that are performed throughout the code. These checks will look at * processing software interrupts as we lower our spl. * * We attempt to process the highest pending software interrupt that we can * which is greater than our current IPL. If none currently exist, then we move * on. We process a software interrupt in a similar fashion to a hardware * interrupt. * * Traditional Interrupt Flow * -------------------------- * * The following diagram tracks the flow of the traditional uppc and pcplusmp * interrupt handlers. The apix driver has its own version of do_interrupt(). * We come into the interrupt handler with all interrupts masked by the IF * flag. This is because we set up the handler using an interrupt-gate, which * is defined architecturally to have cleared the IF flag for us. * * +--------------+ +----------------+ +-----------+ * | _interrupt() |--->| do_interrupt() |--->| *setlvl() | * +--------------+ +----------------+ +-----------+ * | | | * | | | * low-level| | | softint * HW int | | +---------------------------------------+ * +--------------+ | | | * | intr_thread_ |<-----+ | hi-level int | * | prolog() | | +----------+ | * +--------------+ +--->| hilevel_ | Not on intr stack | * | | intr_ |-----------------+ | * | | prolog() | | | * +------------+ +----------+ | | * | switch_sp_ | | On intr v | * | and_call() | | Stack +------------+ | * +------------+ | | switch_sp_ | | * | v | and_call() | | * v +-----------+ +------------+ | * +-----------+ | dispatch_ | | | * | dispatch_ | +-------------------| hilevel() |<------------+ | * | hardint() | | +-----------+ | * +-----------+ | | * | v | * | +-----+ +----------------------+ +-----+ hi-level | * +---->| sti |->| av_dispatch_autovect |->| cli |---------+ | * +-----+ +----------------------+ +-----+ | | * | | | | * v | | | * +----------+ | | | * | for each | | | | * | handler | | | | * | *intr() | | v | * +--------------+ +----------+ | +----------------+ | * | intr_thread_ | low-level | | hilevel_intr_ | | * | epilog() |<-------------------------------+ | epilog() | | * +--------------+ +----------------+ | * | | | | * | +----------------------v v---------------------+ | * | +------------+ | * | +---------------------->| *setlvlx() | | * | | +------------+ | * | | | | * | | v | * | | +--------+ +------------------+ +-------------+ | * | | | return |<----| softint pending? |----->| dosoftint() |<-----+ * | | +--------+ no +------------------+ yes +-------------+ * | | ^ | | * | | | softint pil too low | | * | | +--------------------------------------+ | * | | v * | | +-----------+ +------------+ +-----------+ * | | | dispatch_ |<-----| switch_sp_ |<---------| *setspl() | * | | | softint() | | and_call() | +-----------+ * | | +-----------+ +------------+ * | | | * | | v * | | +-----+ +----------------------+ +-----+ +------------+ * | | | sti |->| av_dispatch_autovect |->| cli |->| dosoftint_ | * | | +-----+ +----------------------+ +-----+ | epilog() | * | | +------------+ * | | | | * | +----------------------------------------------------+ | * v | * +-----------+ | * | interrupt | | * | thread |<---------------------------------------------------+ * | blocked | * +-----------+ * | * v * +----------------+ +------------+ +-----------+ +-------+ +---------+ * | set_base_spl() |->| *setlvlx() |->| splhigh() |->| sti() |->| swtch() | * +----------------+ +------------+ +-----------+ +-------+ +---------+ * * Calls made on Interrupt Stacks and Epilogue routines * * We use the switch_sp_and_call() assembly routine to switch our sp to the * interrupt stacks and then call the appropriate dispatch function. In the * case of interrupts which may block, softints and hardints, we always ensure * that we are still on the interrupt thread when we call the epilog routine. * This is not just important, it's necessary. If the interrupt thread blocked, * we won't return from our switch_sp_and_call() function and instead we'll go * through and set ourselves up to swtch() directly. * * New Interrupt Flow * ------------------ * * The apix module has its own interrupt path. This is done for various * reasons. The first is that rather than having global interrupt vectors, we * now have per-cpu vectors. * * The other substantial change is that the apix design does not use the TPR to * mask interrupts below the current level. In fact, except for one special * case, it does not use the TPR at all. Instead, it only uses the IF flag * (cli/sti) to either block all interrupts or allow any interrupts to come in. * The design is such that when interrupts are allowed to come in, if we are * currently servicing a higher priority interupt, the new interrupt is treated * as pending and serviced later. Specifically, in the pcplusmp module's * apic_intr_enter() the code masks interrupts at or below the current * IPL using the TPR before sending EOI, whereas the apix module's * apix_intr_enter() simply sends EOI. * * The one special case where the apix code uses the TPR is when it calls * through the apic_reg_ops function pointer apic_write_task_reg in * apix_init_intr() to initially mask all levels and then finally to enable all * levels. * * Recall that we come into the interrupt handler with all interrupts masked * by the IF flag. This is because we set up the handler using an * interrupt-gate which is defined architecturally to have cleared the IF flag * for us. * * +--------------+ +---------------------+ * | _interrupt() |--->| apix_do_interrupt() | * +--------------+ +---------------------+ * | * hard int? +----+--------+ softint? * | | (but no low-level looping) * +-----------+ | * | *setlvl() | | * +---------+ +-----------+ +----------------------------------+ * |apix_add_| check IPL | | * |pending_ |<-------------+------+----------------------+ | * |hardint()| low-level int| hi-level int| | * +---------+ v v | * | check IPL +-----------------+ +---------------+ | * +--+-----+ | apix_intr_ | | apix_hilevel_ | | * | | | thread_prolog() | | intr_prolog() | | * | return +-----------------+ +---------------+ | * | | | On intr | * | +------------+ | stack? +------------+ | * | | switch_sp_ | +---------| switch_sp_ | | * | | and_call() | | | and_call() | | * | +------------+ | +------------+ | * | | | | | * | +----------------+ +----------------+ | * | | apix_dispatch_ | | apix_dispatch_ | | * | | lowlevel() | | hilevel() | | * | +----------------+ +----------------+ | * | | | | * | v v | * | +-------------------------+ | * | |apix_dispatch_by_vector()|----+ | * | +-------------------------+ | | * | !XC_HI_PIL| | | | | * | +---+ +-------+ +---+ | | * | |sti| |*intr()| |cli| | | * | +---+ +-------+ +---+ | hi-level? | * | +---------------------------+----+ | * | v low-level? v | * | +----------------+ +----------------+ | * | | apix_intr_ | | apix_hilevel_ | | * | | thread_epilog()| | intr_epilog() | | * | +----------------+ +----------------+ | * | | | | * | v-----------------+--------------------------------+ | * | +------------+ | * | | *setlvlx() | +----------------------------------------------------+ * | +------------+ | * | | | +--------------------------------+ low * v v v------+ v | level * +------------------+ +------------------+ +-----------+ | pending? * | apix_do_pending_ |----->| apix_do_pending_ |----->| apix_do_ |--+ * | hilevel() | | hardint() | | softint() | | * +------------------+ +------------------+ +-----------+ return * | | | * | while pending | while pending | while pending * | hi-level | low-level | softint * | | | * +---------------+ +-----------------+ +-----------------+ * | apix_hilevel_ | | apix_intr_ | | apix_do_ | * | intr_prolog() | | thread_prolog() | | softint_prolog()| * +---------------+ +-----------------+ +-----------------+ * | On intr | | * | stack? +------------+ +------------+ +------------+ * +--------| switch_sp_ | | switch_sp_ | | switch_sp_ | * | | and_call() | | and_call() | | and_call() | * | +------------+ +------------+ +------------+ * | | | | * +------------------+ +------------------+ +------------------------+ * | apix_dispatch_ | | apix_dispatch_ | | apix_dispatch_softint()| * | pending_hilevel()| | pending_hardint()| +------------------------+ * +------------------+ +------------------+ | | | | * | | | | | | | | * | +----------------+ | +----------------+ | | | | * | | apix_hilevel_ | | | apix_intr_ | | | | | * | | intr_epilog() | | | thread_epilog()| | | | | * | +----------------+ | +----------------+ | | | | * | | | | | | | | * | +------------+ | +----------+ +------+ | | | * | | *setlvlx() | | |*setlvlx()| | | | | * | +------------+ | +----------+ | +----------+ | +---------+ * | | +---+ |av_ | +---+ |apix_do_ | * +---------------------------------+ |sti| |dispatch_ | |cli| |softint_ | * | apix_dispatch_pending_autovect()| +---+ |softvect()| +---+ |epilog() | * +---------------------------------+ +----------+ +---------+ * |!XC_HI_PIL | | | | * +---+ +-------+ +---+ +----------+ +-------+ * |sti| |*intr()| |cli| |apix_post_| |*intr()| * +---+ +-------+ +---+ |hardint() | +-------+ * +----------+ */ #include <sys/cpuvar.h> #include <sys/cpu_event.h> #include <sys/regset.h> #include <sys/psw.h> #include <sys/types.h> #include <sys/thread.h> #include <sys/systm.h> #include <sys/segments.h> #include <sys/pcb.h> #include <sys/trap.h> #include <sys/ftrace.h> #include <sys/traptrace.h> #include <sys/clock.h> #include <sys/panic.h> #include <sys/disp.h> #include <vm/seg_kp.h> #include <sys/stack.h> #include <sys/sysmacros.h> #include <sys/cmn_err.h> #include <sys/kstat.h> #include <sys/smp_impldefs.h> #include <sys/pool_pset.h> #include <sys/zone.h> #include <sys/bitmap.h> #include <sys/archsystm.h> #include <sys/machsystm.h> #include <sys/ontrap.h> #include <sys/x86_archext.h> #include <sys/promif.h> #include <sys/smt.h> #include <vm/hat_i86.h> #if defined(__xpv) #include <sys/hypervisor.h> #endif /* If these fail, then the padding numbers in machcpuvar.h are wrong. */ #if !defined(__xpv) #define MCOFF(member) \ (offsetof(cpu_t, cpu_m) + offsetof(struct machcpu, member)) CTASSERT(MCOFF(mcpu_pad) == MACHCPU_SIZE); CTASSERT(MCOFF(mcpu_pad2) == MMU_PAGESIZE); CTASSERT((MCOFF(mcpu_kpti) & 0xF) == 0); CTASSERT(((sizeof (struct kpti_frame)) & 0xF) == 0); CTASSERT((offsetof(struct kpti_frame, kf_tr_rsp) & 0xF) == 0); CTASSERT(MCOFF(mcpu_pad3) < 2 * MMU_PAGESIZE); #endif #if defined(__xpv) && defined(DEBUG) /* * This panic message is intended as an aid to interrupt debugging. * * The associated assertion tests the condition of enabling * events when events are already enabled. The implication * being that whatever code the programmer thought was * protected by having events disabled until the second * enable happened really wasn't protected at all .. */ int stistipanic = 1; /* controls the debug panic check */ const char *stistimsg = "stisti"; ulong_t laststi[NCPU]; /* * This variable tracks the last place events were disabled on each cpu * it assists in debugging when asserts that interrupts are enabled trip. */ ulong_t lastcli[NCPU]; #endif void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp); void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *) = do_interrupt; uintptr_t (*get_intr_handler)(int, short) = NULL; /* * Set cpu's base SPL level to the highest active interrupt level */ void set_base_spl(void) { struct cpu *cpu = CPU; uint16_t active = (uint16_t)cpu->cpu_intr_actv; cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active); } /* * Do all the work necessary to set up the cpu and thread structures * to dispatch a high-level interrupt. * * Returns 0 if we're -not- already on the high-level interrupt stack, * (and *must* switch to it), non-zero if we are already on that stack. * * Called with interrupts masked. * The 'pil' is already set to the appropriate level for rp->r_trapno. */ static int hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp) { struct machcpu *mcpu = &cpu->cpu_m; uint_t mask; hrtime_t intrtime; hrtime_t now = tsc_read(); ASSERT(pil > LOCK_LEVEL); if (pil == CBE_HIGH_PIL) { cpu->cpu_profile_pil = oldpil; if (USERMODE(rp->r_cs)) { cpu->cpu_profile_pc = 0; cpu->cpu_profile_upc = rp->r_pc; cpu->cpu_cpcprofile_pc = 0; cpu->cpu_cpcprofile_upc = rp->r_pc; } else { cpu->cpu_profile_pc = rp->r_pc; cpu->cpu_profile_upc = 0; cpu->cpu_cpcprofile_pc = rp->r_pc; cpu->cpu_cpcprofile_upc = 0; } } mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; if (mask != 0) { int nestpil; /* * We have interrupted another high-level interrupt. * Load starting timestamp, compute interval, update * cumulative counter. */ nestpil = bsrw_insn((uint16_t)mask); ASSERT(nestpil < pil); intrtime = now - mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)]; mcpu->intrstat[nestpil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; /* * Another high-level interrupt is active below this one, so * there is no need to check for an interrupt thread. That * will be done by the lowest priority high-level interrupt * active. */ } else { kthread_t *t = cpu->cpu_thread; /* * See if we are interrupting a low-level interrupt thread. * If so, account for its time slice only if its time stamp * is non-zero. */ if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) { intrtime = now - t->t_intr_start; mcpu->intrstat[t->t_pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; t->t_intr_start = 0; } } smt_begin_intr(pil); /* * Store starting timestamp in CPU structure for this PIL. */ mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now; ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); if (pil == 15) { /* * To support reentrant level 15 interrupts, we maintain a * recursion count in the top half of cpu_intr_actv. Only * when this count hits zero do we clear the PIL 15 bit from * the lower half of cpu_intr_actv. */ uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; (*refcntp)++; } mask = cpu->cpu_intr_actv; cpu->cpu_intr_actv |= (1 << pil); return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK); } /* * Does most of the work of returning from a high level interrupt. * * Returns 0 if there are no more high level interrupts (in which * case we must switch back to the interrupted thread stack) or * non-zero if there are more (in which case we should stay on it). * * Called with interrupts masked */ static int hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum) { struct machcpu *mcpu = &cpu->cpu_m; uint_t mask; hrtime_t intrtime; hrtime_t now = tsc_read(); ASSERT(mcpu->mcpu_pri == pil); cpu->cpu_stats.sys.intr[pil - 1]++; ASSERT(cpu->cpu_intr_actv & (1 << pil)); if (pil == 15) { /* * To support reentrant level 15 interrupts, we maintain a * recursion count in the top half of cpu_intr_actv. Only * when this count hits zero do we clear the PIL 15 bit from * the lower half of cpu_intr_actv. */ uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; ASSERT(*refcntp > 0); if (--(*refcntp) == 0) cpu->cpu_intr_actv &= ~(1 << pil); } else { cpu->cpu_intr_actv &= ~(1 << pil); } ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0); intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)]; mcpu->intrstat[pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; /* * Check for lower-pil nested high-level interrupt beneath * current one. If so, place a starting timestamp in its * pil_high_start entry. */ mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; if (mask != 0) { int nestpil; /* * find PIL of nested interrupt */ nestpil = bsrw_insn((uint16_t)mask); ASSERT(nestpil < pil); mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now; /* * (Another high-level interrupt is active below this one, * so there is no need to check for an interrupt * thread. That will be done by the lowest priority * high-level interrupt active.) */ } else { /* * Check to see if there is a low-level interrupt active. * If so, place a starting timestamp in the thread * structure. */ kthread_t *t = cpu->cpu_thread; if (t->t_flag & T_INTR_THREAD) t->t_intr_start = now; } smt_end_intr(); mcpu->mcpu_pri = oldpil; (void) (*setlvlx)(oldpil, vecnum); return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK); } /* * Set up the cpu, thread and interrupt thread structures for * executing an interrupt thread. The new stack pointer of the * interrupt thread (which *must* be switched to) is returned. */ static caddr_t intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil) { struct machcpu *mcpu = &cpu->cpu_m; kthread_t *t, *volatile it; hrtime_t now = tsc_read(); ASSERT(pil > 0); ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); cpu->cpu_intr_actv |= (1 << pil); /* * Get set to run an interrupt thread. * There should always be an interrupt thread, since we * allocate one for each level on each CPU. * * t_intr_start could be zero due to cpu_intr_swtch_enter. */ t = cpu->cpu_thread; if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { hrtime_t intrtime = now - t->t_intr_start; mcpu->intrstat[t->t_pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; t->t_intr_start = 0; } ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); t->t_sp = (uintptr_t)stackptr; /* mark stack in curthread for resume */ /* * unlink the interrupt thread off the cpu * * Note that the code in kcpc_overflow_intr -relies- on the * ordering of events here - in particular that t->t_lwp of * the interrupt thread is set to the pinned thread *before* * curthread is changed. */ it = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it->t_link; it->t_intr = t; it->t_lwp = t->t_lwp; /* * (threads on the interrupt thread free list could have state * preset to TS_ONPROC, but it helps in debugging if * they're TS_FREE.) */ it->t_state = TS_ONPROC; cpu->cpu_thread = it; /* new curthread on this cpu */ smt_begin_intr(pil); it->t_pil = (uchar_t)pil; it->t_pri = intr_pri + (pri_t)pil; it->t_intr_start = now; return (it->t_stk); } #ifdef DEBUG int intr_thread_cnt; #endif /* * Called with interrupts disabled */ static void intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil) { struct machcpu *mcpu = &cpu->cpu_m; kthread_t *t; kthread_t *it = cpu->cpu_thread; /* curthread */ uint_t pil, basespl; hrtime_t intrtime; hrtime_t now = tsc_read(); pil = it->t_pil; cpu->cpu_stats.sys.intr[pil - 1]++; ASSERT(it->t_intr_start != 0); intrtime = now - it->t_intr_start; mcpu->intrstat[pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; ASSERT(cpu->cpu_intr_actv & (1 << pil)); cpu->cpu_intr_actv &= ~(1 << pil); /* * If there is still an interrupted thread underneath this one * then the interrupt was never blocked and the return is * fairly simple. Otherwise it isn't. */ if ((t = it->t_intr) == NULL) { /* * The interrupted thread is no longer pinned underneath * the interrupt thread. This means the interrupt must * have blocked, and the interrupted thread has been * unpinned, and has probably been running around the * system for a while. * * Since there is no longer a thread under this one, put * this interrupt thread back on the CPU's free list and * resume the idle thread which will dispatch the next * thread to run. */ #ifdef DEBUG intr_thread_cnt++; #endif cpu->cpu_stats.sys.intrblk++; /* * Set CPU's base SPL based on active interrupts bitmask */ set_base_spl(); basespl = cpu->cpu_base_spl; mcpu->mcpu_pri = basespl; (*setlvlx)(basespl, vec); (void) splhigh(); sti(); it->t_state = TS_FREE; /* * Return interrupt thread to pool */ it->t_link = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it; swtch(); panic("intr_thread_epilog: swtch returned"); /*NOTREACHED*/ } /* * Return interrupt thread to the pool */ it->t_link = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it; it->t_state = TS_FREE; basespl = cpu->cpu_base_spl; pil = MAX(oldpil, basespl); mcpu->mcpu_pri = pil; (*setlvlx)(pil, vec); t->t_intr_start = now; smt_end_intr(); cpu->cpu_thread = t; } /* * intr_get_time() is a resource for interrupt handlers to determine how * much time has been spent handling the current interrupt. Such a function * is needed because higher level interrupts can arrive during the * processing of an interrupt. intr_get_time() only returns time spent in the * current interrupt handler. * * The caller must be calling from an interrupt handler running at a pil * below or at lock level. Timings are not provided for high-level * interrupts. * * The first time intr_get_time() is called while handling an interrupt, * it returns the time since the interrupt handler was invoked. Subsequent * calls will return the time since the prior call to intr_get_time(). Time * is returned as ticks. Use scalehrtimef() to convert ticks to nsec. * * Theory Of Intrstat[][]: * * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two * uint64_ts per pil. * * intrstat[pil][0] is a cumulative count of the number of ticks spent * handling all interrupts at the specified pil on this CPU. It is * exported via kstats to the user. * * intrstat[pil][1] is always a count of ticks less than or equal to the * value in [0]. The difference between [1] and [0] is the value returned * by a call to intr_get_time(). At the start of interrupt processing, * [0] and [1] will be equal (or nearly so). As the interrupt consumes * time, [0] will increase, but [1] will remain the same. A call to * intr_get_time() will return the difference, then update [1] to be the * same as [0]. Future calls will return the time since the last call. * Finally, when the interrupt completes, [1] is updated to the same as [0]. * * Implementation: * * intr_get_time() works much like a higher level interrupt arriving. It * "checkpoints" the timing information by incrementing intrstat[pil][0] * to include elapsed running time, and by setting t_intr_start to rdtsc. * It then sets the return value to intrstat[pil][0] - intrstat[pil][1], * and updates intrstat[pil][1] to be the same as the new value of * intrstat[pil][0]. * * In the normal handling of interrupts, after an interrupt handler returns * and the code in intr_thread() updates intrstat[pil][0], it then sets * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1], * the timings are reset, i.e. intr_get_time() will return [0] - [1] which * is 0. * * Whenever interrupts arrive on a CPU which is handling a lower pil * interrupt, they update the lower pil's [0] to show time spent in the * handler that they've interrupted. This results in a growing discrepancy * between [0] and [1], which is returned the next time intr_get_time() is * called. Time spent in the higher-pil interrupt will not be returned in * the next intr_get_time() call from the original interrupt, because * the higher-pil interrupt's time is accumulated in intrstat[higherpil][]. */ uint64_t intr_get_time(void) { struct cpu *cpu; struct machcpu *mcpu; kthread_t *t; uint64_t time, delta, ret; uint_t pil; cli(); cpu = CPU; mcpu = &cpu->cpu_m; t = cpu->cpu_thread; pil = t->t_pil; ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0); ASSERT(t->t_flag & T_INTR_THREAD); ASSERT(pil != 0); ASSERT(t->t_intr_start != 0); time = tsc_read(); delta = time - t->t_intr_start; t->t_intr_start = time; time = mcpu->intrstat[pil][0] + delta; ret = time - mcpu->intrstat[pil][1]; mcpu->intrstat[pil][0] = time; mcpu->intrstat[pil][1] = time; cpu->cpu_intracct[cpu->cpu_mstate] += delta; sti(); return (ret); } static caddr_t dosoftint_prolog( struct cpu *cpu, caddr_t stackptr, uint32_t st_pending, uint_t oldpil) { kthread_t *t, *volatile it; struct machcpu *mcpu = &cpu->cpu_m; uint_t pil; hrtime_t now; top: ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending); pil = bsrw_insn((uint16_t)st_pending); if (pil <= oldpil || pil <= cpu->cpu_base_spl) return (0); /* * XX64 Sigh. * * This is a transliteration of the i386 assembler code for * soft interrupts. One question is "why does this need * to be atomic?" One possible race is -other- processors * posting soft interrupts to us in set_pending() i.e. the * CPU might get preempted just after the address computation, * but just before the atomic transaction, so another CPU would * actually set the original CPU's st_pending bit. However, * it looks like it would be simpler to disable preemption there. * Are there other races for which preemption control doesn't work? * * The i386 assembler version -also- checks to see if the bit * being cleared was actually set; if it wasn't, it rechecks * for more. This seems a bit strange, as the only code that * ever clears the bit is -this- code running with interrupts * disabled on -this- CPU. This code would probably be cheaper: * * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending, * ~(1 << pil)); * * and t->t_preempt--/++ around set_pending() even cheaper, * but at this point, correctness is critical, so we slavishly * emulate the i386 port. */ if (atomic_btr32((uint32_t *) &mcpu->mcpu_softinfo.st_pending, pil) == 0) { st_pending = mcpu->mcpu_softinfo.st_pending; goto top; } mcpu->mcpu_pri = pil; (*setspl)(pil); now = tsc_read(); /* * Get set to run interrupt thread. * There should always be an interrupt thread since we * allocate one for each level on the CPU. */ it = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it->t_link; /* t_intr_start could be zero due to cpu_intr_swtch_enter. */ t = cpu->cpu_thread; if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { hrtime_t intrtime = now - t->t_intr_start; mcpu->intrstat[pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; t->t_intr_start = 0; } /* * Note that the code in kcpc_overflow_intr -relies- on the * ordering of events here - in particular that t->t_lwp of * the interrupt thread is set to the pinned thread *before* * curthread is changed. */ it->t_lwp = t->t_lwp; it->t_state = TS_ONPROC; /* * Push interrupted thread onto list from new thread. * Set the new thread as the current one. * Set interrupted thread's T_SP because if it is the idle thread, * resume() may use that stack between threads. */ ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); t->t_sp = (uintptr_t)stackptr; it->t_intr = t; cpu->cpu_thread = it; smt_begin_intr(pil); /* * Set bit for this pil in CPU's interrupt active bitmask. */ ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); cpu->cpu_intr_actv |= (1 << pil); /* * Initialize thread priority level from intr_pri */ it->t_pil = (uchar_t)pil; it->t_pri = (pri_t)pil + intr_pri; it->t_intr_start = now; return (it->t_stk); } static void dosoftint_epilog(struct cpu *cpu, uint_t oldpil) { struct machcpu *mcpu = &cpu->cpu_m; kthread_t *t, *it; uint_t pil, basespl; hrtime_t intrtime; hrtime_t now = tsc_read(); it = cpu->cpu_thread; pil = it->t_pil; cpu->cpu_stats.sys.intr[pil - 1]++; ASSERT(cpu->cpu_intr_actv & (1 << pil)); cpu->cpu_intr_actv &= ~(1 << pil); intrtime = now - it->t_intr_start; mcpu->intrstat[pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; /* * If there is still an interrupted thread underneath this one * then the interrupt was never blocked and the return is * fairly simple. Otherwise it isn't. */ if ((t = it->t_intr) == NULL) { /* * Put thread back on the interrupt thread list. * This was an interrupt thread, so set CPU's base SPL. */ set_base_spl(); it->t_state = TS_FREE; it->t_link = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it; (void) splhigh(); sti(); swtch(); /*NOTREACHED*/ panic("dosoftint_epilog: swtch returned"); } it->t_link = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it; it->t_state = TS_FREE; smt_end_intr(); cpu->cpu_thread = t; if (t->t_flag & T_INTR_THREAD) t->t_intr_start = now; basespl = cpu->cpu_base_spl; pil = MAX(oldpil, basespl); mcpu->mcpu_pri = pil; (*setspl)(pil); } /* * Make the interrupted thread 'to' be runnable. * * Since t->t_sp has already been saved, t->t_pc is all * that needs to be set in this function. * * Returns the interrupt level of the interrupt thread. */ int intr_passivate( kthread_t *it, /* interrupt thread */ kthread_t *t) /* interrupted thread */ { extern void _sys_rtt(); ASSERT(it->t_flag & T_INTR_THREAD); ASSERT(SA(t->t_sp) == t->t_sp); t->t_pc = (uintptr_t)_sys_rtt; return (it->t_pil); } /* * Create interrupt kstats for this CPU. */ void cpu_create_intrstat(cpu_t *cp) { int i; kstat_t *intr_ksp; kstat_named_t *knp; char name[KSTAT_STRLEN]; zoneid_t zoneid; ASSERT(MUTEX_HELD(&cpu_lock)); if (pool_pset_enabled()) zoneid = GLOBAL_ZONEID; else zoneid = ALL_ZONES; intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc", KSTAT_TYPE_NAMED, PIL_MAX * 2, 0, zoneid); /* * Initialize each PIL's named kstat */ if (intr_ksp != NULL) { intr_ksp->ks_update = cpu_kstat_intrstat_update; knp = (kstat_named_t *)intr_ksp->ks_data; intr_ksp->ks_private = cp; for (i = 0; i < PIL_MAX; i++) { (void) snprintf(name, KSTAT_STRLEN, "level-%d-time", i + 1); kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64); (void) snprintf(name, KSTAT_STRLEN, "level-%d-count", i + 1); kstat_named_init(&knp[(i * 2) + 1], name, KSTAT_DATA_UINT64); } kstat_install(intr_ksp); } } /* * Delete interrupt kstats for this CPU. */ void cpu_delete_intrstat(cpu_t *cp) { kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES); } /* * Convert interrupt statistics from CPU ticks to nanoseconds and * update kstat. */ int cpu_kstat_intrstat_update(kstat_t *ksp, int rw) { kstat_named_t *knp = ksp->ks_data; cpu_t *cpup = (cpu_t *)ksp->ks_private; int i; hrtime_t hrt; if (rw == KSTAT_WRITE) return (EACCES); for (i = 0; i < PIL_MAX; i++) { hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0]; scalehrtimef(&hrt); knp[i * 2].value.ui64 = (uint64_t)hrt; knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i]; } return (0); } /* * An interrupt thread is ending a time slice, so compute the interval it * ran for and update the statistic for its PIL. */ void cpu_intr_swtch_enter(kthread_id_t t) { uint64_t interval; uint64_t start; cpu_t *cpu; ASSERT((t->t_flag & T_INTR_THREAD) != 0); ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL); /* * We could be here with a zero timestamp. This could happen if: * an interrupt thread which no longer has a pinned thread underneath * it (i.e. it blocked at some point in its past) has finished running * its handler. intr_thread() updated the interrupt statistic for its * PIL and zeroed its timestamp. Since there was no pinned thread to * return to, swtch() gets called and we end up here. * * Note that we use atomic ops below (atomic_cas_64 and * atomic_add_64), which we don't use in the functions above, * because we're not called with interrupts blocked, but the * epilog/prolog functions are. */ if (t->t_intr_start) { do { start = t->t_intr_start; interval = tsc_read() - start; } while (atomic_cas_64(&t->t_intr_start, start, 0) != start); cpu = CPU; cpu->cpu_m.intrstat[t->t_pil][0] += interval; atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate], interval); } else ASSERT(t->t_intr == NULL); } /* * An interrupt thread is returning from swtch(). Place a starting timestamp * in its thread structure. */ void cpu_intr_swtch_exit(kthread_id_t t) { uint64_t ts; ASSERT((t->t_flag & T_INTR_THREAD) != 0); ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL); do { ts = t->t_intr_start; } while (atomic_cas_64(&t->t_intr_start, ts, tsc_read()) != ts); } /* * Dispatch a hilevel interrupt (one above LOCK_LEVEL) */ /*ARGSUSED*/ static void dispatch_hilevel(uint_t vector, uint_t arg2) { sti(); av_dispatch_autovect(vector); cli(); } /* * Dispatch a soft interrupt */ /*ARGSUSED*/ static void dispatch_softint(uint_t oldpil, uint_t arg2) { struct cpu *cpu = CPU; sti(); av_dispatch_softvect((int)cpu->cpu_thread->t_pil); cli(); /* * Must run softint_epilog() on the interrupt thread stack, since * there may not be a return from it if the interrupt thread blocked. */ dosoftint_epilog(cpu, oldpil); } /* * Dispatch a normal interrupt */ static void dispatch_hardint(uint_t vector, uint_t oldipl) { struct cpu *cpu = CPU; sti(); av_dispatch_autovect(vector); cli(); /* * Must run intr_thread_epilog() on the interrupt thread stack, since * there may not be a return from it if the interrupt thread blocked. */ intr_thread_epilog(cpu, vector, oldipl); } /* * Deliver any softints the current interrupt priority allows. * Called with interrupts disabled. */ void dosoftint(struct regs *regs) { struct cpu *cpu = CPU; int oldipl; caddr_t newsp; while (cpu->cpu_softinfo.st_pending) { oldipl = cpu->cpu_pri; newsp = dosoftint_prolog(cpu, (caddr_t)regs, cpu->cpu_softinfo.st_pending, oldipl); /* * If returned stack pointer is NULL, priority is too high * to run any of the pending softints now. * Break out and they will be run later. */ if (newsp == NULL) break; switch_sp_and_call(newsp, dispatch_softint, oldipl, 0); } } /* * Interrupt service routine, called with interrupts disabled. */ /*ARGSUSED*/ void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp) { struct cpu *cpu = CPU; int newipl, oldipl = cpu->cpu_pri; uint_t vector; caddr_t newsp; #ifdef TRAPTRACE ttp->ttr_marker = TT_INTERRUPT; ttp->ttr_ipl = 0xff; ttp->ttr_pri = oldipl; ttp->ttr_spl = cpu->cpu_base_spl; ttp->ttr_vector = 0xff; #endif /* TRAPTRACE */ cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR); ++*(uint16_t *)&cpu->cpu_m.mcpu_istamp; /* * If it's a softint go do it now. */ if (rp->r_trapno == T_SOFTINT) { dosoftint(rp); ASSERT(!interrupts_enabled()); return; } /* * Raise the interrupt priority. */ newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno); #ifdef TRAPTRACE ttp->ttr_ipl = newipl; #endif /* TRAPTRACE */ /* * Bail if it is a spurious interrupt */ if (newipl == -1) return; cpu->cpu_pri = newipl; vector = rp->r_trapno; #ifdef TRAPTRACE ttp->ttr_vector = vector; #endif /* TRAPTRACE */ if (newipl > LOCK_LEVEL) { /* * High priority interrupts run on this cpu's interrupt stack. */ if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) { newsp = cpu->cpu_intr_stack; switch_sp_and_call(newsp, dispatch_hilevel, vector, 0); } else { /* already on the interrupt stack */ dispatch_hilevel(vector, 0); } (void) hilevel_intr_epilog(cpu, newipl, oldipl, vector); } else { /* * Run this interrupt in a separate thread. */ newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl); switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl); } #if !defined(__xpv) /* * Deliver any pending soft interrupts. */ if (cpu->cpu_softinfo.st_pending) dosoftint(rp); #endif /* !__xpv */ } /* * Common tasks always done by _sys_rtt, called with interrupts disabled. * Returns 1 if returning to userland, 0 if returning to system mode. */ int sys_rtt_common(struct regs *rp) { kthread_t *tp; extern void mutex_exit_critical_start(); extern long mutex_exit_critical_size; extern void mutex_owner_running_critical_start(); extern long mutex_owner_running_critical_size; loop: /* * Check if returning to user */ tp = CPU->cpu_thread; if (USERMODE(rp->r_cs)) { pcb_t *pcb; /* * Check if AST pending. */ if (tp->t_astflag) { /* * Let trap() handle the AST */ sti(); rp->r_trapno = T_AST; trap(rp, (caddr_t)0, CPU->cpu_id); cli(); goto loop; } pcb = &tp->t_lwp->lwp_pcb; /* * Check to see if we need to initialize the FPU for this * thread. This should be an uncommon occurrence, but may happen * in the case where the system creates an lwp through an * abnormal path such as the agent lwp. Make sure that we still * happen to have the FPU in a good state. */ if ((pcb->pcb_fpu.fpu_flags & FPU_EN) == 0) { kpreempt_disable(); fp_seed(); kpreempt_enable(); PCB_SET_UPDATE_FPU(pcb); } /* * We are done if segment registers do not need updating. */ if (!PCB_NEED_UPDATE(pcb)) return (1); if (PCB_NEED_UPDATE_SEGS(pcb) && update_sregs(rp, tp->t_lwp)) { /* * 1 or more of the selectors is bad. * Deliver a SIGSEGV. */ proc_t *p = ttoproc(tp); sti(); mutex_enter(&p->p_lock); tp->t_lwp->lwp_cursig = SIGSEGV; mutex_exit(&p->p_lock); psig(); tp->t_sig_check = 1; cli(); } PCB_CLEAR_UPDATE_SEGS(pcb); if (PCB_NEED_UPDATE_FPU(pcb)) { fprestore_ctxt(&pcb->pcb_fpu); } PCB_CLEAR_UPDATE_FPU(pcb); ASSERT0(PCB_NEED_UPDATE(pcb)); return (1); } #if !defined(__xpv) /* * Assert that we're not trying to return into the syscall return * trampolines. Things will go baaaaad if we try to do that. * * Note that none of these run with interrupts on, so this should * never happen (even in the sysexit case the STI doesn't take effect * until after sysexit finishes). */ extern void tr_sysc_ret_start(); extern void tr_sysc_ret_end(); ASSERT(!(rp->r_pc >= (uintptr_t)tr_sysc_ret_start && rp->r_pc <= (uintptr_t)tr_sysc_ret_end)); #endif /* * Here if we are returning to supervisor mode. * Check for a kernel preemption request. */ if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) { /* * Do nothing if already in kpreempt */ if (!tp->t_preempt_lk) { tp->t_preempt_lk = 1; sti(); kpreempt(1); /* asynchronous kpreempt call */ cli(); tp->t_preempt_lk = 0; } } /* * If we interrupted the mutex_exit() critical region we must * reset the PC back to the beginning to prevent missed wakeups * See the comments in mutex_exit() for details. */ if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start < mutex_exit_critical_size) { rp->r_pc = (greg_t)mutex_exit_critical_start; } /* * If we interrupted the mutex_owner_running() critical region we * must reset the PC back to the beginning to prevent dereferencing * of a freed thread pointer. See the comments in mutex_owner_running * for details. */ if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_owner_running_critical_start < mutex_owner_running_critical_size) { rp->r_pc = (greg_t)mutex_owner_running_critical_start; } return (0); } void send_dirint(int cpuid, int int_level) { (*send_dirintf)(cpuid, int_level); } #define IS_FAKE_SOFTINT(flag, newpri) \ (((flag) & PS_IE) && \ (((*get_pending_spl)() > (newpri)) || \ bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > (newpri))) /* * do_splx routine, takes new ipl to set * returns the old ipl. * We are careful not to set priority lower than CPU->cpu_base_pri, * even though it seems we're raising the priority, it could be set * higher at any time by an interrupt routine, so we must block interrupts * and look at CPU->cpu_base_pri */ int do_splx(int newpri) { ulong_t flag; cpu_t *cpu; int curpri, basepri; flag = intr_clear(); cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */ curpri = cpu->cpu_m.mcpu_pri; basepri = cpu->cpu_base_spl; if (newpri < basepri) newpri = basepri; cpu->cpu_m.mcpu_pri = newpri; (*setspl)(newpri); /* * If we are going to reenable interrupts see if new priority level * allows pending softint delivery. */ if (IS_FAKE_SOFTINT(flag, newpri)) fakesoftint(); ASSERT(!interrupts_enabled()); intr_restore(flag); return (curpri); } /* * Common spl raise routine, takes new ipl to set * returns the old ipl, will not lower ipl. */ int splr(int newpri) { ulong_t flag; cpu_t *cpu; int curpri, basepri; flag = intr_clear(); cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */ curpri = cpu->cpu_m.mcpu_pri; /* * Only do something if new priority is larger */ if (newpri > curpri) { basepri = cpu->cpu_base_spl; if (newpri < basepri) newpri = basepri; cpu->cpu_m.mcpu_pri = newpri; (*setspl)(newpri); /* * See if new priority level allows pending softint delivery */ if (IS_FAKE_SOFTINT(flag, newpri)) fakesoftint(); } intr_restore(flag); return (curpri); } int getpil(void) { return (CPU->cpu_m.mcpu_pri); } int spl_xcall(void) { return (splr(ipltospl(XCALL_PIL))); } int interrupts_enabled(void) { ulong_t flag; flag = getflags(); return ((flag & PS_IE) == PS_IE); } #ifdef DEBUG void assert_ints_enabled(void) { ASSERT(!interrupts_unleashed || interrupts_enabled()); } #endif /* DEBUG */
