/* $NetBSD: kern_softint.c,v 1.76 2024/03/01 04:32:38 mrg Exp $ */ /*- * Copyright (c) 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Andrew Doran. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Generic software interrupt framework. * * Overview * * The soft interrupt framework provides a mechanism to schedule a * low priority callback that runs with thread context. It allows * for dynamic registration of software interrupts, and for fair * queueing and prioritization of those interrupts. The callbacks * can be scheduled to run from nearly any point in the kernel: by * code running with thread context, by code running from a * hardware interrupt handler, and at any interrupt priority * level. * * Priority levels * * Since soft interrupt dispatch can be tied to the underlying * architecture's interrupt dispatch code, it can be limited * both by the capabilities of the hardware and the capabilities * of the interrupt dispatch code itself. The number of priority * levels is restricted to four. In order of priority (lowest to * highest) the levels are: clock, bio, net, serial. * * The names are symbolic and in isolation do not have any direct * connection with a particular kind of device activity: they are * only meant as a guide. * * The four priority levels map directly to scheduler priority * levels, and where the architecture implements 'fast' software * interrupts, they also map onto interrupt priorities. The * interrupt priorities are intended to be hidden from machine * independent code, which should use thread-safe mechanisms to * synchronize with software interrupts (for example: mutexes). * * Capabilities * * Software interrupts run with limited machine context. In * particular, they do not posess any address space context. They * should not try to operate on user space addresses, or to use * virtual memory facilities other than those noted as interrupt * safe. * * Unlike hardware interrupts, software interrupts do have thread * context. They may block on synchronization objects, sleep, and * resume execution at a later time. * * Since software interrupts are a limited resource and run with * higher priority than most other LWPs in the system, all * block-and-resume activity by a software interrupt must be kept * short to allow further processing at that level to continue. By * extension, code running with process context must take care to * ensure that any lock that may be taken from a software interrupt * can not be held for more than a short period of time. * * The kernel does not allow software interrupts to use facilities * or perform actions that may block for a significant amount of * time. This means that it's not valid for a software interrupt * to sleep on condition variables or wait for resources to become * available (for example, memory). * * Per-CPU operation * * If a soft interrupt is triggered on a CPU, it can only be * dispatched on the same CPU. Each LWP dedicated to handling a * soft interrupt is bound to its home CPU, so if the LWP blocks * and needs to run again, it can only run there. Nearly all data * structures used to manage software interrupts are per-CPU. * * The per-CPU requirement is intended to reduce "ping-pong" of * cache lines between CPUs: lines occupied by data structures * used to manage the soft interrupts, and lines occupied by data * items being passed down to the soft interrupt. As a positive * side effect, this also means that the soft interrupt dispatch * code does not need to to use spinlocks to synchronize. * * Generic implementation * * A generic, low performance implementation is provided that * works across all architectures, with no machine-dependent * modifications needed. This implementation uses the scheduler, * and so has a number of restrictions: * * 1) The software interrupts are not currently preemptive, so * must wait for the currently executing LWP to yield the CPU. * This can introduce latency. * * 2) An expensive context switch is required for a software * interrupt to be handled. * * 'Fast' software interrupts * * If an architectures defines __HAVE_FAST_SOFTINTS, it implements * the fast mechanism. Threads running either in the kernel or in * userspace will be interrupted, but will not be preempted. When * the soft interrupt completes execution, the interrupted LWP * is resumed. Interrupt dispatch code must provide the minimum * level of context necessary for the soft interrupt to block and * be resumed at a later time. The machine-dependent dispatch * path looks something like the following: * * softintr() * { * go to IPL_HIGH if necessary for switch; * save any necessary registers in a format that can be * restored by cpu_switchto if the softint blocks; * arrange for cpu_switchto() to restore into the * trampoline function; * identify LWP to handle this interrupt; * switch to the LWP's stack; * switch register stacks, if necessary; * assign new value of curlwp; * call MI softint_dispatch, passing old curlwp and IPL * to execute interrupt at; * switch back to old stack; * switch back to old register stack, if necessary; * restore curlwp; * return to interrupted LWP; * } * * If the soft interrupt blocks, a trampoline function is returned * to in the context of the interrupted LWP, as arranged for by * softint(): * * softint_ret() * { * unlock soft interrupt LWP; * resume interrupt processing, likely returning to * interrupted LWP or dispatching another, different * interrupt; * } * * Once the soft interrupt has fired (and even if it has blocked), * no further soft interrupts at that level will be triggered by * MI code until the soft interrupt handler has ceased execution. * If a soft interrupt handler blocks and is resumed, it resumes * execution as a normal LWP (kthread) and gains VM context. Only * when it has completed and is ready to fire again will it * interrupt other threads. */ #include <sys/cdefs.h> __KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.76 2024/03/01 04:32:38 mrg Exp $"); #include <sys/param.h> #include <sys/proc.h> #include <sys/intr.h> #include <sys/ipi.h> #include <sys/lock.h> #include <sys/mutex.h> #include <sys/kernel.h> #include <sys/kthread.h> #include <sys/evcnt.h> #include <sys/cpu.h> #include <sys/xcall.h> #include <sys/psref.h> #include <sys/sdt.h> #include <uvm/uvm_extern.h> /* This could overlap with signal info in struct lwp. */ typedef struct softint { SIMPLEQ_HEAD(, softhand) si_q; struct lwp *si_lwp; struct cpu_info *si_cpu; uintptr_t si_machdep; struct evcnt si_evcnt; struct evcnt si_evcnt_block; volatile int si_active; int si_ipl; char si_name[8]; char si_name_block[8+6]; } softint_t; typedef struct softhand { SIMPLEQ_ENTRY(softhand) sh_q; void (*sh_func)(void *); void *sh_arg; softint_t *sh_isr; u_int sh_flags; u_int sh_ipi_id; } softhand_t; typedef struct softcpu { struct cpu_info *sc_cpu; softint_t sc_int[SOFTINT_COUNT]; softhand_t sc_hand[1]; } softcpu_t; static void softint_thread(void *); u_int softint_bytes = 32768; u_int softint_timing; static u_int softint_max; static kmutex_t softint_lock; SDT_PROBE_DEFINE4(sdt, kernel, softint, establish, "void *"/*sih*/, "void (*)(void *)"/*func*/, "void *"/*arg*/, "unsigned"/*flags*/); SDT_PROBE_DEFINE1(sdt, kernel, softint, disestablish, "void *"/*sih*/); SDT_PROBE_DEFINE2(sdt, kernel, softint, schedule, "void *"/*sih*/, "struct cpu_info *"/*ci*/); SDT_PROBE_DEFINE4(sdt, kernel, softint, entry, "void *"/*sih*/, "void (*)(void *)"/*func*/, "void *"/*arg*/, "unsigned"/*flags*/); SDT_PROBE_DEFINE4(sdt, kernel, softint, return, "void *"/*sih*/, "void (*)(void *)"/*func*/, "void *"/*arg*/, "unsigned"/*flags*/); /* * softint_init_isr: * * Initialize a single interrupt level for a single CPU. */ static void softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level, int ipl) { struct cpu_info *ci; softint_t *si; int error; si = &sc->sc_int[level]; ci = sc->sc_cpu; si->si_cpu = ci; SIMPLEQ_INIT(&si->si_q); error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR | KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp, "soft%s/%u", desc, ci->ci_index); if (error != 0) panic("softint_init_isr: error %d", error); snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc, ci->ci_index); evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL, "softint", si->si_name); snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u", desc, ci->ci_index); evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL, "softint", si->si_name_block); si->si_ipl = ipl; si->si_lwp->l_private = si; softint_init_md(si->si_lwp, level, &si->si_machdep); } /* * softint_init: * * Initialize per-CPU data structures. Called from mi_cpu_attach(). */ void softint_init(struct cpu_info *ci) { static struct cpu_info *first; softcpu_t *sc, *scfirst; softhand_t *sh, *shmax; if (first == NULL) { /* Boot CPU. */ first = ci; mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE); softint_bytes = round_page(softint_bytes); softint_max = (softint_bytes - sizeof(softcpu_t)) / sizeof(softhand_t); } /* Use uvm_km(9) for persistent, page-aligned allocation. */ sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); if (sc == NULL) panic("softint_init_cpu: cannot allocate memory"); ci->ci_data.cpu_softcpu = sc; ci->ci_data.cpu_softints = 0; sc->sc_cpu = ci; softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET, IPL_SOFTNET); softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO, IPL_SOFTBIO); softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK, IPL_SOFTCLOCK); softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL, IPL_SOFTSERIAL); if (first != ci) { mutex_enter(&softint_lock); scfirst = first->ci_data.cpu_softcpu; sh = sc->sc_hand; memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max); /* Update pointers for this CPU. */ for (shmax = sh + softint_max; sh < shmax; sh++) { if (sh->sh_func == NULL) continue; sh->sh_isr = &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK]; } mutex_exit(&softint_lock); } } /* * softint_establish: * * Register a software interrupt handler. */ void * softint_establish(u_int flags, void (*func)(void *), void *arg) { CPU_INFO_ITERATOR cii; struct cpu_info *ci; softcpu_t *sc; softhand_t *sh; u_int level, index; u_int ipi_id = 0; void *sih; level = (flags & SOFTINT_LVLMASK); KASSERT(level < SOFTINT_COUNT); KASSERT((flags & SOFTINT_IMPMASK) == 0); mutex_enter(&softint_lock); /* Find a free slot. */ sc = curcpu()->ci_data.cpu_softcpu; for (index = 1; index < softint_max; index++) { if (sc->sc_hand[index].sh_func == NULL) break; } if (index == softint_max) { mutex_exit(&softint_lock); printf("WARNING: softint_establish: table full, " "increase softint_bytes\n"); return NULL; } sih = (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc); if (flags & SOFTINT_RCPU) { if ((ipi_id = ipi_register(softint_schedule, sih)) == 0) { mutex_exit(&softint_lock); return NULL; } } /* Set up the handler on each CPU. */ if (ncpu < 2) { /* XXX hack for machines with no CPU_INFO_FOREACH() early on */ sc = curcpu()->ci_data.cpu_softcpu; sh = &sc->sc_hand[index]; sh->sh_isr = &sc->sc_int[level]; sh->sh_func = func; sh->sh_arg = arg; sh->sh_flags = flags; sh->sh_ipi_id = ipi_id; } else for (CPU_INFO_FOREACH(cii, ci)) { sc = ci->ci_data.cpu_softcpu; sh = &sc->sc_hand[index]; sh->sh_isr = &sc->sc_int[level]; sh->sh_func = func; sh->sh_arg = arg; sh->sh_flags = flags; sh->sh_ipi_id = ipi_id; } mutex_exit(&softint_lock); SDT_PROBE4(sdt, kernel, softint, establish, sih, func, arg, flags); return sih; } /* * softint_disestablish: * * Unregister a software interrupt handler. The soft interrupt could * still be active at this point, but the caller commits not to try * and trigger it again once this call is made. The caller must not * hold any locks that could be taken from soft interrupt context, * because we will wait for the softint to complete if it's still * running. */ void softint_disestablish(void *arg) { CPU_INFO_ITERATOR cii; struct cpu_info *ci; softcpu_t *sc; softhand_t *sh; uintptr_t offset; offset = (uintptr_t)arg; KASSERT(offset != 0); KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u", offset, softint_bytes); /* * Unregister IPI handler if there is any. Note: there is no need * to disable preemption here - ID is stable. */ sc = curcpu()->ci_data.cpu_softcpu; sh = (softhand_t *)((uint8_t *)sc + offset); if (sh->sh_ipi_id) { ipi_unregister(sh->sh_ipi_id); } /* * Run a dummy softint at the same level on all CPUs and wait for * completion, to make sure this softint is no longer running * anywhere. */ xc_barrier(XC_HIGHPRI_IPL(sh->sh_isr->si_ipl)); /* * Notify dtrace probe when the old softint can't be running * any more, but before it can be recycled for a new softint. */ SDT_PROBE1(sdt, kernel, softint, disestablish, arg); /* Clear the handler on each CPU. */ mutex_enter(&softint_lock); for (CPU_INFO_FOREACH(cii, ci)) { sc = ci->ci_data.cpu_softcpu; sh = (softhand_t *)((uint8_t *)sc + offset); KASSERT(sh->sh_func != NULL); sh->sh_func = NULL; } mutex_exit(&softint_lock); } /* * softint_schedule: * * Trigger a software interrupt. Must be called from a hardware * interrupt handler, or with preemption disabled (since we are * using the value of curcpu()). */ void softint_schedule(void *arg) { softhand_t *sh; softint_t *si; uintptr_t offset; int s; SDT_PROBE2(sdt, kernel, softint, schedule, arg, /*ci*/NULL); /* * If this assert fires, rather than disabling preemption explicitly * to make it stop, consider that you are probably using a softint * when you don't need to. */ KASSERT(kpreempt_disabled()); /* Find the handler record for this CPU. */ offset = (uintptr_t)arg; KASSERT(offset != 0); KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u", offset, softint_bytes); sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset); /* If it's already pending there's nothing to do. */ if ((sh->sh_flags & SOFTINT_PENDING) != 0) { return; } /* * Enqueue the handler into the LWP's pending list. * If the LWP is completely idle, then make it run. */ s = splhigh(); if ((sh->sh_flags & SOFTINT_PENDING) == 0) { si = sh->sh_isr; sh->sh_flags |= SOFTINT_PENDING; SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q); if (si->si_active == 0) { si->si_active = 1; softint_trigger(si->si_machdep); } } splx(s); } /* * softint_schedule_cpu: * * Trigger a software interrupt on a target CPU. This invokes * softint_schedule() for the local CPU or send an IPI to invoke * this routine on the remote CPU. Preemption must be disabled. */ void softint_schedule_cpu(void *arg, struct cpu_info *ci) { KASSERT(kpreempt_disabled()); if (curcpu() != ci) { const softcpu_t *sc = ci->ci_data.cpu_softcpu; const uintptr_t offset = (uintptr_t)arg; const softhand_t *sh; SDT_PROBE2(sdt, kernel, softint, schedule, arg, ci); sh = (const softhand_t *)((const uint8_t *)sc + offset); KASSERT((sh->sh_flags & SOFTINT_RCPU) != 0); ipi_trigger(sh->sh_ipi_id, ci); return; } /* Just a local CPU. */ softint_schedule(arg); } /* * softint_execute: * * Invoke handlers for the specified soft interrupt. * Must be entered at splhigh. Will drop the priority * to the level specified, but returns back at splhigh. */ static inline void softint_execute(lwp_t *l, int s) { softint_t *si = l->l_private; softhand_t *sh; KASSERT(si->si_lwp == curlwp); KASSERT(si->si_cpu == curcpu()); KASSERT(si->si_lwp->l_wchan == NULL); KASSERT(si->si_active); KASSERTMSG(l->l_nopreempt == 0, "lwp %p nopreempt %d", l, l->l_nopreempt); /* * Note: due to priority inheritance we may have interrupted a * higher priority LWP. Since the soft interrupt must be quick * and is non-preemptable, we don't bother yielding. */ while (!SIMPLEQ_EMPTY(&si->si_q)) { /* * Pick the longest waiting handler to run. We block * interrupts but do not lock in order to do this, as * we are protecting against the local CPU only. */ sh = SIMPLEQ_FIRST(&si->si_q); SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q); KASSERT((sh->sh_flags & SOFTINT_PENDING) != 0); sh->sh_flags ^= SOFTINT_PENDING; splx(s); /* Run the handler. */ SDT_PROBE4(sdt, kernel, softint, entry, ((const char *)sh - (const char *)curcpu()->ci_data.cpu_softcpu), sh->sh_func, sh->sh_arg, sh->sh_flags); if (__predict_true((sh->sh_flags & SOFTINT_MPSAFE) != 0)) { (*sh->sh_func)(sh->sh_arg); } else { KERNEL_LOCK(1, l); (*sh->sh_func)(sh->sh_arg); KERNEL_UNLOCK_ONE(l); } SDT_PROBE4(sdt, kernel, softint, return, ((const char *)sh - (const char *)curcpu()->ci_data.cpu_softcpu), sh->sh_func, sh->sh_arg, sh->sh_flags); /* Diagnostic: check that spin-locks have not leaked. */ KASSERTMSG(curcpu()->ci_mtx_count == 0, "%s: ci_mtx_count (%d) != 0, sh_func %p\n", __func__, curcpu()->ci_mtx_count, sh->sh_func); /* Diagnostic: check that psrefs have not leaked. */ KASSERTMSG(l->l_psrefs == 0, "%s: l_psrefs=%d, sh_func=%p\n", __func__, l->l_psrefs, sh->sh_func); /* Diagnostic: check that biglocks have not leaked. */ KASSERTMSG(l->l_blcnt == 0, "%s: sh_func=%p leaked %d biglocks", __func__, sh->sh_func, curlwp->l_blcnt); /* Diagnostic: check that LWP nopreempt remains zero. */ KASSERTMSG(l->l_nopreempt == 0, "%s: lwp %p nopreempt %d func %p", __func__, l, l->l_nopreempt, sh->sh_func); (void)splhigh(); } PSREF_DEBUG_BARRIER(); CPU_COUNT(CPU_COUNT_NSOFT, 1); KASSERT(si->si_cpu == curcpu()); KASSERT(si->si_lwp->l_wchan == NULL); KASSERT(si->si_active); si->si_evcnt.ev_count++; si->si_active = 0; } /* * softint_block: * * Update statistics when the soft interrupt blocks. */ void softint_block(lwp_t *l) { softint_t *si = l->l_private; KASSERT((l->l_pflag & LP_INTR) != 0); si->si_evcnt_block.ev_count++; } #ifndef __HAVE_FAST_SOFTINTS #ifdef __HAVE_PREEMPTION #error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS #endif /* * softint_init_md: * * Slow path: perform machine-dependent initialization. */ void softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep) { struct proc *p; softint_t *si; *machdep = (1 << level); si = l->l_private; p = l->l_proc; mutex_enter(p->p_lock); lwp_lock(l); /* Cheat and make the KASSERT in softint_thread() happy. */ si->si_active = 1; setrunnable(l); /* LWP now unlocked */ mutex_exit(p->p_lock); } /* * softint_trigger: * * Slow path: cause a soft interrupt handler to begin executing. * Called at IPL_HIGH. */ void softint_trigger(uintptr_t machdep) { struct cpu_info *ci; lwp_t *l; ci = curcpu(); ci->ci_data.cpu_softints |= machdep; l = ci->ci_onproc; /* * Arrange for mi_switch() to be called. If called from interrupt * mode, we don't know if curlwp is executing in kernel or user, so * post an AST and have it take a trip through userret(). If not in * interrupt mode, curlwp is running in kernel and will notice the * resched soon enough; avoid the AST. */ if (l == ci->ci_data.cpu_idlelwp) { atomic_or_uint(&ci->ci_want_resched, RESCHED_IDLE | RESCHED_UPREEMPT); } else { atomic_or_uint(&ci->ci_want_resched, RESCHED_UPREEMPT); if (cpu_intr_p()) { cpu_signotify(l); } } } /* * softint_thread: * * Slow path: MI software interrupt dispatch. */ void softint_thread(void *cookie) { softint_t *si; lwp_t *l; int s; l = curlwp; si = l->l_private; for (;;) { /* Clear pending status and run it. */ s = splhigh(); l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep; softint_execute(l, s); splx(s); /* Interrupts allowed to run again before switching. */ lwp_lock(l); l->l_stat = LSIDL; spc_lock(l->l_cpu); mi_switch(l); } } /* * softint_picklwp: * * Slow path: called from mi_switch() to pick the highest priority * soft interrupt LWP that needs to run. */ lwp_t * softint_picklwp(void) { struct cpu_info *ci; u_int mask; softint_t *si; lwp_t *l; ci = curcpu(); si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int; mask = ci->ci_data.cpu_softints; if ((mask & (1 << SOFTINT_SERIAL)) != 0) { l = si[SOFTINT_SERIAL].si_lwp; } else if ((mask & (1 << SOFTINT_NET)) != 0) { l = si[SOFTINT_NET].si_lwp; } else if ((mask & (1 << SOFTINT_BIO)) != 0) { l = si[SOFTINT_BIO].si_lwp; } else if ((mask & (1 << SOFTINT_CLOCK)) != 0) { l = si[SOFTINT_CLOCK].si_lwp; } else { panic("softint_picklwp"); } return l; } #else /* !__HAVE_FAST_SOFTINTS */ /* * softint_thread: * * Fast path: the LWP is switched to without restoring any state, * so we should not arrive here - there is a direct handoff between * the interrupt stub and softint_dispatch(). */ void softint_thread(void *cookie) { panic("softint_thread"); } /* * softint_dispatch: * * Fast path: entry point from machine-dependent code. */ void softint_dispatch(lwp_t *pinned, int s) { struct bintime now; u_int timing; lwp_t *l; #ifdef DIAGNOSTIC if ((pinned->l_pflag & LP_RUNNING) == 0 || curlwp->l_stat != LSIDL) { struct lwp *onproc = curcpu()->ci_onproc; int s2 = splhigh(); printf("curcpu=%d, spl=%d curspl=%d\n" "onproc=%p => l_stat=%d l_flag=%08x l_cpu=%d\n" "curlwp=%p => l_stat=%d l_flag=%08x l_cpu=%d\n" "pinned=%p => l_stat=%d l_flag=%08x l_cpu=%d\n", cpu_index(curcpu()), s, s2, onproc, onproc->l_stat, onproc->l_flag, cpu_index(onproc->l_cpu), curlwp, curlwp->l_stat, curlwp->l_flag, cpu_index(curlwp->l_cpu), pinned, pinned->l_stat, pinned->l_flag, cpu_index(pinned->l_cpu)); splx(s2); panic("softint screwup"); } #endif /* * Note the interrupted LWP, and mark the current LWP as running * before proceeding. Although this must as a rule be done with * the LWP locked, at this point no external agents will want to * modify the interrupt LWP's state. */ timing = softint_timing; l = curlwp; l->l_switchto = pinned; l->l_stat = LSONPROC; /* * Dispatch the interrupt. If softints are being timed, charge * for it. */ if (timing) { binuptime(&l->l_stime); membar_producer(); /* for calcru */ l->l_pflag |= LP_TIMEINTR; } l->l_pflag |= LP_RUNNING; softint_execute(l, s); if (timing) { binuptime(&now); updatertime(l, &now); l->l_pflag &= ~LP_TIMEINTR; } /* * If we blocked while handling the interrupt, the pinned LWP is * gone and we are now running as a kthread, so find another LWP to * run. softint_dispatch() won't be reentered until the priority is * finally dropped to IPL_NONE on entry to the next LWP on this CPU. */ l->l_stat = LSIDL; if (l->l_switchto == NULL) { lwp_lock(l); spc_lock(l->l_cpu); mi_switch(l); /* NOTREACHED */ } l->l_switchto = NULL; l->l_pflag &= ~LP_RUNNING; } #endif /* !__HAVE_FAST_SOFTINTS */
