/*      $NetBSD: spl.S,v 1.59 2026/01/17 10:59:09 bouyer Exp $  */

/*
 * Copyright (c) 1998, 2007, 2008, 2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Charles M. Hannum and Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <machine/asm.h>
__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.59 2026/01/17 10:59:09 bouyer Exp $");

#include "opt_ddb.h"
#include "opt_spldebug.h"
#include "opt_xen.h"

#include <machine/trap.h>
#include <machine/segments.h>
#include <machine/frameasm.h>

#include "assym.h"

        .text

/*
 * int splraise(int s);
 */
ENTRY(splraise)
        movl    4(%esp),%edx
        movzbl  CPUVAR(ILEVEL),%eax
        cmpl    %edx,%eax
        ja      1f
        movb    %dl,CPUVAR(ILEVEL)
1:
#ifdef SPLDEBUG
        pushl   %ebp
        movl    %esp,%ebp
        pushl   %eax
        pushl   %edx
        call    _C_LABEL(spldebug_raise)
        addl    $4,%esp
        popl    %eax
        popl    %ebp
#endif /* SPLDEBUG */
        ret
END(splraise)

#ifndef XENPV
/*
 * void spllower(int s);
 *
 * spllower() for i486 and Pentium. Must be the same size as cx8_spllower(),
 * that is, 96 bytes. This must use pushf/cli/popf as it is used early in boot
 * where interrupts are disabled via eflags/IE.
 */
ENTRY(spllower)
        HOTPATCH(HP_NAME_SPLLOWER, 96)
#ifdef SPLDEBUG
        movl    4(%esp),%ecx
        pushl   %ebp
        movl    %esp,%ebp
        pushl   %ecx
        call    _C_LABEL(spldebug_lower)
        addl    $4,%esp
        popl    %ebp
#endif /* SPLDEBUG */
        movl    4(%esp),%ecx
        cmpb    CPUVAR(ILEVEL),%cl
        jae     1f
        PUSHFCLI(%eax, %edx)
        movl    CPUVAR(IUNMASK)(,%ecx,8),%edx
        movl    CPUVAR(IUNMASK)+4(,%ecx,8),%eax
        testl   CPUVAR(IPENDING),%edx
        jnz     2f
        testl   CPUVAR(IPENDING)+4,%eax
        jnz     2f
        movb    %cl,CPUVAR(ILEVEL)
        POPF(%eax)
1:
        ret
2:
        popf
        jmp     _C_LABEL(Xspllower)
        .align  32
END(spllower)
#else  /* XENPV */
STRONG_ALIAS(spllower, cx8_spllower)
#endif /* !XENPV */

/*
 * void cx8_spllower(int s);
 *
 * spllower() optimized for Pentium Pro and later, which have long pipelines
 * that will be stalled by pushf/cli/popf.  Must be the same size as
 * spllower(), ie 96 bytes.  Does not need to restore eflags/IE as is patched
 * in once autoconf is underway.
 *
 * For cmpxchg8b, edx/ecx are the high words and eax/ebx the low.
 *
 * edx : eax = old level + high 24 bit old ipending / low 32 bit old ipending
 * ecx : ebx = new level + high 24 bit old ipending / low 32 bit old ipending
 */
ENTRY(cx8_spllower)
        movl    4(%esp),%ecx
        movzbl  CPUVAR(ILEVEL),%edx
        cmpl    %edx,%ecx                       /* new level is lower? */
        jae     1f
        pushl   %ebx
        pushl   %esi
        pushl   %edi
        movl    %ecx,%esi
        movl    %ecx,%edi
        shll    $24,%edi
0:
        movl    CPUVAR(IPENDING),%eax
        testl   %eax,CPUVAR(IUNMASK)(,%esi,8)   /* deferred interrupts? */
        jnz     2f
        movl    CPUVAR(IPENDING)+4,%edx
        testl   %edx,CPUVAR(IUNMASK)+4(,%esi,8)
        jnz     2f
        movl    %eax,%ebx
        movl    %edx,%ecx
        andl    $0x00ffffff,%ecx
        orl     %edi,%ecx
        cmpxchg8b CPUVAR(ISTATE)                /* swap in new ilevel */
        jnz     0b
        popl    %edi
        popl    %esi
        popl    %ebx
1:
        ret
2:
        popl    %edi
        popl    %esi
        popl    %ebx

        /* The reference must be absolute, hence the indirect jump. */
        movl    $Xspllower,%eax
        jmp     *%eax

        .align  32, 0xCC
LABEL(cx8_spllower_end)
END(cx8_spllower)

/*
 * void Xspllower(int s);
 *
 * Process pending interrupts.
 *
 * Important registers:
 *   ebx - cpl
 *   esi - address to resume loop at
 *   edi - scratch for Xsoftnet
 *
 * It is important that the bit scan instruction is bsr, it will get
 * the highest 2 bits (currently the IPI and clock handlers) first,
 * to avoid deadlocks where one CPU sends an IPI, another one is at
 * splhigh() and defers it, lands in here via splx(), and handles
 * a lower-prio one first, which needs to take the kernel lock -->
 * the sending CPU will never see the that CPU accept the IPI
 * (see pmap_tlb_shootnow).
 */
        nop     /* Don't get confused with cx8_spllower_end */

IDTVEC(spllower)
        pushl   %ebp
        movl    %esp,%ebp
        MCOUNT_ASM
        pushl   %ebx
        pushl   %esi
        pushl   %edi
        movl    8(%ebp),%ebx
        movl    $.Lspllower_resume,%esi         /* address to resume loop at */
1:
        /*
         * Because of the way Xen interrupts work *%esi will in fact be called
         * from Xdoreti via iret. So we have to always disable interrupts here
         * for Xen.
         */
#ifndef XENPV
        CLI(%eax)
#endif
.Lspllower_resume:
#ifdef XENPV
        CLI2(%eax, %edi)
#endif
#if defined(DEBUG)
#ifndef XENPV
        pushf
        popl    %eax
        testl   $PSL_I,%eax
        jnz     .Lspllower_panic
#else
        movl    CPUVAR(VCPU),%eax
        movb    EVTCHN_UPCALL_MASK(%eax),%al
        andb    %al,%al
        jz      .Lspllower_panic
#endif /* XENPV */
#endif /* defined(DEBUG) */
        movl    %ebx,%eax                       /* get cpl */
        movl    CPUVAR(IUNMASK)+4(,%eax,8),%eax
        andl    CPUVAR(IPENDING)+4,%eax         /* any non-masked bits left? */
        jz      10f
        bsrl    %eax,%eax
        btrl    %eax,CPUVAR(IPENDING)+4
        addl    $32,%eax
        movl    CPUVAR(ISOURCES)(,%eax,4),%eax
        jmp     *IS_RECURSE(%eax)
10:
        movl    %ebx,%eax                       /* get cpl */
        movl    CPUVAR(IUNMASK)(,%eax,8),%eax
        andl    CPUVAR(IPENDING),%eax           /* any non-masked bits left? */
        jz      2f
        bsrl    %eax,%eax
        btrl    %eax,CPUVAR(IPENDING)
        movl    CPUVAR(ISOURCES)(,%eax,4),%eax
        jmp     *IS_RECURSE(%eax)
2:
        movb    %bl,CPUVAR(ILEVEL)
#ifdef XENPV
        STIC(%eax)
        jz 4f
        call    _C_LABEL(stipending)
        testl   %eax,%eax
        jnz     1b
4:
#else
        STI(%eax)
#endif
        popl    %edi
        popl    %esi
        popl    %ebx
        leave
        ret
#if defined(DEBUG)
.Lspllower_panic:
        pushl   $1f
        call    _C_LABEL(panic)
1:      .asciz  "SPLLOWER: INTERRUPT ENABLED"
#endif
IDTVEC_END(spllower)

/*
 * Xdoreti: Handle return from interrupt after device handler finishes.
 *
 * Important registers:
 *   ebx - cpl to restore
 *   esi - address to resume loop at
 *   edi - scratch for Xsoftnet
 *
 * called with interrupt disabled.
 */
IDTVEC(doreti)
        IDEPTH_DECR
        popl    %ebx                    /* get previous priority */
.Ldoreti_resume_stic:
        movl    $.Ldoreti_resume,%esi   /* address to resume loop at */
.Ldoreti_resume:

#if defined(DEBUG)
#ifndef XENPV
        pushf
        popl    %eax
        testl   $PSL_I,%eax
        jnz     .Ldoreti_panic
#else
        movl    CPUVAR(VCPU),%eax
        movb    EVTCHN_UPCALL_MASK(%eax),%al
        andb    %al,%al
        jz      .Ldoreti_panic
#endif /* XENPV */
#endif /* defined(DEBUG) */

        movl    %ebx,%eax
        movl    CPUVAR(IUNMASK)+4(,%eax,8),%eax
        andl    CPUVAR(IPENDING)+4,%eax
        jz      10f
        bsrl    %eax,%eax               /* slow, but not worth optimizing */
        btrl    %eax,CPUVAR(IPENDING)+4
        addl    $32,%eax
        movl    CPUVAR(ISOURCES)(,%eax, 4),%eax
        jmp     *IS_RESUME(%eax)
10:
        movl    %ebx,%eax
        movl    CPUVAR(IUNMASK)(,%eax,8),%eax
        andl    CPUVAR(IPENDING),%eax
        jz      2f
        bsrl    %eax,%eax               /* slow, but not worth optimizing */
        btrl    %eax,CPUVAR(IPENDING)
        movl    CPUVAR(ISOURCES)(,%eax, 4),%eax
        jmp     *IS_RESUME(%eax)
2:      /* Check for ASTs on exit to user mode. */
        movb    %bl,CPUVAR(ILEVEL)
5:
        testb   $CHK_UPL,TF_CS(%esp)
        jnz     doreti_checkast
        jmp     6f

        .type   _C_LABEL(doreti_checkast), @function
LABEL(doreti_checkast)
        CHECK_ASTPENDING(%eax)
        jz      3f
        CLEAR_ASTPENDING(%eax)
        STI(%eax)
        movl    $T_ASTFLT,TF_TRAPNO(%esp)       /* XXX undo later.. */
        /* Pushed T_ASTFLT into tf_trapno on entry. */
        pushl   %esp
        call    _C_LABEL(trap)
        addl    $4,%esp
        CLI2(%eax, %esi)
        jmp     5b
END(doreti_checkast)

3:
        CHECK_DEFERRED_SWITCH
        jnz     9f
        HANDLE_DEFERRED_FPU
6:
#ifdef XENPV
        STIC(%eax)
        jz      4f
        call    _C_LABEL(stipending)
        testl   %eax,%eax
        jz      4f
        CLI2(%eax, %ecx)
        jmp     .Ldoreti_resume_stic
4:
#endif
        INTRFASTEXIT
9:
        STI(%eax)
        call    _C_LABEL(pmap_load)
        CLI2(%eax, %ecx)
        jmp     doreti_checkast /* recheck ASTs */

#if defined(DEBUG)
.Ldoreti_panic:
        pushl   $1f
        call    _C_LABEL(panic)
1:      .asciz  "DORETI: INTERRUPT ENABLED"
#endif
IDTVEC_END(doreti)

/*
 * Xsoftintr()
 *
 * Switch to the LWP assigned to handle interrupts from the given
 * source.  We borrow the VM context from the interrupted LWP.
 *
 * On entry:
 *
 *      %eax            intrsource
 *      %esi            address to return to
 */
IDTVEC(softintr)
        pushl   $_C_LABEL(softintr_ret) /* set up struct switchframe */
        pushl   %ebx
        pushl   %esi
        pushl   %edi
        movb    $IPL_HIGH,CPUVAR(ILEVEL)
        STI(%esi)
        movl    CPUVAR(CURLWP),%esi
        movl    IS_LWP(%eax),%edi       /* switch to handler LWP */
        /*
         * Simple MOV to set curlwp to softlwp.  See below on ordering
         * required to restore softlwp like cpu_switchto.
         *
         * 1. Don't need store-before-store barrier because x86 is TSO.
         *
         * 2. Don't need store-before-load barrier because when we
         *    enter a softint lwp, it can't be holding any mutexes, so
         *    it can't release any until after it has acquired them, so
         *    we need not participate in the protocol with
         *    mutex_vector_enter barriers here.
         *
         * Hence no need for XCHG or barriers around MOV.
         */
        movl    %edi,CPUVAR(CURLWP)
        movl    L_PCB(%edi),%edx
        movl    L_PCB(%esi),%ecx
        movl    %esp,PCB_ESP(%ecx)
        movl    %ebp,PCB_EBP(%ecx)
        movl    PCB_ESP0(%edx),%esp     /* onto new stack */
        pushl   IS_MAXLEVEL(%eax)       /* ipl to run at */
        pushl   %esi
        call    _C_LABEL(softint_dispatch)/* run handlers */
        addl    $8,%esp
        CLI(%ecx)
        movl    L_PCB(%esi),%ecx
        movl    PCB_ESP(%ecx),%esp

        /*
         * Use XCHG, not MOV, to coordinate mutex_exit on this CPU with
         * mutex_vector_enter on another CPU.
         *
         * 1. Any prior mutex_exit by the softint must be visible to
         *    other CPUs before we restore curlwp on this one,
         *    requiring store-before-store ordering.
         *
         *    (This is always guaranteed by the x86 memory model, TSO,
         *    but other architectures require a explicit barrier before
         *    the store to ci->ci_curlwp.)
         *
         * 2. Restoring curlwp must be visible on all other CPUs before
         *    any subsequent mutex_exit on this one can even test
         *    whether there might be waiters, requiring
         *    store-before-load ordering.
         *
         *    (This is the only ordering x86 TSO ever requires any kind
         *    of barrier for -- in this case, we take advantage of the
         *    sequential consistency implied by XCHG to obviate the
         *    need for MFENCE or something.)
         *
         * See kern_mutex.c for details -- this is necessary for
         * adaptive mutexes to detect whether the lwp is on the CPU in
         * order to safely block without requiring atomic r/m/w in
         * mutex_exit.  See also cpu_switchto.
         */
        xchgl   %esi,CPUVAR(CURLWP)     /* restore ci_curlwp */
        popl    %edi                    /* unwind switchframe */
        popl    %esi
        addl    $8,%esp
        jmp     *%esi                   /* back to splx/doreti */
IDTVEC_END(softintr)

/*
 * softintr_ret()
 *
 * Trampoline function that gets returned to by cpu_switchto() when
 * an interrupt handler blocks.  On entry:
 *
 *      %eax            prevlwp from cpu_switchto()
 */
ENTRY(softintr_ret)
        incl    CPUVAR(MTX_COUNT)       /* re-adjust after mi_switch */
        CLI(%eax)
        jmp     *%esi                   /* back to splx/doreti */
END(softintr_ret)

/*
 * void softint_trigger(uintptr_t machdep);
 *
 * Software interrupt registration.
 */
ENTRY(softint_trigger)
        movl    4(%esp),%eax
        orl     %eax,CPUVAR(IPENDING)   /* atomic on local cpu */
        ret
END(softint_trigger)

/*
 * Xrecurse_preempt()
 *
 * Handles preemption interrupts via Xspllower().
 */
IDTVEC(recurse_preempt)
        movb    $IPL_PREEMPT,CPUVAR(ILEVEL)
        STI(%eax)
        pushl   $0
        call    _C_LABEL(kpreempt)
        addl    $4,%esp
        CLI(%eax)
        jmp     *%esi
IDTVEC_END(recurse_preempt)

/*
 * Xresume_preempt()
 *
 * Handles preemption interrupts via Xdoreti().
 */
IDTVEC(resume_preempt)
        movb    $IPL_PREEMPT,CPUVAR(ILEVEL)
        STI(%eax)
        testb   $CHK_UPL,TF_CS(%esp)
        jnz     1f
        movl    TF_EIP(%esp),%eax
        pushl   %eax
        call    _C_LABEL(kpreempt)              /* from kernel */
        addl    $4,%esp
        CLI(%eax)
        jmp     *%esi
1:
        call    _C_LABEL(preempt)               /* from user */
        CLI(%eax)
        jmp     *%esi
IDTVEC_END(resume_preempt)