/*      $NetBSD: trap_subr.S,v 1.32 2026/06/20 09:30:07 rkujawa Exp $   */

/*
 * Copyright 2001 Wasabi Systems, Inc.
 * All rights reserved.
 *
 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed for the NetBSD Project by
 *      Wasabi Systems, Inc.
 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
 *    or promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
 * Copyright (C) 1995, 1996 TooLs GmbH.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by TooLs GmbH.
 * 4. The name of TooLs GmbH may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * NOTICE: This is not a standalone file.  to use it, #include it in
 * your port's locore.S, like so:
 *
 *      #include <powerpc/ibm4xx/trap_subr.S>
 */

#ifdef _KERNEL_OPT
#include "opt_ddb.h"
#include "opt_kgdb.h"
#include "opt_ppcarch.h"
#endif

/*
 * XXX Interrupt and spill stacks need to be per-CPU.
 */

#define GET_PCB(rX)     \
        GET_CPUINFO(rX);        \
        lwz     rX,CI_CURPCB(rX)

#define STANDARD_PROLOG(savearea)                               \
        mtsprg1 %r1;                    /* save SP */           \
        GET_CPUINFO(%r1);                                       \
        stmw    %r28,(savearea+CPUSAVE_R28)(%r1); /* free r28-r31 */ \
        mflr    %r28;                   /* save LR */           \
        mfcr    %r29;                   /* save CR */           \
        mfsrr0  %r30;                                           \
        mfsrr1  %r31; /* Test whether we already had PR set */  \
        stmw    %r30,(savearea+CPUSAVE_SRR0)(%r1); /* save srr0/srr1 */ \
        mfsprg1 %r1;                    /* restore SP */        \
        mtcr    %r31;                                           \
        bf      MSR_PR,1f;              /* branch if MSR[PR] is clear */ \
        GET_PCB(%r1);                                           \
        addi    %r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
1:

#define ACCESS_PROLOG(savearea)                                 \
        mtsprg1 %r1;                    /* save SP temporalily */ \
        GET_CPUINFO(%r1);                                       \
        stmw    %r28,(savearea+CPUSAVE_R28)(%r1); /* free r28-r31 */ \
        mflr    %r28;                   /* save LR */           \
        mfcr    %r29;                   /* save CR */           \
        mfspr   %r30,SPR_DEAR;                                  \
        mfspr   %r31,SPR_ESR;                                   \
        stmw    %r30,(savearea+CPUSAVE_DEAR)(%r1); /* save esr/dear */ \
        mfsrr0  %r30;                                           \
        mfsrr1  %r31; /* Test whether we already had PR set */  \
        stmw    %r30,(savearea+CPUSAVE_SRR0)(%r1); /* save srr0/srr1 */ \
        mfsprg1 %r1;                    /* restore SP */        \
        mtcr    %r31;                                           \
        bf      MSR_PR,1f;              /* branch if MSR[PR] is clear */ \
        GET_PCB(%r1);                                           \
        addi    %r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
1:

#ifdef PPC_IBM440
/* Book E: critical state is in CSRR0/CSRR1 instead of SRR2/SRR3 */
#define CRITICAL_PROLOG(savearea)                               \
        mtsprg1 %r1;                    /* save SP */           \
        GET_CPUINFO(%r1);                                       \
        stmw    %r28,(savearea+CPUSAVE_R28)(%r1); /* free r28-r31 */ \
        mflr    %r28;                   /* save LR */           \
        mfcr    %r29;                   /* save CR */           \
        mfspr   %r30,SPR_CSRR0; /* Fake a standard trap */      \
        mfspr   %r31,SPR_CSRR1; /* Test whether we already had PR set */ \
        stmw    %r30,(savearea+CPUSAVE_SRR0)(%r1); /* save srr0/srr1 */ \
        mfsprg1 %r1;                    /* restore SP */        \
        mtcr    %r31;                                           \
        bf      MSR_PR,1f;              /* branch if MSR[PR] is clear */ \
        GET_PCB(%r1);                                           \
        addi    %r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
1:

/* 440x6/460: machine check state is in MCSRR0/MCSRR1 */
#define MCHK_PROLOG(savearea)                                   \
        mtsprg1 %r1;                    /* save SP */           \
        GET_CPUINFO(%r1);                                       \
        stmw    %r28,(savearea+CPUSAVE_R28)(%r1); /* free r28-r31 */ \
        mflr    %r28;                   /* save LR */           \
        mfcr    %r29;                   /* save CR */           \
        mfspr   %r30,SPR_MCSRR0; /* Fake a standard trap */     \
        mfspr   %r31,SPR_MCSRR1; /* Test whether we already had PR set */ \
        stmw    %r30,(savearea+CPUSAVE_SRR0)(%r1); /* save srr0/srr1 */ \
        mfsprg1 %r1;                    /* restore SP */        \
        mtcr    %r31;                                           \
        bf      MSR_PR,1f;              /* branch if MSR[PR] is clear */ \
        GET_PCB(%r1);                                           \
        addi    %r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
1:
#else
#define CRITICAL_PROLOG(savearea)                               \
        mtsprg1 %r1;                    /* save SP */           \
        GET_CPUINFO(%r1);                                       \
        stmw    %r28,(savearea+CPUSAVE_R28)(%r1); /* free r28-r31 */ \
        mflr    %r28;                   /* save LR */           \
        mfcr    %r29;                   /* save CR */           \
        mfsrr2  %r30; /* Fake a standard trap */                \
        mfsrr3  %r31; /* Test whether we already had PR set */  \
        stmw    %r30,(savearea+CPUSAVE_SRR0)(%r1); /* save srr0/srr1 */ \
        mfsprg1 %r1;                    /* restore SP */        \
        mtcr    %r31;                                           \
        bf      MSR_PR,1f;              /* branch if MSR[PR] is clear */ \
        GET_PCB(%r1);                                           \
        addi    %r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
1:
#endif /* PPC_IBM440 */


/* Standard handler saves r1,r28-31,LR,CR, sets up the stack and calls s_trap */
#define STANDARD_EXC_HANDLER(name)\
        .globl  _C_LABEL(name ## trap),_C_LABEL(name ## size) ; \
_C_LABEL(name ## trap):                                         \
        STANDARD_PROLOG(CI_TEMPSAVE);                           \
        bla     s_trap;                                         \
_C_LABEL(name ## size) = .-_C_LABEL(name ## trap)

/* Access exceptions also need DEAR and ESR saved */
#define ACCESS_EXC_HANDLER(name)\
        .globl  _C_LABEL(name ## trap),_C_LABEL(name ## size) ; \
_C_LABEL(name ## trap):                                         \
        ACCESS_PROLOG(CI_TEMPSAVE);                             \
        bla     s_trap;                                         \
_C_LABEL(name ## size) = .-_C_LABEL(name ## trap)

/* Maybe this should call ddb.... */
#ifdef PPC_IBM440
/*
 * Critical-class traps use their own save area.
 */
#define CRITICAL_EXC_HANDLER(name)\
        .globl  _C_LABEL(name ## trap),_C_LABEL(name ## size) ; \
_C_LABEL(name ## trap):                                         \
        CRITICAL_PROLOG(CI_CRITSAVE);                           \
        bla     s_ctrap;                                        \
_C_LABEL(name ## size) = .-_C_LABEL(name ## trap)
#else
#define CRITICAL_EXC_HANDLER(name)\
        .globl  _C_LABEL(name ## trap),_C_LABEL(name ## size) ; \
_C_LABEL(name ## trap):                                         \
        CRITICAL_PROLOG(CI_TEMPSAVE);                           \
        bla     s_trap;                                         \
_C_LABEL(name ## size) = .-_C_LABEL(name ## trap)
#endif /* PPC_IBM440 */

#define INTR_PROLOG(tempsave)                                   \
        mtsprg1 %r1;                    /* save SP */           \
        GET_CPUINFO(%r1);                                       \
        stmw    %r28,(tempsave+CPUSAVE_R28)(%r1); /* free r28-r31 */ \
        mflr    %r28;                   /* save LR */           \
        mfcr    %r29;                   /* save CR */           \
        mfxer   %r30;                   /* save XER */          \
        mfsrr1  %r31;                                           \
        mtcr    %r31;                                           \
        mfsprg1 %r1;                    /* restore SP */        \
        bf      MSR_PR,1f;              /* branch if PSL_PR is false */ \
        GET_PCB(%r1);                                           \
        addi    %r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
1:

/* Machine check handler: separate save/restore regs on 440x6/460 */
#ifdef PPC_IBM440
#define MCHK_EXC_HANDLER(name)\
        .globl  _C_LABEL(name ## trap),_C_LABEL(name ## size) ; \
_C_LABEL(name ## trap):                                         \
        MCHK_PROLOG(CI_CRITSAVE);                               \
        bla     s_mctrap;                                       \
_C_LABEL(name ## size) = .-_C_LABEL(name ## trap)
#endif

        .text
        STANDARD_EXC_HANDLER(default)
        ACCESS_EXC_HANDLER(access)
        CRITICAL_EXC_HANDLER(critical)
#ifdef PPC_IBM440
        MCHK_EXC_HANDLER(mchk)
#endif

/*
 * This one for the external interrupt handler.
 */
        .globl  _C_LABEL(extint),_C_LABEL(extsize)
_C_LABEL(extint):
        INTR_PROLOG(CI_TEMPSAVE)
        ba      extintr
_C_LABEL(extsize) = .-_C_LABEL(extint)


#if defined(DDB) || defined(KGDB)
/*
 * In case of DDB we want a separate trap catcher for it
 */
        .globl  _C_LABEL(ddblow),_C_LABEL(ddbsize)
_C_LABEL(ddblow):
        ACCESS_PROLOG(CI_DDBSAVE)
        bla     ddbtrap
_C_LABEL(ddbsize) = .-_C_LABEL(ddblow)
#endif  /* DDB || KGDB */

#ifdef DEBUG
#define TRAP_IF_ZERO(r) tweqi   r,0
#else
#define TRAP_IF_ZERO(r)
#endif

#define ENABLE_TRANSLATION(pidreg,tmpreg)                               \
        mfspr   pidreg,SPR_PID;                                         \
        li      tmpreg,KERNEL_PID;                                      \
        mtspr   SPR_PID,tmpreg;                                         \
        mfmsr   tmpreg;                                                 \
        ori     tmpreg,tmpreg,(PSL_DR|PSL_IR)@l;                        \
        mtmsr   tmpreg;                                                 \
        isync

/*
 * FRAME_SETUP assumes:
 *      SPRG1           SP (r1)
 *      savearea        r28-r31,DEAR,ESR,SRR0,SRR1
 *                      (DEAR & ESR only for access traps)
 *      %r28            LR
 *      %r29            CR
 *      %r1             kernel stack
 *      LR              trap type
 */
#define CLEAR_WAIT_STATE(reg)   rlwinm reg,reg,0,14,12

#define FRAME_SETUP(savearea)                                           \
/* Have to enable translation to allow access of kernel stack: */       \
        ENABLE_TRANSLATION(%r30,%r31);                                  \
        mfsprg1 %r31;                                                   \
        stwu    %r31,-FRAMELEN(%r1);                                    \
        stw     %r30,FRAME_PID(%r1);                                    \
        stw     %r0,FRAME_R0(%r1);                                      \
        stw     %r31,FRAME_R1(%r1);                                     \
        stw     %r2,FRAME_R2(%r1);                                      \
        GET_CPUINFO(%r2);                                               \
        stw     %r28,FRAME_LR(%r1);                                     \
        stw     %r29,FRAME_CR(%r1);                                     \
        lmw     %r28,(savearea+CPUSAVE_R28)(%r2);                       \
        stmw    %r3,FRAME_R3(%r1);                                      \
        lmw     %r28,(savearea+CPUSAVE_DEAR)(%r2);                      \
        lwz     %r13,CI_CURLWP(%r2);                                    \
        mfxer   %r3;                                                    \
        mfctr   %r4;                                                    \
        mflr    %r5;                                                    \
        andi.   %r5,%r5,0xff00;                                         \
        stw     %r3,FRAME_XER(%r1);                                     \
        stw     %r4,FRAME_CTR(%r1);                                     \
        stw     %r5,FRAME_EXC(%r1);                                     \
        stw     %r28,FRAME_DEAR(%r1);                                   \
        stw     %r29,FRAME_ESR(%r1);                                    \
        stw     %r30,FRAME_SRR0(%r1);                                   \
        CLEAR_WAIT_STATE(%r31);         /* wake from MSR[WE] idle */    \
        stw     %r31,FRAME_SRR1(%r1)

#define FRAME_SAVE_CALLEE                                               \
        stmw    %r14,FRAME_R14(%r1)

#define FRAME_RESTORE                                                   \
        lwz     %r6,FRAME_LR(%r1);                                      \
        lwz     %r7,FRAME_CR(%r1);                                      \
        lwz     %r8,FRAME_XER(%r1);                                     \
        lwz     %r9,FRAME_CTR(%r1);                                     \
        lwz     %r10,FRAME_SRR0(%r1);                                   \
        lwz     %r11,FRAME_SRR1(%r1);                                   \
        mtlr    %r6;                                                    \
        mtcr    %r7;                                                    \
        mtxer   %r8;                                                    \
        mtctr   %r9;                                                    \
        mtsrr0  %r10;                                                   \
        mtsrr1  %r11;                                                   \
        lwz     %r13,FRAME_R13(%r1);                                    \
        lwz     %r12,FRAME_R12(%r1);                                    \
        lwz     %r11,FRAME_R11(%r1);                                    \
        lwz     %r10,FRAME_R10(%r1);                                    \
        lwz     %r9,FRAME_R9(%r1);                                      \
        lwz     %r8,FRAME_R8(%r1);                                      \
        lwz     %r7,FRAME_R7(%r1);                                      \
        lwz     %r6,FRAME_R6(%r1);                                      \
        lwz     %r5,FRAME_R5(%r1);                                      \
        lwz     %r4,FRAME_R4(%r1);                                      \
        lwz     %r3,FRAME_R3(%r1);                                      \
        lwz     %r2,FRAME_R2(%r1);                                      \
        lwz     %r0,FRAME_R1(%r1);                                      \
        mtsprg1 %r0;                                                    \
        lwz     %r0,FRAME_R0(%r1)

/*
 * Now the common trap catching code.
 */
s_trap:
        FRAME_SETUP(CI_TEMPSAVE)
        /* R31 = SRR1 */
/* Now we can recover interrupts again: */
trapagain:
        wrtee   %r31                    /* reenable interrupts */
/* Call C trap code: */
        addi    %r3,%r1,FRAME_TF
        bl      _C_LABEL(trap)
        .globl  _C_LABEL(trapexit)
_C_LABEL(trapexit):
        /* Disable interrupts: */
        wrteei  0

        /* Test AST pending: */
        mtcr    %r31
        bf      MSR_PR,trapleave_to_kernel /* branch if MSR[PR] is false */

        lwz     %r4,L_MD_ASTPENDING(%r13)
        andi.   %r4,%r4,1
        beq     trapleave_to_user

        li      %r6,EXC_AST
        stw     %r6,FRAME_EXC(%r1)
        b       trapagain

trapleave_to_kernel:
        lmw     %r14, FRAME_R14(%r1)    /* restore callee registers */

intrleave_to_kernel:
        FRAME_RESTORE           /* old SP is now in sprg1 */

        mtsprg2 %r30
        mtsprg3 %r31
        mfmsr   %r30
        li      %r31,(PSL_DR|PSL_IR)@l
        andc    %r30,%r30,%r31
        lwz     %r31,FRAME_PID(%r1)
        TRAP_IF_ZERO(%r31)
        /*
         * Now that we are done with the trapframe, we can load the original SP
         */
        mfsprg1 %r1
        mtmsr   %r30    /* disable translation */
        isync
        mtspr   SPR_PID,%r31
        mfsprg3 %r31
        mfsprg2 %r30
        IBM405_ERRATA77_SYNC
        rfi
        ba      .               /* Protect against prefetch */

trapleave_to_user:
        lmw     %r14, FRAME_R14(%r1)    /* restore callee registers */

intrleave_to_user:
/* Now restore regs: */
        lwz     %r3,FRAME_PID(%r1)
        lwz     %r4,FRAME_SRR1(%r1)
        bl      _C_LABEL(ctx_setup)
        TRAP_IF_ZERO(%r3)
        stw     %r3,FRAME_PID(%r1)

        FRAME_RESTORE           /* old SP is now in sprg1 */

        /*
         * We are returning to userspace so we need to switch PIDs.
         * Since the kernel executes out of what would be userspace,
         * we need to turn off translation before we set the PID.
         *
         * Alterantively, we could map a kernel page at 0xfffff000
         * that had the mtpid code in it and branch to it and avoid
         * all this.  (ba foo; foo: mtpid %r31; mfsprg3 %r31; rfi;)
         */
        mtsprg2 %r30
        mtsprg3 %r31
        mfmsr   %r30
        li      %r31,(PSL_DR|PSL_IR)@l
        andc    %r30,%r30,%r31
        lwz     %r31,FRAME_PID(%r1)
        TRAP_IF_ZERO(%r31)
        /*
         * Now that we are done with the trapframe, we can load the original SP
         */
        mfsprg1 %r1
        mtmsr   %r30    /* disable translation */
        isync
        mtspr   SPR_PID,%r31
        mfsprg3 %r31
        mfsprg2 %r30
        IBM405_ERRATA77_SYNC
        rfi
        ba      .       /* Protect against prefetch */

#ifdef PPC_IBM440
/*
 * Critical-class (CSRR) and machine-check (MCSRR) s_trap variants 
 * FRAME_RESTORE with the return state going to CSRR/MCSRR ...
 */
#define FRAME_RESTORE_TO(srr0spr, srr1spr)                              \
        lwz     %r6,FRAME_LR(%r1);                                      \
        lwz     %r7,FRAME_CR(%r1);                                      \
        lwz     %r8,FRAME_XER(%r1);                                     \
        lwz     %r9,FRAME_CTR(%r1);                                     \
        lwz     %r10,FRAME_SRR0(%r1);                                   \
        lwz     %r11,FRAME_SRR1(%r1);                                   \
        mtlr    %r6;                                                    \
        mtcr    %r7;                                                    \
        mtxer   %r8;                                                    \
        mtctr   %r9;                                                    \
        mtspr   srr0spr,%r10;                                           \
        mtspr   srr1spr,%r11;                                           \
        lwz     %r13,FRAME_R13(%r1);                                    \
        lwz     %r12,FRAME_R12(%r1);                                    \
        lwz     %r11,FRAME_R11(%r1);                                    \
        lwz     %r10,FRAME_R10(%r1);                                    \
        lwz     %r9,FRAME_R9(%r1);                                      \
        lwz     %r8,FRAME_R8(%r1);                                      \
        lwz     %r7,FRAME_R7(%r1);                                      \
        lwz     %r6,FRAME_R6(%r1);                                      \
        lwz     %r5,FRAME_R5(%r1);                                      \
        lwz     %r4,FRAME_R4(%r1);                                      \
        lwz     %r3,FRAME_R3(%r1);                                      \
        lwz     %r2,FRAME_R2(%r1);                                      \
        lwz     %r0,FRAME_R1(%r1);                                      \
        mtsprg1 %r0;                                                    \
        lwz     %r0,FRAME_R0(%r1)

/* Common tail; ends just before the return-from-interrupt instruction */
#define CRIT_LEAVE_PROLOGUE                                             \
        mtsprg2 %r30;                                                   \
        mtsprg3 %r31;                                                   \
        mfmsr   %r30;                                                   \
        li      %r31,(PSL_DR|PSL_IR)@l;                                 \
        andc    %r30,%r30,%r31;                                         \
        lwz     %r31,FRAME_PID(%r1);                                    \
        TRAP_IF_ZERO(%r31);                                             \
        mfsprg1 %r1;                                                    \
        mtmsr   %r30;   /* translation off (TS=0) */                    \
        isync;                                                          \
        mtspr   SPR_PID,%r31;                                           \
        mfsprg3 %r31;                                                   \
        mfsprg2 %r30

s_ctrap:
        FRAME_SETUP(CI_CRITSAVE)
        /* R31 = CSRR1 */
        wrtee   %r31                    /* restore EE of interrupted ctx */
        addi    %r3,%r1,FRAME_TF
        bl      _C_LABEL(trap)
        wrteei  0
        lmw     %r14,FRAME_R14(%r1)     /* restore callee registers */
        FRAME_RESTORE_TO(SPR_CSRR0, SPR_CSRR1)
        CRIT_LEAVE_PROLOGUE
        rfci
        ba      .       /* Protect against prefetch */

s_mctrap:
        FRAME_SETUP(CI_CRITSAVE)
        /* R31 = MCSRR1; do not re-enable interrupts in a machine check */
        addi    %r3,%r1,FRAME_TF
        bl      _C_LABEL(trap)
        lmw     %r14,FRAME_R14(%r1)     /* restore callee registers */
        FRAME_RESTORE_TO(SPR_MCSRR0, SPR_MCSRR1)
        CRIT_LEAVE_PROLOGUE
        rfmci
        ba      .       /* Protect against prefetch */
#endif /* PPC_IBM440 */

        .globl  _C_LABEL(sctrap),_C_LABEL(scsize),_C_LABEL(sctrapexit)
_C_LABEL(sctrap):
        STANDARD_PROLOG(CI_TEMPSAVE)
        bla     s_sctrap
_C_LABEL(scsize) = .-_C_LABEL(sctrap)

s_sctrap:
        FRAME_SETUP(CI_TEMPSAVE)
/* Now we can recover interrupts again: */
        wrteei  1                       /* Enable interrupts */
/* Call the appropriate syscall handler: */
        addi    %r3,%r1,FRAME_TF
        lwz     %r4,L_PROC(%r13)
        lwz     %r4,P_MD_SYSCALL(%r4)
        mtctr   %r4
        bctrl
_C_LABEL(sctrapexit):
        b       trapexit

/*
 * External interrupt second level handler
 */

#define INTR_SAVE(tempsave)                                             \
/* Save non-volatile registers: */                                      \
        stwu    %r1,-FRAMELEN(%r1);     /* temporarily */               \
        stw     %r0,FRAME_R0(%r1);                                      \
        mfsprg1 %r0;                    /* get original SP */           \
        stw     %r0,FRAME_R1(%r1);      /* and store it */              \
        stw     %r2,FRAME_R2(%r1);                                      \
        stw     %r3,FRAME_R3(%r1);                                      \
        stw     %r4,FRAME_R4(%r1);                                      \
        stw     %r5,FRAME_R5(%r1);                                      \
        stw     %r6,FRAME_R6(%r1);                                      \
        stw     %r7,FRAME_R7(%r1);                                      \
        stw     %r8,FRAME_R8(%r1);                                      \
        stw     %r9,FRAME_R9(%r1);                                      \
        stw     %r10,FRAME_R10(%r1);                                    \
        stw     %r11,FRAME_R11(%r1);                                    \
        stw     %r12,FRAME_R12(%r1);                                    \
        stw     %r13,FRAME_R13(%r1);                                    \
        mfctr   %r31;                                                   \
        stmw    %r28,FRAME_LR(%r1);     /* save LR, CR, XER, CTR */     \
        GET_CPUINFO(%r5);                                               \
        lmw     %r28,(tempsave+CPUSAVE_R28)(%r5); /* restore r28-r31 */ \
        lwz     %r13,CI_CURLWP(%r5);                                    \
        lwz     %r5,CI_IDEPTH(%r5);                                     \
        mfsrr0  %r4;                                                    \
        mfsrr1  %r3;                                                    \
        stw     %r5,FRAME_IDEPTH(%r1);                                  \
        stw     %r4,FRAME_SRR0(%r1);                                    \
        CLEAR_WAIT_STATE(%r3);          /* wake from MSR[WE] idle */    \
        stw     %r3,FRAME_SRR1(%r1);                                    \
/* interrupts are recoverable here, and enable translation */           \
        ENABLE_TRANSLATION(%r0,%r5);                                    \
        stw     %r0,FRAME_PID(%r1);

        .globl  _C_LABEL(extint_call)
extintr:
        INTR_SAVE(CI_TEMPSAVE)
_C_LABEL(extint_call):
        bl      _C_LABEL(extint_call)   /* to be filled in later */

intr_exit:
/* Disable interrupts */
        wrteei  0
        isync

        lwz     %r4,FRAME_SRR1(%r1)
/* Returning to user mode? */
        mtcr    %r4                     /* saved SRR1 */
        bf      MSR_PR,intrleave_to_kernel /* branch if MSR[PR] is false */

        lwz     %r4,L_MD_ASTPENDING(%r13)/* Test AST pending */
        andi.   %r4,%r4,1
        beq     intrleave_to_user

        FRAME_SAVE_CALLEE               /* save rest of callee registers */
        li      %r6,EXC_AST
        stw     %r6,FRAME_EXC(%r1)
        lwz     %r31,FRAME_SRR1(%r1)    /* move SRR1 to R31 */
        b       trapagain

/*
 * PIT interrupt handler.
 */
        .align  5
_C_LABEL(pitint):
        INTR_PROLOG(CI_TEMPSAVE)
        INTR_SAVE(CI_TEMPSAVE)
        addi    %r3,%r1,FRAME_CF        /* clock frame */
        bl      _C_LABEL(decr_intr)
        b       intr_exit

/*
 * FIT interrupt handler.
 */
        .align  5
_C_LABEL(fitint):
        INTR_PROLOG(CI_TEMPSAVE)
        INTR_SAVE(CI_TEMPSAVE)
        addi    %r3,%r1,FRAME_CF        /* clock frame */
        bl      _C_LABEL(stat_intr)
        b       intr_exit

#if defined(DDB) || defined(KGDB)
/*
 * Deliberate entry to ddbtrap
 */
        .globl  _C_LABEL(ddb_trap)
_C_LABEL(ddb_trap):
        mtsprg1 %r1
        GET_CPUINFO(%r4)
        mfmsr   %r3
        stw     %r3,(CI_DDBSAVE+CPUSAVE_SRR1)(%r4)
        wrteei  0                       /* disable interrupts */
        isync
        stmw    %r28,CI_DDBSAVE(%r4)
        mflr    %r28
        stw     %r28,(CI_DDBSAVE+CPUSAVE_SRR0)(%r4)
        li      %r29,EXC_BPT
        mtlr    %r29
        mfcr    %r29

/*
 * Now the ddb/kgdb trap catching code.
 */
ddbtrap:
        FRAME_SETUP(CI_DDBSAVE)
/* Call C trap code: */
        addi    %r3,%r1,FRAME_TF
        bl      _C_LABEL(ddb_trap_glue)
        or.     %r3,%r3,%r3
        beq     trapagain
        b       trapexit
#endif /* DDB || KGDB */

/*
 * The FPU-equipped 440 cores use the common lazy FPU ctx
 */
#ifdef PPC_IBM440
#include <powerpc/powerpc/fpu_subr.S>
#endif