/*-
 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
 * Copyright (C) 2006 Juniper Networks, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
/*-
 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
 * Copyright (C) 1995, 1996 TooLs GmbH.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by TooLs GmbH.
 * 4. The name of TooLs GmbH may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *      from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
 */

/*
 * NOTICE: This is not a standalone file.  to use it, #include it in
 * your port's locore.S, like so:
 *
 *      #include <powerpc/booke/trap_subr.S>
 */

/*
 * SPRG usage notes
 *
 * SPRG0 - pcpu pointer
 * SPRG1 - all interrupts except TLB miss, critical, machine check
 * SPRG2 - critical
 * SPRG3 - machine check
 * SPRG4-6 - scratch
 *
 */

/* Get the per-CPU data structure */
#define GET_CPUINFO(r) mfsprg0 r

#define RES_GRANULE     64
#define RES_LOCK        0       /* offset to the 'lock' word */
#ifdef __powerpc64__
#define RES_RECURSE     8       /* offset to the 'recurse' word */
#else
#define RES_RECURSE     4       /* offset to the 'recurse' word */
#endif

/*
 * Standard interrupt prolog
 *
 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
 * isrr0-1 - save restore registers with CPU state at interrupt time (may be
 *           SRR0-1, CSRR0-1, MCSRR0-1
 *
 * 1. saves in the given savearea:
 *   - R30-31
 *   - DEAR, ESR
 *   - xSRR0-1
 *
 * 2. saves CR -> R30
 *
 * 3. switches to kstack if needed
 *
 * 4. notes:
 *   - R31 can be used as scratch register until a new frame is laid on
 *     the stack with FRAME_SETUP
 *
 *   - potential TLB miss: NO. Saveareas are always acessible via TLB1 
 *     permanent entries, and within this prolog we do not dereference any
 *     locations potentially not in the TLB
 */
#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1)                \
        mtspr   sprg_sp, %r1;           /* Save SP */                   \
        GET_CPUINFO(%r1);               /* Per-cpu structure */         \
        STORE   %r30, (savearea+CPUSAVE_R30)(%r1);                      \
        STORE   %r31, (savearea+CPUSAVE_R31)(%r1);                      \
        mfspr   %r30, SPR_DEAR;                                         \
        mfspr   %r31, SPR_ESR;                                          \
        STORE   %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);               \
        STORE   %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);                \
        mfspr   %r30, isrr0;                                            \
        mfspr   %r31, isrr1;            /* MSR at interrupt time */     \
        STORE   %r30, (savearea+CPUSAVE_SRR0)(%r1);                     \
        STORE   %r31, (savearea+CPUSAVE_SRR1)(%r1);                     \
        isync;                                                          \
        mfspr   %r1, sprg_sp;           /* Restore SP */                \
        mfcr    %r30;                   /* Save CR */                   \
        /* switch to per-thread kstack if intr taken in user mode */    \
        mtcr    %r31;                   /* MSR at interrupt time  */    \
        bf      17, 1f;                                                 \
        GET_CPUINFO(%r1);               /* Per-cpu structure */         \
        LOAD    %r1, PC_CURPCB(%r1);    /* Per-thread kernel stack */   \
1:

#define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1)           \
        mtspr   sprg_sp, %r1;           /* Save SP */                   \
        GET_CPUINFO(%r1);               /* Per-cpu structure */         \
        STORE   %r30, (savearea+CPUSAVE_R30)(%r1);                      \
        STORE   %r31, (savearea+CPUSAVE_R31)(%r1);                      \
        mfspr   %r30, SPR_DEAR;                                         \
        mfspr   %r31, SPR_ESR;                                          \
        STORE   %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);               \
        STORE   %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);                \
        mfspr   %r30, isrr0;                                            \
        mfspr   %r31, isrr1;            /* MSR at interrupt time */     \
        STORE   %r30, (savearea+CPUSAVE_SRR0)(%r1);                     \
        STORE   %r31, (savearea+CPUSAVE_SRR1)(%r1);                     \
        mfspr   %r30, SPR_SRR0;                                         \
        mfspr   %r31, SPR_SRR1;         /* MSR at interrupt time */     \
        STORE   %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1);              \
        STORE   %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1);              \
        isync;                                                          \
        mfspr   %r1, sprg_sp;           /* Restore SP */                \
        mfcr    %r30;                   /* Save CR */                   \
        /* switch to per-thread kstack if intr taken in user mode */    \
        mtcr    %r31;                   /* MSR at interrupt time  */    \
        bf      17, 1f;                                                 \
        GET_CPUINFO(%r1);               /* Per-cpu structure */         \
        LOAD    %r1, PC_CURPCB(%r1);    /* Per-thread kernel stack */   \
1:

/*
 * FRAME_SETUP assumes:
 *      SPRG{1-3}       SP at the time interrupt occurred
 *      savearea        r30-r31, DEAR, ESR, xSRR0-1
 *      r30             CR
 *      r31             scratch
 *      r1              kernel stack
 *
 * sprg_sp - SPRG reg containing SP at the time interrupt occurred
 * savearea - temp save
 * exc - exception number (EXC_xxx)
 *
 * 1. sets a new frame
 * 2. saves in the frame:
 *   - R0, R1 (SP at the time of interrupt), R2, LR, CR
 *   - R3-31 (R30-31 first restored from savearea)
 *   - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
 *
 * Notes:
 * - potential TLB miss: YES, since we make dereferences to kstack, which
 *   can happen not covered (we can have up to two DTLB misses if fortunate
 *   enough i.e. when kstack crosses page boundary and both pages are
 *   untranslated)
 */
#ifdef __powerpc64__
#define SAVE_REGS(r)                                                    \
        std     %r3, FRAME_3+CALLSIZE(r);                               \
        std     %r4, FRAME_4+CALLSIZE(r);                               \
        std     %r5, FRAME_5+CALLSIZE(r);                               \
        std     %r6, FRAME_6+CALLSIZE(r);                               \
        std     %r7, FRAME_7+CALLSIZE(r);                               \
        std     %r8, FRAME_8+CALLSIZE(r);                               \
        std     %r9, FRAME_9+CALLSIZE(r);                               \
        std     %r10, FRAME_10+CALLSIZE(r);                             \
        std     %r11, FRAME_11+CALLSIZE(r);                             \
        std     %r12, FRAME_12+CALLSIZE(r);                             \
        std     %r13, FRAME_13+CALLSIZE(r);                             \
        std     %r14, FRAME_14+CALLSIZE(r);                             \
        std     %r15, FRAME_15+CALLSIZE(r);                             \
        std     %r16, FRAME_16+CALLSIZE(r);                             \
        std     %r17, FRAME_17+CALLSIZE(r);                             \
        std     %r18, FRAME_18+CALLSIZE(r);                             \
        std     %r19, FRAME_19+CALLSIZE(r);                             \
        std     %r20, FRAME_20+CALLSIZE(r);                             \
        std     %r21, FRAME_21+CALLSIZE(r);                             \
        std     %r22, FRAME_22+CALLSIZE(r);                             \
        std     %r23, FRAME_23+CALLSIZE(r);                             \
        std     %r24, FRAME_24+CALLSIZE(r);                             \
        std     %r25, FRAME_25+CALLSIZE(r);                             \
        std     %r26, FRAME_26+CALLSIZE(r);                             \
        std     %r27, FRAME_27+CALLSIZE(r);                             \
        std     %r28, FRAME_28+CALLSIZE(r);                             \
        std     %r29, FRAME_29+CALLSIZE(r);                             \
        std     %r30, FRAME_30+CALLSIZE(r);                             \
        std     %r31, FRAME_31+CALLSIZE(r)
#define LD_REGS(r)                                                      \
        ld      %r3, FRAME_3+CALLSIZE(r);                               \
        ld      %r4, FRAME_4+CALLSIZE(r);                               \
        ld      %r5, FRAME_5+CALLSIZE(r);                               \
        ld      %r6, FRAME_6+CALLSIZE(r);                               \
        ld      %r7, FRAME_7+CALLSIZE(r);                               \
        ld      %r8, FRAME_8+CALLSIZE(r);                               \
        ld      %r9, FRAME_9+CALLSIZE(r);                               \
        ld      %r10, FRAME_10+CALLSIZE(r);                             \
        ld      %r11, FRAME_11+CALLSIZE(r);                             \
        ld      %r12, FRAME_12+CALLSIZE(r);                             \
        ld      %r13, FRAME_13+CALLSIZE(r);                             \
        ld      %r14, FRAME_14+CALLSIZE(r);                             \
        ld      %r15, FRAME_15+CALLSIZE(r);                             \
        ld      %r16, FRAME_16+CALLSIZE(r);                             \
        ld      %r17, FRAME_17+CALLSIZE(r);                             \
        ld      %r18, FRAME_18+CALLSIZE(r);                             \
        ld      %r19, FRAME_19+CALLSIZE(r);                             \
        ld      %r20, FRAME_20+CALLSIZE(r);                             \
        ld      %r21, FRAME_21+CALLSIZE(r);                             \
        ld      %r22, FRAME_22+CALLSIZE(r);                             \
        ld      %r23, FRAME_23+CALLSIZE(r);                             \
        ld      %r24, FRAME_24+CALLSIZE(r);                             \
        ld      %r25, FRAME_25+CALLSIZE(r);                             \
        ld      %r26, FRAME_26+CALLSIZE(r);                             \
        ld      %r27, FRAME_27+CALLSIZE(r);                             \
        ld      %r28, FRAME_28+CALLSIZE(r);                             \
        ld      %r29, FRAME_29+CALLSIZE(r);                             \
        ld      %r30, FRAME_30+CALLSIZE(r);                             \
        ld      %r31, FRAME_31+CALLSIZE(r)
#else
#define SAVE_REGS(r)                                                    \
        stmw    %r3,  FRAME_3+CALLSIZE(r)
#define LD_REGS(r)                                                      \
        lmw     %r3,  FRAME_3+CALLSIZE(r)
#endif
#define FRAME_SETUP(sprg_sp, savearea, exc)                             \
        mfspr   %r31, sprg_sp;          /* get saved SP */              \
        /* establish a new stack frame and put everything on it */      \
        STU     %r31, -(FRAMELEN+REDZONE)(%r1);                         \
        STORE   %r0, FRAME_0+CALLSIZE(%r1);     /* save r0 in the trapframe */  \
        STORE   %r31, FRAME_1+CALLSIZE(%r1);    /* save SP   "     " */ \
        STORE   %r2, FRAME_2+CALLSIZE(%r1);     /* save r2   "     " */ \
        mflr    %r31;                                                   \
        STORE   %r31, FRAME_LR+CALLSIZE(%r1);   /* save LR   "     " */ \
        STORE   %r30, FRAME_CR+CALLSIZE(%r1);   /* save CR   "     " */ \
        GET_CPUINFO(%r2);                                               \
        LOAD    %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */  \
        LOAD    %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */  \
        /* save R3-31 */                                                \
        SAVE_REGS(%r1);                                                 \
        /* save DEAR, ESR */                                            \
        LOAD    %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2);               \
        LOAD    %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2);                \
        STORE   %r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1);                   \
        STORE   %r29, FRAME_BOOKE_ESR+CALLSIZE(%r1);                    \
        /* save XER, CTR, exc number */                                 \
        mfxer   %r3;                                                    \
        mfctr   %r4;                                                    \
        STORE   %r3, FRAME_XER+CALLSIZE(%r1);                           \
        STORE   %r4, FRAME_CTR+CALLSIZE(%r1);                           \
        li      %r5, exc;                                               \
        STORE   %r5, FRAME_EXC+CALLSIZE(%r1);                           \
        /* save DBCR0 */                                                \
        mfspr   %r3, SPR_DBCR0;                                         \
        STORE   %r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);                   \
        /* save xSSR0-1 */                                              \
        LOAD    %r30, (savearea+CPUSAVE_SRR0)(%r2);                     \
        LOAD    %r31, (savearea+CPUSAVE_SRR1)(%r2);                     \
        STORE   %r30, FRAME_SRR0+CALLSIZE(%r1);                         \
        STORE   %r31, FRAME_SRR1+CALLSIZE(%r1);                         \
        LOAD    THREAD_REG, PC_CURTHREAD(%r2);                          \

/*
 *
 * isrr0-1 - save restore registers to restore CPU state to (may be
 *           SRR0-1, CSRR0-1, MCSRR0-1
 *
 * Notes:
 *  - potential TLB miss: YES. The deref'd kstack may be not covered
 */
#define FRAME_LEAVE(isrr0, isrr1)                                       \
        wrteei 0;                                                       \
        /* restore CTR, XER, LR, CR */                                  \
        LOAD    %r4, FRAME_CTR+CALLSIZE(%r1);                           \
        LOAD    %r5, FRAME_XER+CALLSIZE(%r1);                           \
        LOAD    %r6, FRAME_LR+CALLSIZE(%r1);                            \
        LOAD    %r7, FRAME_CR+CALLSIZE(%r1);                            \
        mtctr   %r4;                                                    \
        mtxer   %r5;                                                    \
        mtlr    %r6;                                                    \
        mtcr    %r7;                                                    \
        /* restore DBCR0 */                                             \
        LOAD    %r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);                   \
        mtspr   SPR_DBCR0, %r4;                                         \
        /* restore xSRR0-1 */                                           \
        LOAD    %r30, FRAME_SRR0+CALLSIZE(%r1);                         \
        LOAD    %r31, FRAME_SRR1+CALLSIZE(%r1);                         \
        mtspr   isrr0, %r30;                                            \
        mtspr   isrr1, %r31;                                            \
        /* restore R2-31, SP */                                         \
        LD_REGS(%r1);                                                   \
        LOAD    %r2, FRAME_2+CALLSIZE(%r1);                             \
        LOAD    %r0, FRAME_0+CALLSIZE(%r1);                             \
        LOAD    %r1, FRAME_1+CALLSIZE(%r1);                             \
        isync

/*
 * TLB miss prolog
 *
 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
 *
 * Notes:
 *  - potential TLB miss: NO. It is crucial that we do not generate a TLB
 *    miss within the TLB prolog itself!
 *  - TLBSAVE is always translated
 */
#ifdef __powerpc64__
#define TLB_SAVE_REGS(br)                                               \
        std     %r20, (TLBSAVE_BOOKE_R20)(br);                          \
        std     %r21, (TLBSAVE_BOOKE_R21)(br);                          \
        std     %r22, (TLBSAVE_BOOKE_R22)(br);                          \
        std     %r23, (TLBSAVE_BOOKE_R23)(br);                          \
        std     %r24, (TLBSAVE_BOOKE_R24)(br);                          \
        std     %r25, (TLBSAVE_BOOKE_R25)(br);                          \
        std     %r26, (TLBSAVE_BOOKE_R26)(br);                          \
        std     %r27, (TLBSAVE_BOOKE_R27)(br);                          \
        std     %r28, (TLBSAVE_BOOKE_R28)(br);                          \
        std     %r29, (TLBSAVE_BOOKE_R29)(br);                          \
        std     %r30, (TLBSAVE_BOOKE_R30)(br);                          \
        std     %r31, (TLBSAVE_BOOKE_R31)(br);                          
#define TLB_RESTORE_REGS(br)                                            \
        ld      %r20, (TLBSAVE_BOOKE_R20)(br);                          \
        ld      %r21, (TLBSAVE_BOOKE_R21)(br);                          \
        ld      %r22, (TLBSAVE_BOOKE_R22)(br);                          \
        ld      %r23, (TLBSAVE_BOOKE_R23)(br);                          \
        ld      %r24, (TLBSAVE_BOOKE_R24)(br);                          \
        ld      %r25, (TLBSAVE_BOOKE_R25)(br);                          \
        ld      %r26, (TLBSAVE_BOOKE_R26)(br);                          \
        ld      %r27, (TLBSAVE_BOOKE_R27)(br);                          \
        ld      %r28, (TLBSAVE_BOOKE_R28)(br);                          \
        ld      %r29, (TLBSAVE_BOOKE_R29)(br);                          \
        ld      %r30, (TLBSAVE_BOOKE_R30)(br);                          \
        ld      %r31, (TLBSAVE_BOOKE_R31)(br);                          
#define TLB_NEST(outr,inr)                                              \
        rlwinm  outr, inr, 7, 23, 24;   /* 8 x TLBSAVE_LEN */
#else
#define TLB_SAVE_REGS(br)                                               \
        stmw    %r20, TLBSAVE_BOOKE_R20(br)
#define TLB_RESTORE_REGS(br)                                            \
        lmw     %r20, TLBSAVE_BOOKE_R20(br)
#define TLB_NEST(outr,inr)                                              \
        rlwinm  outr, inr, 6, 24, 25;   /* 4 x TLBSAVE_LEN */
#endif
#define TLB_PROLOG                                                      \
        mtspr   SPR_SPRG4, %r1;                 /* Save SP */           \
        mtspr   SPR_SPRG5, %r28;                                        \
        mtspr   SPR_SPRG6, %r29;                                        \
        /* calculate TLB nesting level and TLBSAVE instance address */  \
        GET_CPUINFO(%r1);               /* Per-cpu structure */         \
        LOAD    %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
        TLB_NEST(%r29,%r28);                                            \
        addi    %r28, %r28, 1;                                          \
        STORE   %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
        addi    %r29, %r29, PC_BOOKE_TLBSAVE@l;                         \
        add     %r1, %r1, %r29;         /* current TLBSAVE ptr */       \
                                                                        \
        /* save R20-31 */                                               \
        mfspr   %r28, SPR_SPRG5;                                        \
        mfspr   %r29, SPR_SPRG6;                                        \
        TLB_SAVE_REGS(%r1);                     \
        /* save LR, CR */                                               \
        mflr    %r30;                                                   \
        mfcr    %r31;                                                   \
        STORE   %r30, (TLBSAVE_BOOKE_LR)(%r1);                          \
        STORE   %r31, (TLBSAVE_BOOKE_CR)(%r1);                          \
        /* save SRR0-1 */                                               \
        mfsrr0  %r30;           /* execution addr at interrupt time */  \
        mfsrr1  %r31;           /* MSR at interrupt time*/              \
        STORE   %r30, (TLBSAVE_BOOKE_SRR0)(%r1);        /* save SRR0 */ \
        STORE   %r31, (TLBSAVE_BOOKE_SRR1)(%r1);        /* save SRR1 */ \
        isync;                                                          \
        mfspr   %r1, SPR_SPRG4

/*
 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
 *
 * same notes as for the TLB_PROLOG
 */
#define TLB_RESTORE                                                     \
        mtspr   SPR_SPRG4, %r1;                 /* Save SP */           \
        GET_CPUINFO(%r1);               /* Per-cpu structure */         \
        /* calculate TLB nesting level and TLBSAVE instance addr */     \
        LOAD    %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
        subi    %r28, %r28, 1;                                          \
        STORE   %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
        TLB_NEST(%r29,%r28);                                            \
        addi    %r29, %r29, PC_BOOKE_TLBSAVE@l;                         \
        add     %r1, %r1, %r29;                                         \
                                                                        \
        /* restore LR, CR */                                            \
        LOAD    %r30, (TLBSAVE_BOOKE_LR)(%r1);                          \
        LOAD    %r31, (TLBSAVE_BOOKE_CR)(%r1);                          \
        mtlr    %r30;                                                   \
        mtcr    %r31;                                                   \
        /* restore SRR0-1 */                                            \
        LOAD    %r30, (TLBSAVE_BOOKE_SRR0)(%r1);                        \
        LOAD    %r31, (TLBSAVE_BOOKE_SRR1)(%r1);                        \
        mtsrr0  %r30;                                                   \
        mtsrr1  %r31;                                                   \
        /* restore R20-31 */                                            \
        TLB_RESTORE_REGS(%r1);                                          \
        mfspr   %r1, SPR_SPRG4

#ifdef SMP
#define TLB_LOCK                                                        \
        GET_CPUINFO(%r20);                                              \
        LOAD    %r21, PC_CURTHREAD(%r20);                               \
        LOAD    %r22, PC_BOOKE_TLB_LOCK(%r20);                          \
                                                                        \
1:      LOADX   %r23, 0, %r22;                                          \
        CMPI    %r23, TLB_UNLOCKED;                                     \
        beq     2f;                                                     \
                                                                        \
        /* check if this is recursion */                                \
        CMPL    cr0, %r21, %r23;                                        \
        bne-    1b;                                                     \
                                                                        \
2:      /* try to acquire lock */                                       \
        STOREX  %r21, 0, %r22;                                          \
        bne-    1b;                                                     \
                                                                        \
        /* got it, update recursion counter */                          \
        lwz     %r21, RES_RECURSE(%r22);                                \
        addi    %r21, %r21, 1;                                          \
        stw     %r21, RES_RECURSE(%r22);                                \
        isync;                                                          \
        msync

#define TLB_UNLOCK                                                      \
        GET_CPUINFO(%r20);                                              \
        LOAD    %r21, PC_CURTHREAD(%r20);                               \
        LOAD    %r22, PC_BOOKE_TLB_LOCK(%r20);                          \
                                                                        \
        /* update recursion counter */                                  \
        lwz     %r23, RES_RECURSE(%r22);                                \
        subi    %r23, %r23, 1;                                          \
        stw     %r23, RES_RECURSE(%r22);                                \
                                                                        \
        cmplwi  %r23, 0;                                                \
        bne     1f;                                                     \
        isync;                                                          \
        msync;                                                          \
                                                                        \
        /* release the lock */                                          \
        li      %r23, TLB_UNLOCKED;                                     \
        STORE   %r23, 0(%r22);                                          \
1:      isync;                                                          \
        msync
#else
#define TLB_LOCK
#define TLB_UNLOCK
#endif  /* SMP */

#define INTERRUPT(label)                                                \
        .globl  label;                                                  \
        .align  5;                                                      \
        CNAME(label):

/*
 * Interrupt handling routines in BookE can be flexibly placed and do not have
 * to live in pre-defined vectors location. Note they need to be TLB-mapped at
 * all times in order to be able to handle exceptions. We thus arrange for
 * them to be part of kernel text which is always TLB-accessible.
 *
 * The interrupt handling routines have to be 16 bytes aligned: we align them
 * to 32 bytes (cache line length) which supposedly performs better.
 *
 */
        .text
        .globl CNAME(interrupt_vector_base)
        .align 5
interrupt_vector_base:
/*****************************************************************************
 * Catch-all handler to handle uninstalled IVORs
 ****************************************************************************/
INTERRUPT(int_unknown)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD)
        b       trap_common

/*****************************************************************************
 * Critical input interrupt
 ****************************************************************************/
INTERRUPT(int_critical_input)
        STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
        FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
        GET_TOCBASE(%r2)
        addi    %r3, %r1, CALLSIZE
        bl      CNAME(powerpc_interrupt)
        TOC_RESTORE
        FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
        rfci


/*****************************************************************************
 * Machine check interrupt
 ****************************************************************************/
INTERRUPT(int_machine_check)
        STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
        FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
        GET_TOCBASE(%r2)
        addi    %r3, %r1, CALLSIZE
        bl      CNAME(powerpc_interrupt)
        TOC_RESTORE
        FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
        rfmci


/*****************************************************************************
 * Data storage interrupt
 ****************************************************************************/
INTERRUPT(int_data_storage)
        STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
        b       trap_common


/*****************************************************************************
 * Instruction storage interrupt
 ****************************************************************************/
INTERRUPT(int_instr_storage)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
        b       trap_common


/*****************************************************************************
 * External input interrupt
 ****************************************************************************/
INTERRUPT(int_external_input)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
        b       trap_common


INTERRUPT(int_alignment)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
        b       trap_common


INTERRUPT(int_program)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
        b       trap_common


INTERRUPT(int_fpu)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU)
        b       trap_common


/*****************************************************************************
 * System call
 ****************************************************************************/
INTERRUPT(int_syscall)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
        b       trap_common


/*****************************************************************************
 * Decrementer interrupt
 ****************************************************************************/
INTERRUPT(int_decrementer)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
        b       trap_common


/*****************************************************************************
 * Fixed interval timer
 ****************************************************************************/
INTERRUPT(int_fixed_interval_timer)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
        b       trap_common


/*****************************************************************************
 * Watchdog interrupt
 ****************************************************************************/
INTERRUPT(int_watchdog)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
        b       trap_common


/*****************************************************************************
 * Altivec Unavailable interrupt
 ****************************************************************************/
INTERRUPT(int_vec)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC)
        b       trap_common


/*****************************************************************************
 * Altivec Assist interrupt
 ****************************************************************************/
INTERRUPT(int_vecast)
        STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E)
        b       trap_common


#ifdef HWPMC_HOOKS
/*****************************************************************************
 * PMC Interrupt
 ****************************************************************************/
INTERRUPT(int_performance_counter)
        STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
        FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
        b       trap_common
#endif


/*****************************************************************************
 * Data TLB miss interrupt
 *
 * There can be nested TLB misses - while handling a TLB miss we reference
 * data structures that may be not covered by translations. We support up to
 * TLB_NESTED_MAX-1 nested misses.
 * 
 * Registers use:
 *      r31 - dear
 *      r30 - unused
 *      r29 - saved mas0
 *      r28 - saved mas1
 *      r27 - saved mas2
 *      r26 - pmap address
 *      r25 - pte address
 *
 *      r20:r23 - scratch registers
 ****************************************************************************/
INTERRUPT(int_data_tlb_error)
        TLB_PROLOG
        TLB_LOCK

        mfspr   %r31, SPR_DEAR

        /*
         * Save MAS0-MAS2 registers. There might be another tlb miss during
         * pte lookup overwriting current contents (which was hw filled).
         */
        mfspr   %r29, SPR_MAS0
        mfspr   %r28, SPR_MAS1
        mfspr   %r27, SPR_MAS2

        /* Check faulting address. */
        LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS)
        CMPL    cr0, %r31, %r21
        blt     search_user_pmap
        
        /* If it's kernel address, allow only supervisor mode misses. */
        mfsrr1  %r21
        mtcr    %r21
        bt      17, search_failed       /* check MSR[PR] */

#ifdef __powerpc64__
        srdi    %r21, %r31, 48
        cmpldi  cr0, %r21, VM_MIN_KERNEL_ADDRESS@highest
#else
        lis     %r21, VM_MIN_KERNEL_ADDRESS@h
        cmplw   cr0, %r31, %r21
#endif
        blt     search_failed

search_kernel_pmap:
        /* Load r26 with kernel_pmap address */
        bl      1f
#ifdef __powerpc64__
        .llong kernel_pmap_store-.
#else
        .long kernel_pmap_store-.
#endif
1:      mflr    %r21
        LOAD    %r26, 0(%r21)
        add     %r26, %r21, %r26        /* kernel_pmap_store in r26 */

        /* Force kernel tid, set TID to 0 in MAS1. */
        li      %r21, 0
        rlwimi  %r28, %r21, 0, 8, 15    /* clear TID bits */

tlb_miss_handle:
        /* This may result in nested tlb miss. */
        bl      pte_lookup              /* returns PTE address in R25 */

        CMPI    %r25, 0                 /* pte found? */
        beq     search_failed

        /* Finish up, write TLB entry. */
        bl      tlb_fill_entry

tlb_miss_return:
        TLB_UNLOCK
        TLB_RESTORE
        rfi

search_user_pmap:
        /* Load r26 with current user space process pmap */
        GET_CPUINFO(%r26)
        LOAD    %r26, PC_CURPMAP(%r26)

        b       tlb_miss_handle

search_failed:
        /*
         * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
         * the faulting virtual address anyway, but put a fake RPN and no
         * access rights. This should cause a following {D,I}SI exception.
         */
        lis     %r23, 0xffff0000@h      /* revoke all permissions */

        /* Load MAS registers. */
        mtspr   SPR_MAS0, %r29
        mtspr   SPR_MAS1, %r28
        mtspr   SPR_MAS2, %r27
        mtspr   SPR_MAS3, %r23

        li      %r23, 0
        mtspr   SPR_MAS7, %r23

        isync
        tlbwe
        msync
        isync
        b       tlb_miss_return

/*****************************************************************************
 *
 * Return pte address that corresponds to given pmap/va.  If there is no valid
 * entry return 0.
 *
 * input: r26 - pmap
 * input: r31 - dear
 * output: r25 - pte address
 *
 * scratch regs used: r21
 *
 ****************************************************************************/
pte_lookup:
        CMPI    %r26, 0
        beq     1f                      /* fail quickly if pmap is invalid */

#ifdef __powerpc64__
        rldicl  %r21, %r31, (64 - PG_ROOT_L), (64 - PG_ROOT_NUM) /* pp2d offset */
        slwi    %r21, %r21, PG_ROOT_ENTRY_SHIFT /* multiply by pp2d entry size */
        ld      %r25, PM_ROOT(%r26)             /* pmap pm_pp2d[] address */
        ldx     %r25, %r25, %r21                /* get pdir address, i.e.  pmap->pm_pp2d[pp2d_idx] * */

        cmpdi   %r25, 0
        beq 2f

        rldicl  %r21, %r31, (64 - PDIR_L1_L), (64 - PDIR_L1_NUM) /* pp2d offset */
        slwi    %r21, %r21, PDIR_L1_ENTRY_SHIFT /* multiply by pp2d entry size */
        ldx     %r25, %r25, %r21                /* get pdir address, i.e.  pmap->pm_pp2d[pp2d_idx] * */

        cmpdi   %r25, 0
        beq 2f

        rldicl  %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM)      /* pdir offset */
        slwi    %r21, %r21, PDIR_ENTRY_SHIFT    /* multiply by pdir entry size */
        ldx     %r25, %r25, %r21                /* get ptbl address, i.e.  pmap->pm_pp2d[pp2d_idx][pdir_idx] */

        cmpdi   %r25, 0
        beq     2f

        rldicl  %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */
        slwi    %r21, %r21, PTBL_ENTRY_SHIFT   /* multiply by pte entry size */

#else
        srwi    %r21, %r31, PDIR_SHIFT          /* pdir offset */
        slwi    %r21, %r21, PDIR_ENTRY_SHIFT    /* multiply by pdir entry size */

        lwz     %r25, PM_PDIR(%r26)     /* pmap pm_dir[] address */
        /*
         * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
         * This load may cause a Data TLB miss for non-kernel pmap!
         */
        lwzx    %r25, %r25, %r21        /* offset within pm_pdir[] table */
        cmpwi   %r25, 0
        beq     2f

        lis     %r21, PTBL_MASK@h
        ori     %r21, %r21, PTBL_MASK@l
        and     %r21, %r21, %r31

        /* ptbl offset, multiply by ptbl entry size */
        srwi    %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
#endif

        add     %r25, %r25, %r21                /* address of pte entry */
        /*
         * Get pte->flags
         * This load may cause a Data TLB miss for non-kernel pmap!
         */
        lwz     %r21, PTE_FLAGS(%r25)
        andi.   %r21, %r21, PTE_VALID@l
        bne     2f
1:
        li      %r25, 0
2:
        blr

/*****************************************************************************
 *
 * Load MAS1-MAS3 registers with data, write TLB entry
 *
 * input:
 * r29 - mas0
 * r28 - mas1
 * r27 - mas2
 * r25 - pte
 *
 * output: none
 *
 * scratch regs: r21-r23
 *
 ****************************************************************************/
tlb_fill_entry:
        /*
         * Update PTE flags: we have to do it atomically, as pmap_protect()
         * running on other CPUs could attempt to update the flags at the same
         * time.
         */
        li      %r23, PTE_FLAGS
1:
        lwarx   %r21, %r23, %r25                /* get pte->flags */
        oris    %r21, %r21, PTE_REFERENCED@h    /* set referenced bit */

        andi.   %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */
        beq     2f
        ori     %r21, %r21, PTE_MODIFIED@l      /* set modified bit */
2:
        stwcx.  %r21, %r23, %r25                /* write it back */
        bne-    1b

        /* Update MAS2. */
        rlwimi  %r27, %r21, 13, 27, 30          /* insert WIMG bits from pte */

        /* Setup MAS3 value in r23. */
        LOAD    %r23, PTE_RPN(%r25)             /* get pte->rpn */
#ifdef __powerpc64__
        rldicr  %r22, %r23, 52, 51              /* extract MAS3 portion of RPN */
        rldicl  %r23, %r23, 20, 54              /* extract MAS7 portion of RPN */

        rlwimi  %r22, %r21, 30, 26, 31          /* insert protection bits from pte */
#else
        rlwinm  %r22, %r23, 20, 0, 11           /* extract MAS3 portion of RPN */

        rlwimi  %r22, %r21, 30, 26, 31          /* insert protection bits from pte */
        rlwimi  %r22, %r21, 20, 12, 19          /* insert lower 8 RPN bits to MAS3 */
        rlwinm  %r23, %r23, 20, 24, 31          /* MAS7 portion of RPN */
#endif

        /* Load MAS registers. */
        mtspr   SPR_MAS0, %r29
        mtspr   SPR_MAS1, %r28
        mtspr   SPR_MAS2, %r27
        mtspr   SPR_MAS3, %r22
        mtspr   SPR_MAS7, %r23

        isync
        tlbwe
        isync
        msync
        blr

/*****************************************************************************
 * Instruction TLB miss interrupt
 *
 * Same notes as for the Data TLB miss
 ****************************************************************************/
INTERRUPT(int_inst_tlb_error)
        TLB_PROLOG
        TLB_LOCK

        mfsrr0  %r31                    /* faulting address */

        /*
         * Save MAS0-MAS2 registers. There might be another tlb miss during pte
         * lookup overwriting current contents (which was hw filled).
         */
        mfspr   %r29, SPR_MAS0
        mfspr   %r28, SPR_MAS1
        mfspr   %r27, SPR_MAS2

        mfsrr1  %r21
        mtcr    %r21

        /* check MSR[PR] */
        bt      17, search_user_pmap
        b       search_kernel_pmap


        .globl  interrupt_vector_top
interrupt_vector_top:

/*****************************************************************************
 * Debug interrupt
 ****************************************************************************/
INTERRUPT(int_debug)
        STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
        FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
        bl      int_debug_int
        FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
        rfci

INTERRUPT(int_debug_ed)
        STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1)
        FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
        bl      int_debug_int
        FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1)
        rfdi
        /* .long 0x4c00004e */

/* Internal helper for debug interrupt handling. */
/* Common code between e500v1/v2 and e500mc-based cores. */
int_debug_int:
        mflr    %r14
        GET_CPUINFO(%r3)
        LOAD    %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
        bl      0f
        ADDR(interrupt_vector_base-.)
        ADDR(interrupt_vector_top-.)
0:      mflr    %r5
        LOAD    %r4,0(%r5)      /* interrupt_vector_base in r4 */
        add     %r4,%r4,%r5
        CMPL    cr0, %r3, %r4
        blt     trap_common
        LOAD    %r4,WORD_SIZE(%r5)      /* interrupt_vector_top in r4 */
        add     %r4,%r4,%r5
        addi    %r4,%r4,4
        CMPL    cr0, %r3, %r4
        bge     trap_common
        /* Disable single-stepping for the interrupt handlers. */
        LOAD    %r3, FRAME_SRR1+CALLSIZE(%r1);
        rlwinm  %r3, %r3, 0, 23, 21
        STORE   %r3, FRAME_SRR1+CALLSIZE(%r1);
        /* Restore srr0 and srr1 as they could have been clobbered. */
        GET_CPUINFO(%r4)
        LOAD    %r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4);
        mtspr   SPR_SRR0, %r3
        LOAD    %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4);
        mtspr   SPR_SRR1, %r4
        mtlr    %r14
        blr

/*****************************************************************************
 * Common trap code
 ****************************************************************************/
trap_common:
        /* Call C trap dispatcher */
        GET_TOCBASE(%r2)
        addi    %r3, %r1, CALLSIZE
        bl      CNAME(powerpc_interrupt)
        TOC_RESTORE

        .globl  CNAME(trapexit)         /* exported for db_backtrace use */
CNAME(trapexit):
        /* disable interrupts */
        wrteei  0

        /* Test AST pending - makes sense for user process only */
        LOAD    %r5, FRAME_SRR1+CALLSIZE(%r1)
        mtcr    %r5
        bf      17, 1f

        GET_CPUINFO(%r3)
        LOAD    %r4, PC_CURTHREAD(%r3)
        lwz     %r4, TD_AST(%r4)
        cmpwi   %r4, 0
        beq     1f

        /* re-enable interrupts before calling ast() */
        wrteei  1

        addi    %r3, %r1, CALLSIZE
        bl      CNAME(ast)
        TOC_RESTORE
        .globl  CNAME(asttrapexit)      /* db_backtrace code sentinel #2 */
CNAME(asttrapexit):
        b       trapexit                /* test ast ret value ? */
1:
        FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
        rfi


#if defined(KDB)
/*
 * Deliberate entry to dbtrap
 */
        /* .globl       CNAME(breakpoint)*/
ASENTRY_NOPROF(breakpoint)
        mtsprg1 %r1
        mfmsr   %r3
        mtsrr1  %r3
        li      %r4, ~(PSL_EE | PSL_ME)@l
        oris    %r4, %r4, ~(PSL_EE | PSL_ME)@h
        and     %r3, %r3, %r4
        mtmsr   %r3                     /* disable interrupts */
        isync
        GET_CPUINFO(%r3)
        STORE   %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
        STORE   %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)

        mflr    %r31
        mtsrr0  %r31

        mfspr   %r30, SPR_DEAR
        mfspr   %r31, SPR_ESR
        STORE   %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
        STORE   %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)

        mfsrr0  %r30
        mfsrr1  %r31
        STORE   %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
        STORE   %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
        isync

        mfcr    %r30

/*
 * Now the kdb trap catching code.
 */
dbtrap:
        FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
/* Call C trap code: */
        GET_TOCBASE(%r2)
        addi    %r3, %r1, CALLSIZE
        bl      CNAME(db_trap_glue)
        TOC_RESTORE
        or.     %r3, %r3, %r3
        bne     dbleave
/* This wasn't for KDB, so switch to real trap: */
        b       trap_common

dbleave:
        FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
        rfi
ASEND(breakpoint)
#endif /* KDB */

#ifdef SMP
ENTRY(tlb_lock)
        GET_CPUINFO(%r5)
        LOAD    %r5, PC_CURTHREAD(%r5)
1:      LOADX   %r4, 0, %r3
        CMPI    %r4, TLB_UNLOCKED
        bne     1b
        STOREX  %r5, 0, %r3
        bne-    1b
        isync
        msync
        blr
END(tlb_lock)

ENTRY(tlb_unlock)
        isync
        msync
        li      %r4, TLB_UNLOCKED
        STORE   %r4, 0(%r3)
        isync
        msync
        blr
END(tlb_unlock)

/*
 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
 * only a single word from this granule will actually be used as a spin lock
 * for mutual exclusion between TLB miss handler and pmap layer that
 * manipulates page table contents.
 */
        .data
        .align  5
GLOBAL(tlb0_miss_locks)
        .space  RES_GRANULE * MAXCPU
#endif